diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2022-03-25 02:19:43 +0300 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2022-03-25 02:19:43 +0300 |
| commit | b14ffae378aa1db993e62b01392e70d1e585fb23 (patch) | |
| tree | 0ac179d24e8a62ec4c2732ed18d90d83da4b82d7 /drivers/gpu/drm/i915 | |
| parent | 52deda9551a01879b3562e7b41748e85c591f14c (diff) | |
| parent | c6e90a1c660874736bd09c1fec6312b4b4c2ff7b (diff) | |
| download | linux-b14ffae378aa1db993e62b01392e70d1e585fb23.tar.xz | |
Merge tag 'drm-next-2022-03-24' of git://anongit.freedesktop.org/drm/drm
Pull drm updates from Dave Airlie:
"Lots of work all over, Intel improving DG2 support, amdkfd CRIU
support, msm new hw support, and faster fbdev support.
dma-buf:
- rename dma-buf-map to iosys-map
core:
- move buddy allocator to core
- add pci/platform init macros
- improve EDID parser deep color handling
- EDID timing type 7 support
- add GPD Win Max quirk
- add yes/no helpers to string_helpers
- flatten syncobj chains
- add nomodeset support to lots of drivers
- improve fb-helper clipping support
- add default property value interface
fbdev:
- improve fbdev ops speed
ttm:
- add a backpointer from ttm bo->ttm resource
dp:
- move displayport headers
- add a dp helper module
bridge:
- anx7625 atomic support, HDCP support
panel:
- split out panel-lvds and lvds bindings
- find panels in OF subnodes
privacy:
- add chromeos privacy screen support
fb:
- hot unplug fw fb on forced removal
simpledrm:
- request region instead of marking ioresource busy
- add panel oreintation property
udmabuf:
- fix oops with 0 pages
amdgpu:
- power management code cleanup
- Enable freesync video mode by default
- RAS code cleanup
- Improve VRAM access for debug using SDMA
- SR-IOV rework special register access and fixes
- profiling power state request ioctl
- expose IP discovery via sysfs
- Cyan skillfish updates
- GC 10.3.7, SDMA 5.2.7, DCN 3.1.6 updates
- expose benchmark tests via debugfs
- add module param to disable XGMI for testing
- GPU reset debugfs register dumping support
amdkfd:
- CRIU support
- SDMA queue fixes
radeon:
- UVD suspend fix
- iMac backlight fix
i915:
- minimal parallel submission for execlists
- DG2-G12 subplatform added
- DG2 programming workarounds
- DG2 accelerated migration support
- flat CCS and CCS engine support for XeHP
- initial small BAR support
- drop fake LMEM support
- ADL-N PCH support
- bigjoiner updates
- introduce VMA resources and async unbinding
- register definitions cleanups
- multi-FBC refactoring
- DG1 OPROM over SPI support
- ADL-N platform enabling
- opregion mailbox #5 support
- DP MST ESI improvements
- drm device based logging
- async flip optimisation for DG2
- CPU arch abstraction fixes
- improve GuC ADS init to work on aarch64
- tweak TTM LRU priority hint
- GuC 69.0.3 support
- remove short term execbuf pins
nouveau:
- higher DP/eDP bitrates
- backlight fixes
msm:
- dpu + dp support for sc8180x
- dp support for sm8350
- dpu + dsi support for qcm2290
- 10nm dsi phy tuning support
- bridge support for dp encoder
- gpu support for additional 7c3 SKUs
ingenic:
- HDMI support for JZ4780
- aux channel EDID support
ast:
- AST2600 support
- add wide screen support
- create DP/DVI connectors
omapdrm:
- fix implicit dma_buf fencing
vc4:
- add CSC + full range support
- better display firmware handoff
panfrost:
- add initial dual-core GPU support
stm:
- new revision support
- fb handover support
mediatek:
- transfer display binding document to yaml format.
- add mt8195 display device binding.
- allow commands to be sent during video mode.
- add wait_for_event for crtc disable by cmdq.
tegra:
- YUV format support
rcar-du:
- LVDS support for M3-W+ (R8A77961)
exynos:
- BGR pixel format for FIMD device"
* tag 'drm-next-2022-03-24' of git://anongit.freedesktop.org/drm/drm: (1529 commits)
drm/i915/display: Do not re-enable PSR after it was marked as not reliable
drm/i915/display: Fix HPD short pulse handling for eDP
drm/amdgpu: Use drm_mode_copy()
drm/radeon: Use drm_mode_copy()
drm/amdgpu: Use ternary operator in `vcn_v1_0_start()`
drm/amdgpu: Remove pointless on stack mode copies
drm/amd/pm: fix indenting in __smu_cmn_reg_print_error()
drm/amdgpu/dc: fix typos in comments
drm/amdgpu: fix typos in comments
drm/amd/pm: fix typos in comments
drm/amdgpu: Add stolen reserved memory for MI25 SRIOV.
drm/amdgpu: Merge get_reserved_allocation to get_vbios_allocations.
drm/amdkfd: evict svm bo worker handle error
drm/amdgpu/vcn: fix vcn ring test failure in igt reload test
drm/amdgpu: only allow secure submission on rings which support that
drm/amdgpu: fixed the warnings reported by kernel test robot
drm/amd/display: 3.2.177
drm/amd/display: [FW Promotion] Release 0.0.108.0
drm/amd/display: Add save/restore PANEL_PWRSEQ_REF_DIV2
drm/amd/display: Wait for hubp read line for Pollock
...
Diffstat (limited to 'drivers/gpu/drm/i915')
335 files changed, 14797 insertions, 11852 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index cfd932514da2..98c5450b8eac 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -3,12 +3,14 @@ config DRM_I915 tristate "Intel 8xx/9xx/G3x/G4x/HD Graphics" depends on DRM depends on X86 && PCI + depends on !PREEMPT_RT select INTEL_GTT select INTERVAL_TREE # we need shmfs for the swappable backing store, and in particular # the shmem_readpage() which depends upon tmpfs select SHMEM select TMPFS + select DRM_DP_HELPER select DRM_KMS_HELPER select DRM_PANEL select DRM_MIPI_DSI @@ -27,6 +29,7 @@ config DRM_I915 select CEC_CORE if CEC_NOTIFIER select VMAP_PFN select DRM_TTM + select DRM_BUDDY help Choose this option if you have a system that has "Intel Graphics Media Accelerator" or "HD Graphics" integrated graphics, diff --git a/drivers/gpu/drm/i915/Kconfig.unstable b/drivers/gpu/drm/i915/Kconfig.unstable index 0c2276155c2b..cf151a297ed7 100644 --- a/drivers/gpu/drm/i915/Kconfig.unstable +++ b/drivers/gpu/drm/i915/Kconfig.unstable @@ -19,11 +19,3 @@ config DRM_I915_UNSTABLE Recommended for driver developers _only_. If in the slightest bit of doubt, say "N". - -config DRM_I915_UNSTABLE_FAKE_LMEM - bool "Enable the experimental fake lmem" - depends on DRM_I915_UNSTABLE - default n - help - Convert some system memory into a fake local memory region for - testing. diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 1b62b9f65196..9d588d936e3d 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -13,6 +13,7 @@ # will most likely get a sudden build breakage... Hopefully we will fix # new warnings before CI updates! subdir-ccflags-y := -Wall -Wextra +subdir-ccflags-y += -Wno-format-security subdir-ccflags-y += -Wno-unused-parameter subdir-ccflags-y += -Wno-type-limits subdir-ccflags-y += -Wno-missing-field-initializers @@ -32,8 +33,9 @@ subdir-ccflags-y += -I$(srctree)/$(src) # core driver code i915-y += i915_driver.o \ i915_config.o \ - i915_irq.o \ i915_getparam.o \ + i915_ioctl.o \ + i915_irq.o \ i915_mitigations.o \ i915_module.o \ i915_params.o \ @@ -161,7 +163,6 @@ gem-y += \ i915-y += \ $(gem-y) \ i915_active.o \ - i915_buddy.o \ i915_cmd_parser.o \ i915_deps.o \ i915_gem_evict.o \ @@ -174,7 +175,7 @@ i915-y += \ i915_trace_points.o \ i915_ttm_buddy_manager.o \ i915_vma.o \ - i915_vma_snapshot.o \ + i915_vma_resource.o \ intel_wopcm.o # general-purpose microcontroller (GuC) support @@ -197,6 +198,7 @@ i915-y += gt/uc/intel_uc.o \ # modesetting core code i915-y += \ + display/hsw_ips.o \ display/intel_atomic.o \ display/intel_atomic_plane.o \ display/intel_audio.o \ diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c index f37677df6ebf..f67bbaaad8e0 100644 --- a/drivers/gpu/drm/i915/display/g4x_dp.c +++ b/drivers/gpu/drm/i915/display/g4x_dp.c @@ -18,6 +18,7 @@ #include "intel_fifo_underrun.h" #include "intel_hdmi.h" #include "intel_hotplug.h" +#include "intel_pch_display.h" #include "intel_pps.h" #include "vlv_sideband.h" @@ -333,6 +334,21 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder, return ret; } +static void g4x_dp_get_m_n(struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + if (crtc_state->has_pch_encoder) { + intel_pch_transcoder_get_m1_n1(crtc, &crtc_state->dp_m_n); + intel_pch_transcoder_get_m2_n2(crtc, &crtc_state->dp_m2_n2); + } else { + intel_cpu_transcoder_get_m1_n1(crtc, crtc_state->cpu_transcoder, + &crtc_state->dp_m_n); + intel_cpu_transcoder_get_m2_n2(crtc, crtc_state->cpu_transcoder, + &crtc_state->dp_m2_n2); + } +} + static void intel_dp_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { @@ -384,7 +400,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder, pipe_config->lane_count = ((tmp & DP_PORT_WIDTH_MASK) >> DP_PORT_WIDTH_SHIFT) + 1; - intel_dp_get_m_n(crtc, pipe_config); + g4x_dp_get_m_n(pipe_config); if (port == PORT_A) { if ((intel_de_read(dev_priv, DP_A) & DP_PLL_FREQ_MASK) == DP_PLL_FREQ_162MHZ) diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.h b/drivers/gpu/drm/i915/display/g4x_hdmi.h index 7aca14b602c6..db9a93bc9321 100644 --- a/drivers/gpu/drm/i915/display/g4x_hdmi.h +++ b/drivers/gpu/drm/i915/display/g4x_hdmi.h @@ -8,7 +8,7 @@ #include <linux/types.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" enum port; struct drm_i915_private; diff --git a/drivers/gpu/drm/i915/display/hsw_ips.c b/drivers/gpu/drm/i915/display/hsw_ips.c new file mode 100644 index 000000000000..38014e0cc9ad --- /dev/null +++ b/drivers/gpu/drm/i915/display/hsw_ips.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "hsw_ips.h" +#include "i915_drv.h" +#include "i915_reg.h" +#include "intel_de.h" +#include "intel_display_types.h" +#include "intel_pcode.h" + +static void hsw_ips_enable(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + + if (!crtc_state->ips_enabled) + return; + + /* + * We can only enable IPS after we enable a plane and wait for a vblank + * This function is called from post_plane_update, which is run after + * a vblank wait. + */ + drm_WARN_ON(&i915->drm, + !(crtc_state->active_planes & ~BIT(PLANE_CURSOR))); + + if (IS_BROADWELL(i915)) { + drm_WARN_ON(&i915->drm, + snb_pcode_write(i915, DISPLAY_IPS_CONTROL, + IPS_ENABLE | IPS_PCODE_CONTROL)); + /* + * Quoting Art Runyan: "its not safe to expect any particular + * value in IPS_CTL bit 31 after enabling IPS through the + * mailbox." Moreover, the mailbox may return a bogus state, + * so we need to just enable it and continue on. + */ + } else { + intel_de_write(i915, IPS_CTL, IPS_ENABLE); + /* + * The bit only becomes 1 in the next vblank, so this wait here + * is essentially intel_wait_for_vblank. If we don't have this + * and don't wait for vblanks until the end of crtc_enable, then + * the HW state readout code will complain that the expected + * IPS_CTL value is not the one we read. + */ + if (intel_de_wait_for_set(i915, IPS_CTL, IPS_ENABLE, 50)) + drm_err(&i915->drm, + "Timed out waiting for IPS enable\n"); + } +} + +bool hsw_ips_disable(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + bool need_vblank_wait = false; + + if (!crtc_state->ips_enabled) + return need_vblank_wait; + + if (IS_BROADWELL(i915)) { + drm_WARN_ON(&i915->drm, + snb_pcode_write(i915, DISPLAY_IPS_CONTROL, 0)); + /* + * Wait for PCODE to finish disabling IPS. The BSpec specified + * 42ms timeout value leads to occasional timeouts so use 100ms + * instead. + */ + if (intel_de_wait_for_clear(i915, IPS_CTL, IPS_ENABLE, 100)) + drm_err(&i915->drm, + "Timed out waiting for IPS disable\n"); + } else { + intel_de_write(i915, IPS_CTL, 0); + intel_de_posting_read(i915, IPS_CTL); + } + + /* We need to wait for a vblank before we can disable the plane. */ + need_vblank_wait = true; + + return need_vblank_wait; +} + +static bool hsw_ips_need_disable(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (!old_crtc_state->ips_enabled) + return false; + + if (intel_crtc_needs_modeset(new_crtc_state)) + return true; + + /* + * Workaround : Do not read or write the pipe palette/gamma data while + * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. + * + * Disable IPS before we program the LUT. + */ + if (IS_HASWELL(i915) && + (new_crtc_state->uapi.color_mgmt_changed || + new_crtc_state->update_pipe) && + new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) + return true; + + return !new_crtc_state->ips_enabled; +} + +bool hsw_ips_pre_update(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + + if (!hsw_ips_need_disable(state, crtc)) + return false; + + return hsw_ips_disable(old_crtc_state); +} + +static bool hsw_ips_need_enable(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (!new_crtc_state->ips_enabled) + return false; + + if (intel_crtc_needs_modeset(new_crtc_state)) + return true; + + /* + * Workaround : Do not read or write the pipe palette/gamma data while + * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. + * + * Re-enable IPS after the LUT has been programmed. + */ + if (IS_HASWELL(i915) && + (new_crtc_state->uapi.color_mgmt_changed || + new_crtc_state->update_pipe) && + new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) + return true; + + /* + * We can't read out IPS on broadwell, assume the worst and + * forcibly enable IPS on the first fastset. + */ + if (new_crtc_state->update_pipe && old_crtc_state->inherited) + return true; + + return !old_crtc_state->ips_enabled; +} + +void hsw_ips_post_update(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (!hsw_ips_need_enable(state, crtc)) + return; + + hsw_ips_enable(new_crtc_state); +} + +/* IPS only exists on ULT machines and is tied to pipe A. */ +bool hsw_crtc_supports_ips(struct intel_crtc *crtc) +{ + return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A; +} + +bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + + /* IPS only exists on ULT machines and is tied to pipe A. */ + if (!hsw_crtc_supports_ips(crtc)) + return false; + + if (!i915->params.enable_ips) + return false; + + if (crtc_state->pipe_bpp > 24) + return false; + + /* + * We compare against max which means we must take + * the increased cdclk requirement into account when + * calculating the new cdclk. + * + * Should measure whether using a lower cdclk w/o IPS + */ + if (IS_BROADWELL(i915) && + crtc_state->pixel_rate > i915->max_cdclk_freq * 95 / 100) + return false; + + return true; +} + +int hsw_ips_compute_config(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + crtc_state->ips_enabled = false; + + if (!hsw_crtc_state_ips_capable(crtc_state)) + return 0; + + /* + * When IPS gets enabled, the pipe CRC changes. Since IPS gets + * enabled and disabled dynamically based on package C states, + * user space can't make reliable use of the CRCs, so let's just + * completely disable it. + */ + if (crtc_state->crc_enabled) + return 0; + + /* IPS should be fine as long as at least one plane is enabled. */ + if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR))) + return 0; + + if (IS_BROADWELL(i915)) { + const struct intel_cdclk_state *cdclk_state; + + cdclk_state = intel_atomic_get_cdclk_state(state); + if (IS_ERR(cdclk_state)) + return PTR_ERR(cdclk_state); + + /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ + if (crtc_state->pixel_rate > cdclk_state->logical.cdclk * 95 / 100) + return 0; + } + + crtc_state->ips_enabled = true; + + return 0; +} + +void hsw_ips_get_config(struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + + if (!hsw_crtc_supports_ips(crtc)) + return; + + if (IS_HASWELL(i915)) { + crtc_state->ips_enabled = intel_de_read(i915, IPS_CTL) & IPS_ENABLE; + } else { + /* + * We cannot readout IPS state on broadwell, set to + * true so we can set it to a defined state on first + * commit. + */ + crtc_state->ips_enabled = true; + } +} diff --git a/drivers/gpu/drm/i915/display/hsw_ips.h b/drivers/gpu/drm/i915/display/hsw_ips.h new file mode 100644 index 000000000000..4564dee497d7 --- /dev/null +++ b/drivers/gpu/drm/i915/display/hsw_ips.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __HSW_IPS_H__ +#define __HSW_IPS_H__ + +#include <linux/types.h> + +struct intel_atomic_state; +struct intel_crtc; +struct intel_crtc_state; + +bool hsw_ips_disable(const struct intel_crtc_state *crtc_state); +bool hsw_ips_pre_update(struct intel_atomic_state *state, + struct intel_crtc *crtc); +void hsw_ips_post_update(struct intel_atomic_state *state, + struct intel_crtc *crtc); +bool hsw_crtc_supports_ips(struct intel_crtc *crtc); +bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state); +int hsw_ips_compute_config(struct intel_atomic_state *state, + struct intel_crtc *crtc); +void hsw_ips_get_config(struct intel_crtc_state *crtc_state); + +#endif /* __HSW_IPS_H__ */ diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c index 85950ff67609..a87b65cd41fd 100644 --- a/drivers/gpu/drm/i915/display/i9xx_plane.c +++ b/drivers/gpu/drm/i915/display/i9xx_plane.c @@ -125,7 +125,7 @@ static struct intel_fbc *i9xx_plane_fbc(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { if (i9xx_plane_has_fbc(dev_priv, i9xx_plane)) - return dev_priv->fbc; + return dev_priv->fbc[INTEL_FBC_A]; else return NULL; } @@ -155,51 +155,51 @@ static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state, unsigned int rotation = plane_state->hw.rotation; u32 dspcntr; - dspcntr = DISPLAY_PLANE_ENABLE; + dspcntr = DISP_ENABLE; if (IS_G4X(dev_priv) || IS_IRONLAKE(dev_priv) || IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) - dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; + dspcntr |= DISP_TRICKLE_FEED_DISABLE; switch (fb->format->format) { case DRM_FORMAT_C8: - dspcntr |= DISPPLANE_8BPP; + dspcntr |= DISP_FORMAT_8BPP; break; case DRM_FORMAT_XRGB1555: - dspcntr |= DISPPLANE_BGRX555; + dspcntr |= DISP_FORMAT_BGRX555; break; case DRM_FORMAT_ARGB1555: - dspcntr |= DISPPLANE_BGRA555; + dspcntr |= DISP_FORMAT_BGRA555; break; case DRM_FORMAT_RGB565: - dspcntr |= DISPPLANE_BGRX565; + dspcntr |= DISP_FORMAT_BGRX565; break; case DRM_FORMAT_XRGB8888: - dspcntr |= DISPPLANE_BGRX888; + dspcntr |= DISP_FORMAT_BGRX888; break; case DRM_FORMAT_XBGR8888: - dspcntr |= DISPPLANE_RGBX888; + dspcntr |= DISP_FORMAT_RGBX888; break; case DRM_FORMAT_ARGB8888: - dspcntr |= DISPPLANE_BGRA888; + dspcntr |= DISP_FORMAT_BGRA888; break; case DRM_FORMAT_ABGR8888: - dspcntr |= DISPPLANE_RGBA888; + dspcntr |= DISP_FORMAT_RGBA888; break; case DRM_FORMAT_XRGB2101010: - dspcntr |= DISPPLANE_BGRX101010; + dspcntr |= DISP_FORMAT_BGRX101010; break; case DRM_FORMAT_XBGR2101010: - dspcntr |= DISPPLANE_RGBX101010; + dspcntr |= DISP_FORMAT_RGBX101010; break; case DRM_FORMAT_ARGB2101010: - dspcntr |= DISPPLANE_BGRA101010; + dspcntr |= DISP_FORMAT_BGRA101010; break; case DRM_FORMAT_ABGR2101010: - dspcntr |= DISPPLANE_RGBA101010; + dspcntr |= DISP_FORMAT_RGBA101010; break; case DRM_FORMAT_XBGR16161616F: - dspcntr |= DISPPLANE_RGBX161616; + dspcntr |= DISP_FORMAT_RGBX161616; break; default: MISSING_CASE(fb->format->format); @@ -208,13 +208,13 @@ static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state, if (DISPLAY_VER(dev_priv) >= 4 && fb->modifier == I915_FORMAT_MOD_X_TILED) - dspcntr |= DISPPLANE_TILED; + dspcntr |= DISP_TILED; if (rotation & DRM_MODE_ROTATE_180) - dspcntr |= DISPPLANE_ROTATE_180; + dspcntr |= DISP_ROTATE_180; if (rotation & DRM_MODE_REFLECT_X) - dspcntr |= DISPPLANE_MIRROR; + dspcntr |= DISP_MIRROR; return dspcntr; } @@ -354,13 +354,13 @@ static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state) u32 dspcntr = 0; if (crtc_state->gamma_enable) - dspcntr |= DISPPLANE_GAMMA_ENABLE; + dspcntr |= DISP_PIPE_GAMMA_ENABLE; if (crtc_state->csc_enable) - dspcntr |= DISPPLANE_PIPE_CSC_ENABLE; + dspcntr |= DISP_PIPE_CSC_ENABLE; if (DISPLAY_VER(dev_priv) < 5) - dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe); + dspcntr |= DISP_PIPE_SEL(crtc->pipe); return dspcntr; } @@ -437,9 +437,9 @@ static void i9xx_plane_update_noarm(struct intel_plane *plane, * program whatever is there. */ intel_de_write_fw(dev_priv, DSPPOS(i9xx_plane), - (crtc_y << 16) | crtc_x); + DISP_POS_Y(crtc_y) | DISP_POS_X(crtc_x)); intel_de_write_fw(dev_priv, DSPSIZE(i9xx_plane), - ((crtc_h - 1) << 16) | (crtc_w - 1)); + DISP_HEIGHT(crtc_h - 1) | DISP_WIDTH(crtc_w - 1)); } spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); @@ -474,20 +474,20 @@ static void i9xx_plane_update_arm(struct intel_plane *plane, int crtc_h = drm_rect_height(&plane_state->uapi.dst); intel_de_write_fw(dev_priv, PRIMPOS(i9xx_plane), - (crtc_y << 16) | crtc_x); + PRIM_POS_Y(crtc_y) | PRIM_POS_X(crtc_x)); intel_de_write_fw(dev_priv, PRIMSIZE(i9xx_plane), - ((crtc_h - 1) << 16) | (crtc_w - 1)); + PRIM_HEIGHT(crtc_h - 1) | PRIM_WIDTH(crtc_w - 1)); intel_de_write_fw(dev_priv, PRIMCNSTALPHA(i9xx_plane), 0); } if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { intel_de_write_fw(dev_priv, DSPOFFSET(i9xx_plane), - (y << 16) | x); + DISP_OFFSET_Y(y) | DISP_OFFSET_X(x)); } else if (DISPLAY_VER(dev_priv) >= 4) { intel_de_write_fw(dev_priv, DSPLINOFF(i9xx_plane), linear_offset); intel_de_write_fw(dev_priv, DSPTILEOFF(i9xx_plane), - (y << 16) | x); + DISP_OFFSET_Y(y) | DISP_OFFSET_X(x)); } /* @@ -564,7 +564,7 @@ g4x_primary_async_flip(struct intel_plane *plane, unsigned long irqflags; if (async_flip) - dspcntr |= DISPPLANE_ASYNC_FLIP; + dspcntr |= DISP_ASYNC_FLIP; spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); intel_de_write_fw(dev_priv, DSPCNTR(i9xx_plane), dspcntr); @@ -696,13 +696,12 @@ static bool i9xx_plane_get_hw_state(struct intel_plane *plane, val = intel_de_read(dev_priv, DSPCNTR(i9xx_plane)); - ret = val & DISPLAY_PLANE_ENABLE; + ret = val & DISP_ENABLE; if (DISPLAY_VER(dev_priv) >= 5) *pipe = plane->pipe; else - *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >> - DISPPLANE_SEL_PIPE_SHIFT; + *pipe = REG_FIELD_GET(DISP_PIPE_SEL_MASK, val); intel_display_power_put(dev_priv, power_domain, wakeref); @@ -958,32 +957,32 @@ fail: static int i9xx_format_to_fourcc(int format) { switch (format) { - case DISPPLANE_8BPP: + case DISP_FORMAT_8BPP: return DRM_FORMAT_C8; - case DISPPLANE_BGRA555: + case DISP_FORMAT_BGRA555: return DRM_FORMAT_ARGB1555; - case DISPPLANE_BGRX555: + case DISP_FORMAT_BGRX555: return DRM_FORMAT_XRGB1555; - case DISPPLANE_BGRX565: + case DISP_FORMAT_BGRX565: return DRM_FORMAT_RGB565; default: - case DISPPLANE_BGRX888: + case DISP_FORMAT_BGRX888: return DRM_FORMAT_XRGB8888; - case DISPPLANE_RGBX888: + case DISP_FORMAT_RGBX888: return DRM_FORMAT_XBGR8888; - case DISPPLANE_BGRA888: + case DISP_FORMAT_BGRA888: return DRM_FORMAT_ARGB8888; - case DISPPLANE_RGBA888: + case DISP_FORMAT_RGBA888: return DRM_FORMAT_ABGR8888; - case DISPPLANE_BGRX101010: + case DISP_FORMAT_BGRX101010: return DRM_FORMAT_XRGB2101010; - case DISPPLANE_RGBX101010: + case DISP_FORMAT_RGBX101010: return DRM_FORMAT_XBGR2101010; - case DISPPLANE_BGRA101010: + case DISP_FORMAT_BGRA101010: return DRM_FORMAT_ARGB2101010; - case DISPPLANE_RGBA101010: + case DISP_FORMAT_RGBA101010: return DRM_FORMAT_ABGR2101010; - case DISPPLANE_RGBX161616: + case DISP_FORMAT_RGBX161616: return DRM_FORMAT_XBGR16161616F; } } @@ -1021,26 +1020,26 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc, val = intel_de_read(dev_priv, DSPCNTR(i9xx_plane)); if (DISPLAY_VER(dev_priv) >= 4) { - if (val & DISPPLANE_TILED) { + if (val & DISP_TILED) { plane_config->tiling = I915_TILING_X; fb->modifier = I915_FORMAT_MOD_X_TILED; } - if (val & DISPPLANE_ROTATE_180) + if (val & DISP_ROTATE_180) plane_config->rotation = DRM_MODE_ROTATE_180; } if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B && - val & DISPPLANE_MIRROR) + val & DISP_MIRROR) plane_config->rotation |= DRM_MODE_REFLECT_X; - pixel_format = val & DISPPLANE_PIXFORMAT_MASK; + pixel_format = val & DISP_FORMAT_MASK; fourcc = i9xx_format_to_fourcc(pixel_format); fb->format = drm_format_info(fourcc); if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { offset = intel_de_read(dev_priv, DSPOFFSET(i9xx_plane)); - base = intel_de_read(dev_priv, DSPSURF(i9xx_plane)) & 0xfffff000; + base = intel_de_read(dev_priv, DSPSURF(i9xx_plane)) & DISP_ADDR_MASK; } else if (DISPLAY_VER(dev_priv) >= 4) { if (plane_config->tiling) offset = intel_de_read(dev_priv, @@ -1048,15 +1047,15 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc, else offset = intel_de_read(dev_priv, DSPLINOFF(i9xx_plane)); - base = intel_de_read(dev_priv, DSPSURF(i9xx_plane)) & 0xfffff000; + base = intel_de_read(dev_priv, DSPSURF(i9xx_plane)) & DISP_ADDR_MASK; } else { base = intel_de_read(dev_priv, DSPADDR(i9xx_plane)); } plane_config->base = base; val = intel_de_read(dev_priv, PIPESRC(pipe)); - fb->width = ((val >> 16) & 0xfff) + 1; - fb->height = ((val >> 0) & 0xfff) + 1; + fb->width = REG_FIELD_GET(PIPESRC_WIDTH_MASK, val) + 1; + fb->height = REG_FIELD_GET(PIPESRC_HEIGHT_MASK, val) + 1; val = intel_de_read(dev_priv, DSPSTRIDE(i9xx_plane)); fb->pitches[0] = val & 0xffffffc0; diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index 5781e9fac8b4..13b07c6fd6be 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -29,9 +29,11 @@ #include <drm/drm_mipi_dsi.h> #include "icl_dsi.h" +#include "icl_dsi_regs.h" #include "intel_atomic.h" #include "intel_backlight.h" #include "intel_combo_phy.h" +#include "intel_combo_phy_regs.h" #include "intel_connector.h" #include "intel_crtc.h" #include "intel_ddi.h" @@ -569,7 +571,7 @@ gen11_dsi_setup_dphy_timings(struct intel_encoder *encoder, /* Program T-INIT master registers */ for_each_dsi_port(port, intel_dsi->ports) { tmp = intel_de_read(dev_priv, ICL_DSI_T_INIT_MASTER(port)); - tmp &= ~MASTER_INIT_TIMER_MASK; + tmp &= ~DSI_T_INIT_MASTER_MASK; tmp |= intel_dsi->init_count; intel_de_write(dev_priv, ICL_DSI_T_INIT_MASTER(port), tmp); } @@ -787,14 +789,14 @@ gen11_dsi_configure_transcoder(struct intel_encoder *encoder, /* program DSI operation mode */ if (is_vid_mode(intel_dsi)) { tmp &= ~OP_MODE_MASK; - switch (intel_dsi->video_mode_format) { + switch (intel_dsi->video_mode) { default: - MISSING_CASE(intel_dsi->video_mode_format); + MISSING_CASE(intel_dsi->video_mode); fallthrough; - case VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS: + case NON_BURST_SYNC_EVENTS: tmp |= VIDEO_MODE_SYNC_EVENT; break; - case VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE: + case NON_BURST_SYNC_PULSE: tmp |= VIDEO_MODE_SYNC_PULSE; break; } @@ -959,8 +961,7 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder, /* TRANS_HSYNC register to be programmed only for video mode */ if (is_vid_mode(intel_dsi)) { - if (intel_dsi->video_mode_format == - VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE) { + if (intel_dsi->video_mode == NON_BURST_SYNC_PULSE) { /* BSPEC: hsync size should be atleast 16 pixels */ if (hsync_size < 16) drm_err(&dev_priv->drm, @@ -1050,7 +1051,7 @@ static void gen11_dsi_enable_transcoder(struct intel_encoder *encoder) /* wait for transcoder to be enabled */ if (intel_de_wait_for_set(dev_priv, PIPECONF(dsi_trans), - I965_PIPECONF_ACTIVE, 10)) + PIPECONF_STATE_ENABLE, 10)) drm_err(&dev_priv->drm, "DSI transcoder not enabled\n"); } @@ -1232,8 +1233,6 @@ static void gen11_dsi_pre_enable(struct intel_atomic_state *state, intel_dsc_dsi_pps_write(encoder, pipe_config); - intel_dsc_enable(pipe_config); - /* step6c: configure transcoder timings */ gen11_dsi_set_transcoder_timings(encoder, pipe_config); } @@ -1320,7 +1319,7 @@ static void gen11_dsi_disable_transcoder(struct intel_encoder *encoder) /* wait for transcoder to be disabled */ if (intel_de_wait_for_clear(dev_priv, PIPECONF(dsi_trans), - I965_PIPECONF_ACTIVE, 50)) + PIPECONF_STATE_ENABLE, 50)) drm_err(&dev_priv->drm, "DSI trancoder not disabled\n"); } diff --git a/drivers/gpu/drm/i915/display/icl_dsi_regs.h b/drivers/gpu/drm/i915/display/icl_dsi_regs.h new file mode 100644 index 000000000000..f78f28b8dd94 --- /dev/null +++ b/drivers/gpu/drm/i915/display/icl_dsi_regs.h @@ -0,0 +1,342 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __ICL_DSI_REGS_H__ +#define __ICL_DSI_REGS_H__ + +#include "i915_reg_defs.h" + +/* Gen11 DSI */ +#define _MMIO_DSI(tc, dsi0, dsi1) _MMIO_TRANS((tc) - TRANSCODER_DSI_0, \ + dsi0, dsi1) +#define _ICL_DSI_ESC_CLK_DIV0 0x6b090 +#define _ICL_DSI_ESC_CLK_DIV1 0x6b890 +#define ICL_DSI_ESC_CLK_DIV(port) _MMIO_PORT((port), \ + _ICL_DSI_ESC_CLK_DIV0, \ + _ICL_DSI_ESC_CLK_DIV1) +#define _ICL_DPHY_ESC_CLK_DIV0 0x162190 +#define _ICL_DPHY_ESC_CLK_DIV1 0x6C190 +#define ICL_DPHY_ESC_CLK_DIV(port) _MMIO_PORT((port), \ + _ICL_DPHY_ESC_CLK_DIV0, \ + _ICL_DPHY_ESC_CLK_DIV1) +#define ICL_BYTE_CLK_PER_ESC_CLK_MASK (0x1f << 16) +#define ICL_BYTE_CLK_PER_ESC_CLK_SHIFT 16 +#define ICL_ESC_CLK_DIV_MASK 0x1ff +#define ICL_ESC_CLK_DIV_SHIFT 0 +#define DSI_MAX_ESC_CLK 20000 /* in KHz */ + +#define _ADL_MIPIO_REG 0x180 +#define ADL_MIPIO_DW(port, dw) _MMIO(_ICL_COMBOPHY(port) + _ADL_MIPIO_REG + 4 * (dw)) +#define TX_ESC_CLK_DIV_PHY_SEL REGBIT(16) +#define TX_ESC_CLK_DIV_PHY_MASK REG_GENMASK(23, 16) +#define TX_ESC_CLK_DIV_PHY REG_FIELD_PREP(TX_ESC_CLK_DIV_PHY_MASK, 0x7f) + +#define _DSI_CMD_FRMCTL_0 0x6b034 +#define _DSI_CMD_FRMCTL_1 0x6b834 +#define DSI_CMD_FRMCTL(port) _MMIO_PORT(port, \ + _DSI_CMD_FRMCTL_0,\ + _DSI_CMD_FRMCTL_1) +#define DSI_FRAME_UPDATE_REQUEST (1 << 31) +#define DSI_PERIODIC_FRAME_UPDATE_ENABLE (1 << 29) +#define DSI_NULL_PACKET_ENABLE (1 << 28) +#define DSI_FRAME_IN_PROGRESS (1 << 0) + +#define _DSI_INTR_MASK_REG_0 0x6b070 +#define _DSI_INTR_MASK_REG_1 0x6b870 +#define DSI_INTR_MASK_REG(port) _MMIO_PORT(port, \ + _DSI_INTR_MASK_REG_0,\ + _DSI_INTR_MASK_REG_1) + +#define _DSI_INTR_IDENT_REG_0 0x6b074 +#define _DSI_INTR_IDENT_REG_1 0x6b874 +#define DSI_INTR_IDENT_REG(port) _MMIO_PORT(port, \ + _DSI_INTR_IDENT_REG_0,\ + _DSI_INTR_IDENT_REG_1) +#define DSI_TE_EVENT (1 << 31) +#define DSI_RX_DATA_OR_BTA_TERMINATED (1 << 30) +#define DSI_TX_DATA (1 << 29) +#define DSI_ULPS_ENTRY_DONE (1 << 28) +#define DSI_NON_TE_TRIGGER_RECEIVED (1 << 27) +#define DSI_HOST_CHKSUM_ERROR (1 << 26) +#define DSI_HOST_MULTI_ECC_ERROR (1 << 25) +#define DSI_HOST_SINGL_ECC_ERROR (1 << 24) +#define DSI_HOST_CONTENTION_DETECTED (1 << 23) +#define DSI_HOST_FALSE_CONTROL_ERROR (1 << 22) +#define DSI_HOST_TIMEOUT_ERROR (1 << 21) +#define DSI_HOST_LOW_POWER_TX_SYNC_ERROR (1 << 20) +#define DSI_HOST_ESCAPE_MODE_ENTRY_ERROR (1 << 19) +#define DSI_FRAME_UPDATE_DONE (1 << 16) +#define DSI_PROTOCOL_VIOLATION_REPORTED (1 << 15) +#define DSI_INVALID_TX_LENGTH (1 << 13) +#define DSI_INVALID_VC (1 << 12) +#define DSI_INVALID_DATA_TYPE (1 << 11) +#define DSI_PERIPHERAL_CHKSUM_ERROR (1 << 10) +#define DSI_PERIPHERAL_MULTI_ECC_ERROR (1 << 9) +#define DSI_PERIPHERAL_SINGLE_ECC_ERROR (1 << 8) +#define DSI_PERIPHERAL_CONTENTION_DETECTED (1 << 7) +#define DSI_PERIPHERAL_FALSE_CTRL_ERROR (1 << 6) +#define DSI_PERIPHERAL_TIMEOUT_ERROR (1 << 5) +#define DSI_PERIPHERAL_LP_TX_SYNC_ERROR (1 << 4) +#define DSI_PERIPHERAL_ESC_MODE_ENTRY_CMD_ERR (1 << 3) +#define DSI_EOT_SYNC_ERROR (1 << 2) +#define DSI_SOT_SYNC_ERROR (1 << 1) +#define DSI_SOT_ERROR (1 << 0) + +/* ICL DSI MODE control */ +#define _ICL_DSI_IO_MODECTL_0 0x6B094 +#define _ICL_DSI_IO_MODECTL_1 0x6B894 +#define ICL_DSI_IO_MODECTL(port) _MMIO_PORT(port, \ + _ICL_DSI_IO_MODECTL_0, \ + _ICL_DSI_IO_MODECTL_1) +#define COMBO_PHY_MODE_DSI (1 << 0) + +/* TGL DSI Chicken register */ +#define _TGL_DSI_CHKN_REG_0 0x6B0C0 +#define _TGL_DSI_CHKN_REG_1 0x6B8C0 +#define TGL_DSI_CHKN_REG(port) _MMIO_PORT(port, \ + _TGL_DSI_CHKN_REG_0, \ + _TGL_DSI_CHKN_REG_1) +#define TGL_DSI_CHKN_LSHS_GB_MASK REG_GENMASK(15, 12) +#define TGL_DSI_CHKN_LSHS_GB(byte_clocks) REG_FIELD_PREP(TGL_DSI_CHKN_LSHS_GB_MASK, \ + (byte_clocks)) +#define _ICL_DSI_T_INIT_MASTER_0 0x6b088 +#define _ICL_DSI_T_INIT_MASTER_1 0x6b888 +#define ICL_DSI_T_INIT_MASTER(port) _MMIO_PORT(port, \ + _ICL_DSI_T_INIT_MASTER_0,\ + _ICL_DSI_T_INIT_MASTER_1) +#define DSI_T_INIT_MASTER_MASK REG_GENMASK(15, 0) + +#define _DPHY_CLK_TIMING_PARAM_0 0x162180 +#define _DPHY_CLK_TIMING_PARAM_1 0x6c180 +#define DPHY_CLK_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DPHY_CLK_TIMING_PARAM_0,\ + _DPHY_CLK_TIMING_PARAM_1) +#define _DSI_CLK_TIMING_PARAM_0 0x6b080 +#define _DSI_CLK_TIMING_PARAM_1 0x6b880 +#define DSI_CLK_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DSI_CLK_TIMING_PARAM_0,\ + _DSI_CLK_TIMING_PARAM_1) +#define CLK_PREPARE_OVERRIDE (1 << 31) +#define CLK_PREPARE(x) ((x) << 28) +#define CLK_PREPARE_MASK (0x7 << 28) +#define CLK_PREPARE_SHIFT 28 +#define CLK_ZERO_OVERRIDE (1 << 27) +#define CLK_ZERO(x) ((x) << 20) +#define CLK_ZERO_MASK (0xf << 20) +#define CLK_ZERO_SHIFT 20 +#define CLK_PRE_OVERRIDE (1 << 19) +#define CLK_PRE(x) ((x) << 16) +#define CLK_PRE_MASK (0x3 << 16) +#define CLK_PRE_SHIFT 16 +#define CLK_POST_OVERRIDE (1 << 15) +#define CLK_POST(x) ((x) << 8) +#define CLK_POST_MASK (0x7 << 8) +#define CLK_POST_SHIFT 8 +#define CLK_TRAIL_OVERRIDE (1 << 7) +#define CLK_TRAIL(x) ((x) << 0) +#define CLK_TRAIL_MASK (0xf << 0) +#define CLK_TRAIL_SHIFT 0 + +#define _DPHY_DATA_TIMING_PARAM_0 0x162184 +#define _DPHY_DATA_TIMING_PARAM_1 0x6c184 +#define DPHY_DATA_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DPHY_DATA_TIMING_PARAM_0,\ + _DPHY_DATA_TIMING_PARAM_1) +#define _DSI_DATA_TIMING_PARAM_0 0x6B084 +#define _DSI_DATA_TIMING_PARAM_1 0x6B884 +#define DSI_DATA_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DSI_DATA_TIMING_PARAM_0,\ + _DSI_DATA_TIMING_PARAM_1) +#define HS_PREPARE_OVERRIDE (1 << 31) +#define HS_PREPARE(x) ((x) << 24) +#define HS_PREPARE_MASK (0x7 << 24) +#define HS_PREPARE_SHIFT 24 +#define HS_ZERO_OVERRIDE (1 << 23) +#define HS_ZERO(x) ((x) << 16) +#define HS_ZERO_MASK (0xf << 16) +#define HS_ZERO_SHIFT 16 +#define HS_TRAIL_OVERRIDE (1 << 15) +#define HS_TRAIL(x) ((x) << 8) +#define HS_TRAIL_MASK (0x7 << 8) +#define HS_TRAIL_SHIFT 8 +#define HS_EXIT_OVERRIDE (1 << 7) +#define HS_EXIT(x) ((x) << 0) +#define HS_EXIT_MASK (0x7 << 0) +#define HS_EXIT_SHIFT 0 + +#define _DPHY_TA_TIMING_PARAM_0 0x162188 +#define _DPHY_TA_TIMING_PARAM_1 0x6c188 +#define DPHY_TA_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DPHY_TA_TIMING_PARAM_0,\ + _DPHY_TA_TIMING_PARAM_1) +#define _DSI_TA_TIMING_PARAM_0 0x6b098 +#define _DSI_TA_TIMING_PARAM_1 0x6b898 +#define DSI_TA_TIMING_PARAM(port) _MMIO_PORT(port, \ + _DSI_TA_TIMING_PARAM_0,\ + _DSI_TA_TIMING_PARAM_1) +#define TA_SURE_OVERRIDE (1 << 31) +#define TA_SURE(x) ((x) << 16) +#define TA_SURE_MASK (0x1f << 16) +#define TA_SURE_SHIFT 16 +#define TA_GO_OVERRIDE (1 << 15) +#define TA_GO(x) ((x) << 8) +#define TA_GO_MASK (0xf << 8) +#define TA_GO_SHIFT 8 +#define TA_GET_OVERRIDE (1 << 7) +#define TA_GET(x) ((x) << 0) +#define TA_GET_MASK (0xf << 0) +#define TA_GET_SHIFT 0 + +/* DSI transcoder configuration */ +#define _DSI_TRANS_FUNC_CONF_0 0x6b030 +#define _DSI_TRANS_FUNC_CONF_1 0x6b830 +#define DSI_TRANS_FUNC_CONF(tc) _MMIO_DSI(tc, \ + _DSI_TRANS_FUNC_CONF_0,\ + _DSI_TRANS_FUNC_CONF_1) +#define OP_MODE_MASK (0x3 << 28) +#define OP_MODE_SHIFT 28 +#define CMD_MODE_NO_GATE (0x0 << 28) +#define CMD_MODE_TE_GATE (0x1 << 28) +#define VIDEO_MODE_SYNC_EVENT (0x2 << 28) +#define VIDEO_MODE_SYNC_PULSE (0x3 << 28) +#define TE_SOURCE_GPIO (1 << 27) +#define LINK_READY (1 << 20) +#define PIX_FMT_MASK (0x3 << 16) +#define PIX_FMT_SHIFT 16 +#define PIX_FMT_RGB565 (0x0 << 16) +#define PIX_FMT_RGB666_PACKED (0x1 << 16) +#define PIX_FMT_RGB666_LOOSE (0x2 << 16) +#define PIX_FMT_RGB888 (0x3 << 16) +#define PIX_FMT_RGB101010 (0x4 << 16) +#define PIX_FMT_RGB121212 (0x5 << 16) +#define PIX_FMT_COMPRESSED (0x6 << 16) +#define BGR_TRANSMISSION (1 << 15) +#define PIX_VIRT_CHAN(x) ((x) << 12) +#define PIX_VIRT_CHAN_MASK (0x3 << 12) +#define PIX_VIRT_CHAN_SHIFT 12 +#define PIX_BUF_THRESHOLD_MASK (0x3 << 10) +#define PIX_BUF_THRESHOLD_SHIFT 10 +#define PIX_BUF_THRESHOLD_1_4 (0x0 << 10) +#define PIX_BUF_THRESHOLD_1_2 (0x1 << 10) +#define PIX_BUF_THRESHOLD_3_4 (0x2 << 10) +#define PIX_BUF_THRESHOLD_FULL (0x3 << 10) +#define CONTINUOUS_CLK_MASK (0x3 << 8) +#define CONTINUOUS_CLK_SHIFT 8 +#define CLK_ENTER_LP_AFTER_DATA (0x0 << 8) +#define CLK_HS_OR_LP (0x2 << 8) +#define CLK_HS_CONTINUOUS (0x3 << 8) +#define LINK_CALIBRATION_MASK (0x3 << 4) +#define LINK_CALIBRATION_SHIFT 4 +#define CALIBRATION_DISABLED (0x0 << 4) +#define CALIBRATION_ENABLED_INITIAL_ONLY (0x2 << 4) +#define CALIBRATION_ENABLED_INITIAL_PERIODIC (0x3 << 4) +#define BLANKING_PACKET_ENABLE (1 << 2) +#define S3D_ORIENTATION_LANDSCAPE (1 << 1) +#define EOTP_DISABLED (1 << 0) + +#define _DSI_CMD_RXCTL_0 0x6b0d4 +#define _DSI_CMD_RXCTL_1 0x6b8d4 +#define DSI_CMD_RXCTL(tc) _MMIO_DSI(tc, \ + _DSI_CMD_RXCTL_0,\ + _DSI_CMD_RXCTL_1) +#define READ_UNLOADS_DW (1 << 16) +#define RECEIVED_UNASSIGNED_TRIGGER (1 << 15) +#define RECEIVED_ACKNOWLEDGE_TRIGGER (1 << 14) +#define RECEIVED_TEAR_EFFECT_TRIGGER (1 << 13) +#define RECEIVED_RESET_TRIGGER (1 << 12) +#define RECEIVED_PAYLOAD_WAS_LOST (1 << 11) +#define RECEIVED_CRC_WAS_LOST (1 << 10) +#define NUMBER_RX_PLOAD_DW_MASK (0xff << 0) +#define NUMBER_RX_PLOAD_DW_SHIFT 0 + +#define _DSI_CMD_TXCTL_0 0x6b0d0 +#define _DSI_CMD_TXCTL_1 0x6b8d0 +#define DSI_CMD_TXCTL(tc) _MMIO_DSI(tc, \ + _DSI_CMD_TXCTL_0,\ + _DSI_CMD_TXCTL_1) +#define KEEP_LINK_IN_HS (1 << 24) +#define FREE_HEADER_CREDIT_MASK (0x1f << 8) +#define FREE_HEADER_CREDIT_SHIFT 0x8 +#define FREE_PLOAD_CREDIT_MASK (0xff << 0) +#define FREE_PLOAD_CREDIT_SHIFT 0 +#define MAX_HEADER_CREDIT 0x10 +#define MAX_PLOAD_CREDIT 0x40 + +#define _DSI_CMD_TXHDR_0 0x6b100 +#define _DSI_CMD_TXHDR_1 0x6b900 +#define DSI_CMD_TXHDR(tc) _MMIO_DSI(tc, \ + _DSI_CMD_TXHDR_0,\ + _DSI_CMD_TXHDR_1) +#define PAYLOAD_PRESENT (1 << 31) +#define LP_DATA_TRANSFER (1 << 30) +#define VBLANK_FENCE (1 << 29) +#define PARAM_WC_MASK (0xffff << 8) +#define PARAM_WC_LOWER_SHIFT 8 +#define PARAM_WC_UPPER_SHIFT 16 +#define VC_MASK (0x3 << 6) +#define VC_SHIFT 6 +#define DT_MASK (0x3f << 0) +#define DT_SHIFT 0 + +#define _DSI_CMD_TXPYLD_0 0x6b104 +#define _DSI_CMD_TXPYLD_1 0x6b904 +#define DSI_CMD_TXPYLD(tc) _MMIO_DSI(tc, \ + _DSI_CMD_TXPYLD_0,\ + _DSI_CMD_TXPYLD_1) + +#define _DSI_LP_MSG_0 0x6b0d8 +#define _DSI_LP_MSG_1 0x6b8d8 +#define DSI_LP_MSG(tc) _MMIO_DSI(tc, \ + _DSI_LP_MSG_0,\ + _DSI_LP_MSG_1) +#define LPTX_IN_PROGRESS (1 << 17) +#define LINK_IN_ULPS (1 << 16) +#define LINK_ULPS_TYPE_LP11 (1 << 8) +#define LINK_ENTER_ULPS (1 << 0) + +/* DSI timeout registers */ +#define _DSI_HSTX_TO_0 0x6b044 +#define _DSI_HSTX_TO_1 0x6b844 +#define DSI_HSTX_TO(tc) _MMIO_DSI(tc, \ + _DSI_HSTX_TO_0,\ + _DSI_HSTX_TO_1) +#define HSTX_TIMEOUT_VALUE_MASK (0xffff << 16) +#define HSTX_TIMEOUT_VALUE_SHIFT 16 +#define HSTX_TIMEOUT_VALUE(x) ((x) << 16) +#define HSTX_TIMED_OUT (1 << 0) + +#define _DSI_LPRX_HOST_TO_0 0x6b048 +#define _DSI_LPRX_HOST_TO_1 0x6b848 +#define DSI_LPRX_HOST_TO(tc) _MMIO_DSI(tc, \ + _DSI_LPRX_HOST_TO_0,\ + _DSI_LPRX_HOST_TO_1) +#define LPRX_TIMED_OUT (1 << 16) +#define LPRX_TIMEOUT_VALUE_MASK (0xffff << 0) +#define LPRX_TIMEOUT_VALUE_SHIFT 0 +#define LPRX_TIMEOUT_VALUE(x) ((x) << 0) + +#define _DSI_PWAIT_TO_0 0x6b040 +#define _DSI_PWAIT_TO_1 0x6b840 +#define DSI_PWAIT_TO(tc) _MMIO_DSI(tc, \ + _DSI_PWAIT_TO_0,\ + _DSI_PWAIT_TO_1) +#define PRESET_TIMEOUT_VALUE_MASK (0xffff << 16) +#define PRESET_TIMEOUT_VALUE_SHIFT 16 +#define PRESET_TIMEOUT_VALUE(x) ((x) << 16) +#define PRESPONSE_TIMEOUT_VALUE_MASK (0xffff << 0) +#define PRESPONSE_TIMEOUT_VALUE_SHIFT 0 +#define PRESPONSE_TIMEOUT_VALUE(x) ((x) << 0) + +#define _DSI_TA_TO_0 0x6b04c +#define _DSI_TA_TO_1 0x6b84c +#define DSI_TA_TO(tc) _MMIO_DSI(tc, \ + _DSI_TA_TO_0,\ + _DSI_TA_TO_1) +#define TA_TIMED_OUT (1 << 16) +#define TA_TIMEOUT_VALUE_MASK (0xffff << 0) +#define TA_TIMEOUT_VALUE_SHIFT 0 +#define TA_TIMEOUT_VALUE(x) ((x) << 0) + +#endif /* __ICL_DSI_REGS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c index a62550711e98..40da7910f845 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic.c +++ b/drivers/gpu/drm/i915/display/intel_atomic.c @@ -34,6 +34,8 @@ #include <drm/drm_fourcc.h> #include <drm/drm_plane_helper.h> +#include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_cdclk.h" #include "intel_display_types.h" @@ -260,6 +262,7 @@ intel_crtc_duplicate_state(struct drm_crtc *crtc) crtc_state->preload_luts = false; crtc_state->inherited = false; crtc_state->wm.need_postvbl_update = false; + crtc_state->do_async_flip = false; crtc_state->fb_bits = 0; crtc_state->update_planes = 0; crtc_state->dsb = NULL; @@ -279,17 +282,6 @@ void intel_crtc_free_hw_state(struct intel_crtc_state *crtc_state) intel_crtc_put_color_blobs(crtc_state); } -void intel_crtc_copy_color_blobs(struct intel_crtc_state *crtc_state, - const struct intel_crtc_state *from_crtc_state) -{ - drm_property_replace_blob(&crtc_state->hw.degamma_lut, - from_crtc_state->uapi.degamma_lut); - drm_property_replace_blob(&crtc_state->hw.gamma_lut, - from_crtc_state->uapi.gamma_lut); - drm_property_replace_blob(&crtc_state->hw.ctm, - from_crtc_state->uapi.ctm); -} - /** * intel_crtc_destroy_state - destroy crtc state * @crtc: drm crtc diff --git a/drivers/gpu/drm/i915/display/intel_atomic.h b/drivers/gpu/drm/i915/display/intel_atomic.h index d2700c74c9da..1dc439983dd9 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic.h +++ b/drivers/gpu/drm/i915/display/intel_atomic.h @@ -44,8 +44,6 @@ struct drm_crtc_state *intel_crtc_duplicate_state(struct drm_crtc *crtc); void intel_crtc_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *state); void intel_crtc_free_hw_state(struct intel_crtc_state *crtc_state); -void intel_crtc_copy_color_blobs(struct intel_crtc_state *crtc_state, - const struct intel_crtc_state *from_crtc_state); struct drm_atomic_state *intel_atomic_state_alloc(struct drm_device *dev); void intel_atomic_state_free(struct drm_atomic_state *state); void intel_atomic_state_clear(struct drm_atomic_state *state); diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c index c2c512cd8ec0..5712688232fb 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c @@ -45,6 +45,7 @@ #include "intel_fb_pin.h" #include "intel_pm.h" #include "intel_sprite.h" +#include "skl_scaler.h" static void intel_plane_state_reset(struct intel_plane_state *plane_state, struct intel_plane *plane) @@ -321,6 +322,7 @@ void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); crtc_state->active_planes &= ~BIT(plane->id); + crtc_state->scaled_planes &= ~BIT(plane->id); crtc_state->nv12_planes &= ~BIT(plane->id); crtc_state->c8_planes &= ~BIT(plane->id); crtc_state->data_rate[plane->id] = 0; @@ -329,6 +331,185 @@ void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, plane_state->uapi.visible = false; } +/* FIXME nuke when all wm code is atomic */ +static bool intel_wm_need_update(const struct intel_plane_state *cur, + struct intel_plane_state *new) +{ + /* Update watermarks on tiling or size changes. */ + if (new->uapi.visible != cur->uapi.visible) + return true; + + if (!cur->hw.fb || !new->hw.fb) + return false; + + if (cur->hw.fb->modifier != new->hw.fb->modifier || + cur->hw.rotation != new->hw.rotation || + drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) || + drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) || + drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) || + drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst)) + return true; + + return false; +} + +static bool intel_plane_is_scaled(const struct intel_plane_state *plane_state) +{ + int src_w = drm_rect_width(&plane_state->uapi.src) >> 16; + int src_h = drm_rect_height(&plane_state->uapi.src) >> 16; + int dst_w = drm_rect_width(&plane_state->uapi.dst); + int dst_h = drm_rect_height(&plane_state->uapi.dst); + + return src_w != dst_w || src_h != dst_h; +} + +static bool intel_plane_do_async_flip(struct intel_plane *plane, + const struct intel_crtc_state *old_crtc_state, + const struct intel_crtc_state *new_crtc_state) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + if (!plane->async_flip) + return false; + + if (!new_crtc_state->uapi.async_flip) + return false; + + /* + * In platforms after DISPLAY13, we might need to override + * first async flip in order to change watermark levels + * as part of optimization. + * So for those, we are checking if this is a first async flip. + * For platforms earlier than DISPLAY13 we always do async flip. + */ + return DISPLAY_VER(i915) < 13 || old_crtc_state->uapi.async_flip; +} + +static int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, + struct intel_crtc_state *new_crtc_state, + const struct intel_plane_state *old_plane_state, + struct intel_plane_state *new_plane_state) +{ + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); + struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + bool mode_changed = intel_crtc_needs_modeset(new_crtc_state); + bool was_crtc_enabled = old_crtc_state->hw.active; + bool is_crtc_enabled = new_crtc_state->hw.active; + bool turn_off, turn_on, visible, was_visible; + int ret; + + if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_CURSOR) { + ret = skl_update_scaler_plane(new_crtc_state, new_plane_state); + if (ret) + return ret; + } + + was_visible = old_plane_state->uapi.visible; + visible = new_plane_state->uapi.visible; + + if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible)) + was_visible = false; + + /* + * Visibility is calculated as if the crtc was on, but + * after scaler setup everything depends on it being off + * when the crtc isn't active. + * + * FIXME this is wrong for watermarks. Watermarks should also + * be computed as if the pipe would be active. Perhaps move + * per-plane wm computation to the .check_plane() hook, and + * only combine the results from all planes in the current place? + */ + if (!is_crtc_enabled) { + intel_plane_set_invisible(new_crtc_state, new_plane_state); + visible = false; + } + + if (!was_visible && !visible) + return 0; + + turn_off = was_visible && (!visible || mode_changed); + turn_on = visible && (!was_visible || mode_changed); + + drm_dbg_atomic(&dev_priv->drm, + "[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n", + crtc->base.base.id, crtc->base.name, + plane->base.base.id, plane->base.name, + was_visible, visible, + turn_off, turn_on, mode_changed); + + if (turn_on) { + if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) + new_crtc_state->update_wm_pre = true; + + /* must disable cxsr around plane enable/disable */ + if (plane->id != PLANE_CURSOR) + new_crtc_state->disable_cxsr = true; + } else if (turn_off) { + if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) + new_crtc_state->update_wm_post = true; + + /* must disable cxsr around plane enable/disable */ + if (plane->id != PLANE_CURSOR) + new_crtc_state->disable_cxsr = true; + } else if (intel_wm_need_update(old_plane_state, new_plane_state)) { + if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) { + /* FIXME bollocks */ + new_crtc_state->update_wm_pre = true; + new_crtc_state->update_wm_post = true; + } + } + + if (visible || was_visible) + new_crtc_state->fb_bits |= plane->frontbuffer_bit; + + /* + * ILK/SNB DVSACNTR/Sprite Enable + * IVB SPR_CTL/Sprite Enable + * "When in Self Refresh Big FIFO mode, a write to enable the + * plane will be internally buffered and delayed while Big FIFO + * mode is exiting." + * + * Which means that enabling the sprite can take an extra frame + * when we start in big FIFO mode (LP1+). Thus we need to drop + * down to LP0 and wait for vblank in order to make sure the + * sprite gets enabled on the next vblank after the register write. + * Doing otherwise would risk enabling the sprite one frame after + * we've already signalled flip completion. We can resume LP1+ + * once the sprite has been enabled. + * + * + * WaCxSRDisabledForSpriteScaling:ivb + * IVB SPR_SCALE/Scaling Enable + * "Low Power watermarks must be disabled for at least one + * frame before enabling sprite scaling, and kept disabled + * until sprite scaling is disabled." + * + * ILK/SNB DVSASCALE/Scaling Enable + * "When in Self Refresh Big FIFO mode, scaling enable will be + * masked off while Big FIFO mode is exiting." + * + * Despite the w/a only being listed for IVB we assume that + * the ILK/SNB note has similar ramifications, hence we apply + * the w/a on all three platforms. + * + * With experimental results seems this is needed also for primary + * plane, not only sprite plane. + */ + if (plane->id != PLANE_CURSOR && + (IS_IRONLAKE(dev_priv) || IS_SANDYBRIDGE(dev_priv) || + IS_IVYBRIDGE(dev_priv)) && + (turn_on || (!intel_plane_is_scaled(old_plane_state) && + intel_plane_is_scaled(new_plane_state)))) + new_crtc_state->disable_lp_wm = true; + + if (intel_plane_do_async_flip(plane, old_crtc_state, new_crtc_state)) + new_crtc_state->do_async_flip = true; + + return 0; +} + int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, struct intel_crtc_state *new_crtc_state, const struct intel_plane_state *old_plane_state, @@ -356,6 +537,10 @@ int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_ new_crtc_state->active_planes |= BIT(plane->id); if (new_plane_state->uapi.visible && + intel_plane_is_scaled(new_plane_state)) + new_crtc_state->scaled_planes |= BIT(plane->id); + + if (new_plane_state->uapi.visible && intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) new_crtc_state->nv12_planes |= BIT(plane->id); @@ -402,10 +587,11 @@ int intel_plane_atomic_check(struct intel_atomic_state *state, struct intel_crtc_state *new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - if (new_crtc_state && new_crtc_state->bigjoiner_slave) { + if (new_crtc_state && intel_crtc_is_bigjoiner_slave(new_crtc_state)) { + struct intel_crtc *master_crtc = + intel_master_crtc(new_crtc_state); struct intel_plane *master_plane = - intel_crtc_get_plane(new_crtc_state->bigjoiner_linked_crtc, - plane->id); + intel_crtc_get_plane(master_crtc, plane->id); new_master_plane_state = intel_atomic_get_new_plane_state(state, master_plane); @@ -491,7 +677,7 @@ void intel_plane_update_arm(struct intel_plane *plane, trace_intel_plane_update_arm(&plane->base, crtc); - if (crtc_state->uapi.async_flip && plane->async_flip) + if (crtc_state->do_async_flip && plane->async_flip) plane->async_flip(plane, crtc_state, plane_state, true); else plane->update_arm(plane, crtc_state, plane_state); @@ -506,8 +692,8 @@ void intel_plane_disable_arm(struct intel_plane *plane, plane->disable_arm(plane, crtc_state); } -void intel_update_planes_on_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc) +void intel_crtc_planes_update_noarm(struct intel_atomic_state *state, + struct intel_crtc *crtc) { struct intel_crtc_state *new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); @@ -516,7 +702,7 @@ void intel_update_planes_on_crtc(struct intel_atomic_state *state, struct intel_plane *plane; int i; - if (new_crtc_state->uapi.async_flip) + if (new_crtc_state->do_async_flip) return; /* @@ -535,8 +721,8 @@ void intel_update_planes_on_crtc(struct intel_atomic_state *state, } } -void skl_arm_planes_on_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc) +static void skl_crtc_planes_update_arm(struct intel_atomic_state *state, + struct intel_crtc *crtc) { struct intel_crtc_state *old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc); @@ -570,8 +756,8 @@ void skl_arm_planes_on_crtc(struct intel_atomic_state *state, } } -void i9xx_arm_planes_on_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc) +static void i9xx_crtc_planes_update_arm(struct intel_atomic_state *state, + struct intel_crtc *crtc) { struct intel_crtc_state *new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); @@ -596,11 +782,23 @@ void i9xx_arm_planes_on_crtc(struct intel_atomic_state *state, } } +void intel_crtc_planes_update_arm(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + + if (DISPLAY_VER(i915) >= 9) + skl_crtc_planes_update_arm(state, crtc); + else + i9xx_crtc_planes_update_arm(state, crtc); +} + int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, struct intel_crtc_state *crtc_state, int min_scale, int max_scale, bool can_position) { + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); struct drm_framebuffer *fb = plane_state->hw.fb; struct drm_rect *src = &plane_state->uapi.src; struct drm_rect *dst = &plane_state->uapi.dst; @@ -619,7 +817,7 @@ int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale); vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale); if (hscale < 0 || vscale < 0) { - DRM_DEBUG_KMS("Invalid scaling of plane\n"); + drm_dbg_kms(&i915->drm, "Invalid scaling of plane\n"); drm_rect_debug_print("src: ", src, true); drm_rect_debug_print("dst: ", dst, false); return -ERANGE; @@ -631,7 +829,7 @@ int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, } /* right side of the image is on the slave crtc, adjust dst to match */ - if (crtc_state->bigjoiner_slave) + if (intel_crtc_is_bigjoiner_slave(crtc_state)) drm_rect_translate(dst, -crtc_state->pipe_src_w, 0); /* @@ -644,7 +842,7 @@ int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, if (!can_position && plane_state->uapi.visible && !drm_rect_equals(dst, &clip)) { - DRM_DEBUG_KMS("Plane must cover entire CRTC\n"); + drm_dbg_kms(&i915->drm, "Plane must cover entire CRTC\n"); drm_rect_debug_print("dst: ", dst, false); drm_rect_debug_print("clip: ", &clip, false); return -EINVAL; diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.h b/drivers/gpu/drm/i915/display/intel_atomic_plane.h index 7907f601598e..f4763a53541e 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.h +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.h @@ -16,6 +16,7 @@ struct intel_crtc; struct intel_crtc_state; struct intel_plane; struct intel_plane_state; +enum plane_id; unsigned int intel_adjusted_rate(const struct drm_rect *src, const struct drm_rect *dst, @@ -43,22 +44,16 @@ void intel_plane_free(struct intel_plane *plane); struct drm_plane_state *intel_plane_duplicate_state(struct drm_plane *plane); void intel_plane_destroy_state(struct drm_plane *plane, struct drm_plane_state *state); -void intel_update_planes_on_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc); -void skl_arm_planes_on_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc); -void i9xx_arm_planes_on_crtc(struct intel_atomic_state *state, - struct intel_crtc *crtc); +void intel_crtc_planes_update_noarm(struct intel_atomic_state *state, + struct intel_crtc *crtc); +void intel_crtc_planes_update_arm(struct intel_atomic_state *state, + struct intel_crtc *crtc); int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, struct intel_crtc_state *crtc_state, const struct intel_plane_state *old_plane_state, struct intel_plane_state *intel_state); int intel_plane_atomic_check(struct intel_atomic_state *state, struct intel_plane *plane); -int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *crtc_state, - const struct intel_plane_state *old_plane_state, - struct intel_plane_state *plane_state); int intel_plane_calc_min_cdclk(struct intel_atomic_state *state, struct intel_plane *plane, bool *need_cdclk_calc); diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c index 9523411cddd8..98f7ea44042f 100644 --- a/drivers/gpu/drm/i915/display/intel_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_backlight.c @@ -13,6 +13,7 @@ #include "intel_dp_aux_backlight.h" #include "intel_dsi_dcs_backlight.h" #include "intel_panel.h" +#include "intel_pci_config.h" /** * scale - scale values from one range to another @@ -433,6 +434,8 @@ static void ext_pwm_disable_backlight(const struct drm_connector_state *old_conn struct intel_connector *connector = to_intel_connector(old_conn_state->connector); struct intel_panel *panel = &connector->panel; + intel_backlight_set_pwm_level(old_conn_state, level); + panel->backlight.pwm_state.enabled = false; pwm_apply_state(panel->backlight.pwm, &panel->backlight.pwm_state); } diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c index 9d989c9f5da4..40b5e7ed12c2 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.c +++ b/drivers/gpu/drm/i915/display/intel_bios.c @@ -25,13 +25,14 @@ * */ -#include <drm/drm_dp_helper.h> +#include <drm/dp/drm_dp_helper.h> #include "display/intel_display.h" #include "display/intel_display_types.h" #include "display/intel_gmbus.h" #include "i915_drv.h" +#include "i915_reg.h" #define _INTEL_BIOS_PRIVATE #include "intel_vbt_defs.h" @@ -595,6 +596,12 @@ parse_general_features(struct drm_i915_private *i915, } else { i915->vbt.orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN; } + + if (bdb->version >= 249 && general->afc_startup_config) { + i915->vbt.override_afc_startup = true; + i915->vbt.override_afc_startup_val = general->afc_startup_config == 0x1 ? 0x0 : 0x7; + } + drm_dbg_kms(&i915->drm, "BDB_GENERAL_FEATURES int_tv_support %d int_crt_support %d lvds_use_ssc %d lvds_ssc_freq %d display_clock_mode %d fdi_rx_polarity_inverted %d\n", i915->vbt.int_tv_support, @@ -905,26 +912,6 @@ parse_psr(struct drm_i915_private *i915, const struct bdb_header *bdb) i915->vbt.psr.idle_frames = psr_table->idle_frames < 0 ? 0 : psr_table->idle_frames > 15 ? 15 : psr_table->idle_frames; - switch (psr_table->lines_to_wait) { - case 0: - i915->vbt.psr.lines_to_wait = PSR_0_LINES_TO_WAIT; - break; - case 1: - i915->vbt.psr.lines_to_wait = PSR_1_LINE_TO_WAIT; - break; - case 2: - i915->vbt.psr.lines_to_wait = PSR_4_LINES_TO_WAIT; - break; - case 3: - i915->vbt.psr.lines_to_wait = PSR_8_LINES_TO_WAIT; - break; - default: - drm_dbg_kms(&i915->drm, - "VBT has unknown PSR lines to wait %u\n", - psr_table->lines_to_wait); - break; - } - /* * New psr options 0=500us, 1=100us, 2=2500us, 3=0us * Old decimal value is wake up time in multiples of 100 us. @@ -2073,14 +2060,16 @@ static void parse_ddi_port(struct drm_i915_private *i915, i915->vbt.ports[port] = devdata; } +static bool has_ddi_port_info(struct drm_i915_private *i915) +{ + return DISPLAY_VER(i915) >= 5 || IS_G4X(i915); +} + static void parse_ddi_ports(struct drm_i915_private *i915) { struct intel_bios_encoder_data *devdata; - if (!HAS_DDI(i915) && !IS_CHERRYVIEW(i915)) - return; - - if (i915->vbt.version < 155) + if (!has_ddi_port_info(i915)) return; list_for_each_entry(devdata, &i915->vbt.display_devices, node) @@ -2335,6 +2324,63 @@ bool intel_bios_is_valid_vbt(const void *buf, size_t size) return vbt; } +static struct vbt_header *spi_oprom_get_vbt(struct drm_i915_private *i915) +{ + u32 count, data, found, store = 0; + u32 static_region, oprom_offset; + u32 oprom_size = 0x200000; + u16 vbt_size; + u32 *vbt; + + static_region = intel_uncore_read(&i915->uncore, SPI_STATIC_REGIONS); + static_region &= OPTIONROM_SPI_REGIONID_MASK; + intel_uncore_write(&i915->uncore, PRIMARY_SPI_REGIONID, static_region); + + oprom_offset = intel_uncore_read(&i915->uncore, OROM_OFFSET); + oprom_offset &= OROM_OFFSET_MASK; + + for (count = 0; count < oprom_size; count += 4) { + intel_uncore_write(&i915->uncore, PRIMARY_SPI_ADDRESS, oprom_offset + count); + data = intel_uncore_read(&i915->uncore, PRIMARY_SPI_TRIGGER); + + if (data == *((const u32 *)"$VBT")) { + found = oprom_offset + count; + break; + } + } + + if (count >= oprom_size) + goto err_not_found; + + /* Get VBT size and allocate space for the VBT */ + intel_uncore_write(&i915->uncore, PRIMARY_SPI_ADDRESS, found + + offsetof(struct vbt_header, vbt_size)); + vbt_size = intel_uncore_read(&i915->uncore, PRIMARY_SPI_TRIGGER); + vbt_size &= 0xffff; + + vbt = kzalloc(round_up(vbt_size, 4), GFP_KERNEL); + if (!vbt) + goto err_not_found; + + for (count = 0; count < vbt_size; count += 4) { + intel_uncore_write(&i915->uncore, PRIMARY_SPI_ADDRESS, found + count); + data = intel_uncore_read(&i915->uncore, PRIMARY_SPI_TRIGGER); + *(vbt + store++) = data; + } + + if (!intel_bios_is_valid_vbt(vbt, vbt_size)) + goto err_free_vbt; + + drm_dbg_kms(&i915->drm, "Found valid VBT in SPI flash\n"); + + return (struct vbt_header *)vbt; + +err_free_vbt: + kfree(vbt); +err_not_found: + return NULL; +} + static struct vbt_header *oprom_get_vbt(struct drm_i915_private *i915) { struct pci_dev *pdev = to_pci_dev(i915->drm.dev); @@ -2384,6 +2430,8 @@ static struct vbt_header *oprom_get_vbt(struct drm_i915_private *i915) pci_unmap_rom(pdev, oprom); + drm_dbg_kms(&i915->drm, "Found valid VBT in PCI ROM\n"); + return vbt; err_free_vbt: @@ -2418,17 +2466,23 @@ void intel_bios_init(struct drm_i915_private *i915) init_vbt_defaults(i915); - /* If the OpRegion does not have VBT, look in PCI ROM. */ + /* + * If the OpRegion does not have VBT, look in SPI flash through MMIO or + * PCI mapping + */ + if (!vbt && IS_DGFX(i915)) { + oprom_vbt = spi_oprom_get_vbt(i915); + vbt = oprom_vbt; + } + if (!vbt) { oprom_vbt = oprom_get_vbt(i915); - if (!oprom_vbt) - goto out; - vbt = oprom_vbt; - - drm_dbg_kms(&i915->drm, "Found valid VBT in PCI ROM\n"); } + if (!vbt) + goto out; + bdb = get_bdb_header(vbt); i915->vbt.version = bdb->version; @@ -2596,37 +2650,10 @@ bool intel_bios_is_lvds_present(struct drm_i915_private *i915, u8 *i2c_pin) */ bool intel_bios_is_port_present(struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata; - const struct child_device_config *child; - static const struct { - u16 dp, hdmi; - } port_mapping[] = { - [PORT_B] = { DVO_PORT_DPB, DVO_PORT_HDMIB, }, - [PORT_C] = { DVO_PORT_DPC, DVO_PORT_HDMIC, }, - [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, }, - [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, }, - [PORT_F] = { DVO_PORT_DPF, DVO_PORT_HDMIF, }, - }; - - if (HAS_DDI(i915)) - return i915->vbt.ports[port]; - - /* FIXME maybe deal with port A as well? */ - if (drm_WARN_ON(&i915->drm, - port == PORT_A) || port >= ARRAY_SIZE(port_mapping)) - return false; - - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { - child = &devdata->child; - - if ((child->dvo_port == port_mapping[port].dp || - child->dvo_port == port_mapping[port].hdmi) && - (child->device_type & (DEVICE_TYPE_TMDS_DVI_SIGNALING | - DEVICE_TYPE_DISPLAYPORT_OUTPUT))) - return true; - } + if (WARN_ON(!has_ddi_port_info(i915))) + return true; - return false; + return i915->vbt.ports[port]; } /** @@ -2638,40 +2665,18 @@ bool intel_bios_is_port_present(struct drm_i915_private *i915, enum port port) */ bool intel_bios_is_port_edp(struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata; - const struct child_device_config *child; - static const short port_mapping[] = { - [PORT_B] = DVO_PORT_DPB, - [PORT_C] = DVO_PORT_DPC, - [PORT_D] = DVO_PORT_DPD, - [PORT_E] = DVO_PORT_DPE, - [PORT_F] = DVO_PORT_DPF, - }; - - if (HAS_DDI(i915)) { - const struct intel_bios_encoder_data *devdata; - - devdata = intel_bios_encoder_data_lookup(i915, port); + const struct intel_bios_encoder_data *devdata = + intel_bios_encoder_data_lookup(i915, port); - return devdata && intel_bios_encoder_supports_edp(devdata); - } - - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { - child = &devdata->child; - - if (child->dvo_port == port_mapping[port] && - (child->device_type & DEVICE_TYPE_eDP_BITS) == - (DEVICE_TYPE_eDP & DEVICE_TYPE_eDP_BITS)) - return true; - } - - return false; + return devdata && intel_bios_encoder_supports_edp(devdata); } -static bool child_dev_is_dp_dual_mode(const struct child_device_config *child) +static bool intel_bios_encoder_supports_dp_dual_mode(const struct intel_bios_encoder_data *devdata) { - if ((child->device_type & DEVICE_TYPE_DP_DUAL_MODE_BITS) != - (DEVICE_TYPE_DP_DUAL_MODE & DEVICE_TYPE_DP_DUAL_MODE_BITS)) + const struct child_device_config *child = &devdata->child; + + if (!intel_bios_encoder_supports_dp(devdata) || + !intel_bios_encoder_supports_hdmi(devdata)) return false; if (dvo_port_type(child->dvo_port) == DVO_PORT_DPA) @@ -2688,40 +2693,10 @@ static bool child_dev_is_dp_dual_mode(const struct child_device_config *child) bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *i915, enum port port) { - static const struct { - u16 dp, hdmi; - } port_mapping[] = { - /* - * Buggy VBTs may declare DP ports as having - * HDMI type dvo_port :( So let's check both. - */ - [PORT_B] = { DVO_PORT_DPB, DVO_PORT_HDMIB, }, - [PORT_C] = { DVO_PORT_DPC, DVO_PORT_HDMIC, }, - [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, }, - [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, }, - [PORT_F] = { DVO_PORT_DPF, DVO_PORT_HDMIF, }, - }; - const struct intel_bios_encoder_data *devdata; - - if (HAS_DDI(i915)) { - const struct intel_bios_encoder_data *devdata; - - devdata = intel_bios_encoder_data_lookup(i915, port); + const struct intel_bios_encoder_data *devdata = + intel_bios_encoder_data_lookup(i915, port); - return devdata && child_dev_is_dp_dual_mode(&devdata->child); - } - - if (port == PORT_A || port >= ARRAY_SIZE(port_mapping)) - return false; - - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { - if ((devdata->child.dvo_port == port_mapping[port].dp || - devdata->child.dvo_port == port_mapping[port].hdmi) && - child_dev_is_dp_dual_mode(&devdata->child)) - return true; - } - - return false; + return devdata && intel_bios_encoder_supports_dp_dual_mode(devdata); } /** diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c index 8ac196e814d5..ad1564ca7269 100644 --- a/drivers/gpu/drm/i915/display/intel_bw.c +++ b/drivers/gpu/drm/i915/display/intel_bw.c @@ -5,10 +5,12 @@ #include <drm/drm_atomic_state_helper.h> +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_bw.h" #include "intel_cdclk.h" #include "intel_display_types.h" +#include "intel_mchbar_regs.h" #include "intel_pcode.h" #include "intel_pm.h" @@ -75,10 +77,9 @@ static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv, u16 dclk; int ret; - ret = sandybridge_pcode_read(dev_priv, - ICL_PCODE_MEM_SUBSYSYSTEM_INFO | - ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point), - &val, &val2); + ret = snb_pcode_read(dev_priv, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | + ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point), + &val, &val2); if (ret) return ret; @@ -102,10 +103,8 @@ static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv, int ret; int i; - ret = sandybridge_pcode_read(dev_priv, - ICL_PCODE_MEM_SUBSYSYSTEM_INFO | - ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, - &val, NULL); + ret = snb_pcode_read(dev_priv, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | + ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL); if (ret) return ret; @@ -675,6 +674,49 @@ intel_atomic_get_bw_state(struct intel_atomic_state *state) return to_intel_bw_state(bw_state); } +static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state, + const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe]; + enum plane_id plane_id; + + memset(&crtc_bw->used_bw, 0, sizeof(crtc_bw->used_bw)); + + if (!crtc_state->hw.active) + return; + + for_each_plane_id_on_crtc(crtc, plane_id) { + const struct skl_ddb_entry *ddb_y = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; + const struct skl_ddb_entry *ddb_uv = + &crtc_state->wm.skl.plane_ddb_uv[plane_id]; + unsigned int data_rate = crtc_state->data_rate[plane_id]; + unsigned int dbuf_mask = 0; + enum dbuf_slice slice; + + dbuf_mask |= skl_ddb_dbuf_slice_mask(i915, ddb_y); + dbuf_mask |= skl_ddb_dbuf_slice_mask(i915, ddb_uv); + + /* + * FIXME: To calculate that more properly we probably + * need to split per plane data_rate into data_rate_y + * and data_rate_uv for multiplanar formats in order not + * to get accounted those twice if they happen to reside + * on different slices. + * However for pre-icl this would work anyway because + * we have only single slice and for icl+ uv plane has + * non-zero data rate. + * So in worst case those calculation are a bit + * pessimistic, which shouldn't pose any significant + * problem anyway. + */ + for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) + crtc_bw->used_bw[slice] += data_rate; + } +} + int skl_bw_calc_min_cdclk(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); @@ -687,50 +729,13 @@ int skl_bw_calc_min_cdclk(struct intel_atomic_state *state) int i; for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { - enum plane_id plane_id; - struct intel_dbuf_bw *crtc_bw; - new_bw_state = intel_atomic_get_bw_state(state); if (IS_ERR(new_bw_state)) return PTR_ERR(new_bw_state); old_bw_state = intel_atomic_get_old_bw_state(state); - crtc_bw = &new_bw_state->dbuf_bw[crtc->pipe]; - - memset(&crtc_bw->used_bw, 0, sizeof(crtc_bw->used_bw)); - - if (!crtc_state->hw.active) - continue; - - for_each_plane_id_on_crtc(crtc, plane_id) { - const struct skl_ddb_entry *plane_alloc = - &crtc_state->wm.skl.plane_ddb_y[plane_id]; - const struct skl_ddb_entry *uv_plane_alloc = - &crtc_state->wm.skl.plane_ddb_uv[plane_id]; - unsigned int data_rate = crtc_state->data_rate[plane_id]; - unsigned int dbuf_mask = 0; - enum dbuf_slice slice; - - dbuf_mask |= skl_ddb_dbuf_slice_mask(dev_priv, plane_alloc); - dbuf_mask |= skl_ddb_dbuf_slice_mask(dev_priv, uv_plane_alloc); - - /* - * FIXME: To calculate that more properly we probably - * need to to split per plane data_rate into data_rate_y - * and data_rate_uv for multiplanar formats in order not - * to get accounted those twice if they happen to reside - * on different slices. - * However for pre-icl this would work anyway because - * we have only single slice and for icl+ uv plane has - * non-zero data rate. - * So in worst case those calculation are a bit - * pessimistic, which shouldn't pose any significant - * problem anyway. - */ - for_each_dbuf_slice_in_mask(dev_priv, slice, dbuf_mask) - crtc_bw->used_bw[slice] += data_rate; - } + skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state); } if (!old_bw_state) @@ -811,26 +816,11 @@ int intel_bw_calc_min_cdclk(struct intel_atomic_state *state) return 0; } -int intel_bw_atomic_check(struct intel_atomic_state *state) +static u16 icl_qgv_points_mask(struct drm_i915_private *i915) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_crtc_state *new_crtc_state, *old_crtc_state; - struct intel_bw_state *new_bw_state = NULL; - const struct intel_bw_state *old_bw_state = NULL; - unsigned int data_rate; - unsigned int num_active_planes; - struct intel_crtc *crtc; - int i, ret; - u32 allowed_points = 0; - unsigned int max_bw_point = 0, max_bw = 0; - unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points; - unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points; - bool changed = false; - u32 mask = 0; - - /* FIXME earlier gens need some checks too */ - if (DISPLAY_VER(dev_priv) < 11) - return 0; + unsigned int num_psf_gv_points = i915->max_bw[0].num_psf_gv_points; + unsigned int num_qgv_points = i915->max_bw[0].num_qgv_points; + u16 mask = 0; /* * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects @@ -843,6 +833,16 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) if (num_psf_gv_points > 0) mask |= REG_GENMASK(num_psf_gv_points - 1, 0) << ADLS_PSF_PT_SHIFT; + return mask; +} + +static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_crtc_state *new_crtc_state, *old_crtc_state; + struct intel_crtc *crtc; + int i; + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { unsigned int old_data_rate = @@ -853,6 +853,7 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) intel_bw_crtc_num_active_planes(old_crtc_state); unsigned int new_active_planes = intel_bw_crtc_num_active_planes(new_crtc_state); + struct intel_bw_state *new_bw_state; /* * Avoid locking the bw state when @@ -869,15 +870,40 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) new_bw_state->data_rate[crtc->pipe] = new_data_rate; new_bw_state->num_active_planes[crtc->pipe] = new_active_planes; - changed = true; + *changed = true; - drm_dbg_kms(&dev_priv->drm, - "pipe %c data rate %u num active planes %u\n", - pipe_name(crtc->pipe), + drm_dbg_kms(&i915->drm, + "[CRTC:%d:%s] data rate %u num active planes %u\n", + crtc->base.base.id, crtc->base.name, new_bw_state->data_rate[crtc->pipe], new_bw_state->num_active_planes[crtc->pipe]); } + return 0; +} + +int intel_bw_atomic_check(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_bw_state *old_bw_state; + struct intel_bw_state *new_bw_state; + unsigned int data_rate; + unsigned int num_active_planes; + int i, ret; + u32 allowed_points = 0; + unsigned int max_bw_point = 0, max_bw = 0; + unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points; + unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points; + bool changed = false; + + /* FIXME earlier gens need some checks too */ + if (DISPLAY_VER(dev_priv) < 11) + return 0; + + ret = intel_bw_check_data_rate(state, &changed); + if (ret) + return ret; + old_bw_state = intel_atomic_get_old_bw_state(state); new_bw_state = intel_atomic_get_new_bw_state(state); @@ -974,7 +1000,8 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) * We store the ones which need to be masked as that is what PCode * actually accepts as a parameter. */ - new_bw_state->qgv_points_mask = ~allowed_points & mask; + new_bw_state->qgv_points_mask = ~allowed_points & + icl_qgv_points_mask(dev_priv); /* * If the actual mask had changed we need to make sure that diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index c30cf8d2b835..8888fda8b701 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -23,6 +23,7 @@ #include <linux/time.h> +#include "hsw_ips.h" #include "intel_atomic.h" #include "intel_atomic_plane.h" #include "intel_audio.h" @@ -31,6 +32,8 @@ #include "intel_crtc.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_mchbar_regs.h" +#include "intel_pci_config.h" #include "intel_pcode.h" #include "intel_psr.h" #include "vlv_sideband.h" @@ -63,6 +66,17 @@ * dividers can be programmed correctly. */ +struct intel_cdclk_funcs { + void (*get_cdclk)(struct drm_i915_private *i915, + struct intel_cdclk_config *cdclk_config); + void (*set_cdclk)(struct drm_i915_private *i915, + const struct intel_cdclk_config *cdclk_config, + enum pipe pipe); + int (*bw_calc_min_cdclk)(struct intel_atomic_state *state); + int (*modeset_calc_cdclk)(struct intel_cdclk_state *state); + u8 (*calc_voltage_level)(int cdclk); +}; + void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv, struct intel_cdclk_config *cdclk_config) { @@ -793,8 +807,7 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv, "trying to change cdclk frequency with cdclk not enabled\n")) return; - ret = sandybridge_pcode_write(dev_priv, - BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0); + ret = snb_pcode_write(dev_priv, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0); if (ret) { drm_err(&dev_priv->drm, "failed to inform pcode about cdclk change\n"); @@ -822,8 +835,8 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv, LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1)) drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n"); - sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, - cdclk_config->voltage_level); + snb_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, + cdclk_config->voltage_level); intel_de_write(dev_priv, CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1); @@ -1126,8 +1139,8 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, intel_de_posting_read(dev_priv, CDCLK_CTL); /* inform PCU of the change */ - sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, - cdclk_config->voltage_level); + snb_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, + cdclk_config->voltage_level); intel_update_cdclk(dev_priv); } @@ -1145,7 +1158,7 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv) goto sanitize; intel_update_cdclk(dev_priv); - intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK"); + intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK"); /* Is PLL enabled and locked ? */ if (dev_priv->cdclk.hw.vco == 0 || @@ -1614,7 +1627,7 @@ static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco) /* Timeout 200us */ if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1)) - DRM_ERROR("timeout waiting for FREQ change request ack\n"); + drm_err(&dev_priv->drm, "timeout waiting for FREQ change request ack\n"); val &= ~BXT_DE_PLL_FREQ_REQ; intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val); @@ -1705,10 +1718,9 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, * BSpec requires us to wait up to 150usec, but that leads to * timeouts; the 2ms used here is based on experiment. */ - ret = sandybridge_pcode_write_timeout(dev_priv, - HSW_PCODE_DE_WRITE_FREQ_REQ, - 0x80000000, 150, 2); - + ret = snb_pcode_write_timeout(dev_priv, + HSW_PCODE_DE_WRITE_FREQ_REQ, + 0x80000000, 150, 2); if (ret) { drm_err(&dev_priv->drm, "Failed to inform PCU about cdclk change (err %d, freq %d)\n", @@ -1769,8 +1781,8 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe)); if (DISPLAY_VER(dev_priv) >= 11) { - ret = sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, - cdclk_config->voltage_level); + ret = snb_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, + cdclk_config->voltage_level); } else { /* * The timeout isn't specified, the 2ms used here is based on @@ -1778,10 +1790,10 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, * FIXME: Waiting for the request completion could be delayed * until the next PCODE request based on BSpec. */ - ret = sandybridge_pcode_write_timeout(dev_priv, - HSW_PCODE_DE_WRITE_FREQ_REQ, - cdclk_config->voltage_level, - 150, 2); + ret = snb_pcode_write_timeout(dev_priv, + HSW_PCODE_DE_WRITE_FREQ_REQ, + cdclk_config->voltage_level, + 150, 2); } if (ret) { @@ -1807,7 +1819,7 @@ static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv) int cdclk, clock, vco; intel_update_cdclk(dev_priv); - intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK"); + intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK"); if (dev_priv->cdclk.hw.vco == 0 || dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass) @@ -2047,13 +2059,14 @@ static bool intel_cdclk_changed(const struct intel_cdclk_config *a, a->voltage_level != b->voltage_level; } -void intel_dump_cdclk_config(const struct intel_cdclk_config *cdclk_config, +void intel_cdclk_dump_config(struct drm_i915_private *i915, + const struct intel_cdclk_config *cdclk_config, const char *context) { - DRM_DEBUG_DRIVER("%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n", - context, cdclk_config->cdclk, cdclk_config->vco, - cdclk_config->ref, cdclk_config->bypass, - cdclk_config->voltage_level); + drm_dbg_kms(&i915->drm, "%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n", + context, cdclk_config->cdclk, cdclk_config->vco, + cdclk_config->ref, cdclk_config->bypass, + cdclk_config->voltage_level); } /** @@ -2077,7 +2090,7 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->cdclk_funcs->set_cdclk)) return; - intel_dump_cdclk_config(cdclk_config, "Changing CDCLK to"); + intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to"); for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); @@ -2120,8 +2133,8 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, if (drm_WARN(&dev_priv->drm, intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config), "cdclk state doesn't match!\n")) { - intel_dump_cdclk_config(&dev_priv->cdclk.hw, "[hw state]"); - intel_dump_cdclk_config(cdclk_config, "[sw state]"); + intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "[hw state]"); + intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]"); } } diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.h b/drivers/gpu/drm/i915/display/intel_cdclk.h index fc638522e445..df66f66fbad0 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.h +++ b/drivers/gpu/drm/i915/display/intel_cdclk.h @@ -8,7 +8,6 @@ #include <linux/types.h> -#include "i915_drv.h" #include "intel_display.h" #include "intel_global_state.h" @@ -16,6 +15,11 @@ struct drm_i915_private; struct intel_atomic_state; struct intel_crtc_state; +struct intel_cdclk_config { + unsigned int cdclk, vco, ref, bypass; + u8 voltage_level; +}; + struct intel_cdclk_state { struct intel_global_state base; @@ -58,7 +62,8 @@ bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a, const struct intel_cdclk_config *b); void intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state); void intel_set_cdclk_post_plane_update(struct intel_atomic_state *state); -void intel_dump_cdclk_config(const struct intel_cdclk_config *cdclk_config, +void intel_cdclk_dump_config(struct drm_i915_private *i915, + const struct intel_cdclk_config *cdclk_config, const char *context); int intel_modeset_calc_cdclk(struct intel_atomic_state *state); void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv, diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index de3ded1e327a..e94ec57260f1 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -28,6 +28,25 @@ #include "intel_dpll.h" #include "vlv_dsi_pll.h" +struct intel_color_funcs { + int (*color_check)(struct intel_crtc_state *crtc_state); + /* + * Program double buffered color management registers during + * vblank evasion. The registers should then latch during the + * next vblank start, alongside any other double buffered registers + * involved with the same commit. + */ + void (*color_commit)(const struct intel_crtc_state *crtc_state); + /* + * Load LUTs (and other single buffered color management + * registers). Will (hopefully) be called during the vblank + * following the latching of any double buffered registers + * involved with the same commit. + */ + void (*load_luts)(const struct intel_crtc_state *crtc_state); + void (*read_luts)(struct intel_crtc_state *crtc_state); +}; + #define CTM_COEFF_SIGN (1ULL << 63) #define CTM_COEFF_1_0 (1ULL << 32) @@ -160,29 +179,29 @@ static void ilk_update_pipe_csc(struct intel_crtc *crtc, struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - intel_de_write(dev_priv, PIPE_CSC_PREOFF_HI(pipe), preoff[0]); - intel_de_write(dev_priv, PIPE_CSC_PREOFF_ME(pipe), preoff[1]); - intel_de_write(dev_priv, PIPE_CSC_PREOFF_LO(pipe), preoff[2]); + intel_de_write_fw(dev_priv, PIPE_CSC_PREOFF_HI(pipe), preoff[0]); + intel_de_write_fw(dev_priv, PIPE_CSC_PREOFF_ME(pipe), preoff[1]); + intel_de_write_fw(dev_priv, PIPE_CSC_PREOFF_LO(pipe), preoff[2]); - intel_de_write(dev_priv, PIPE_CSC_COEFF_RY_GY(pipe), - coeff[0] << 16 | coeff[1]); - intel_de_write(dev_priv, PIPE_CSC_COEFF_BY(pipe), coeff[2] << 16); + intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_RY_GY(pipe), + coeff[0] << 16 | coeff[1]); + intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_BY(pipe), coeff[2] << 16); - intel_de_write(dev_priv, PIPE_CSC_COEFF_RU_GU(pipe), - coeff[3] << 16 | coeff[4]); - intel_de_write(dev_priv, PIPE_CSC_COEFF_BU(pipe), coeff[5] << 16); + intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_RU_GU(pipe), + coeff[3] << 16 | coeff[4]); + intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_BU(pipe), coeff[5] << 16); - intel_de_write(dev_priv, PIPE_CSC_COEFF_RV_GV(pipe), - coeff[6] << 16 | coeff[7]); - intel_de_write(dev_priv, PIPE_CSC_COEFF_BV(pipe), coeff[8] << 16); + intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_RV_GV(pipe), + coeff[6] << 16 | coeff[7]); + intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_BV(pipe), coeff[8] << 16); if (DISPLAY_VER(dev_priv) >= 7) { - intel_de_write(dev_priv, PIPE_CSC_POSTOFF_HI(pipe), - postoff[0]); - intel_de_write(dev_priv, PIPE_CSC_POSTOFF_ME(pipe), - postoff[1]); - intel_de_write(dev_priv, PIPE_CSC_POSTOFF_LO(pipe), - postoff[2]); + intel_de_write_fw(dev_priv, PIPE_CSC_POSTOFF_HI(pipe), + postoff[0]); + intel_de_write_fw(dev_priv, PIPE_CSC_POSTOFF_ME(pipe), + postoff[1]); + intel_de_write_fw(dev_priv, PIPE_CSC_POSTOFF_LO(pipe), + postoff[2]); } } @@ -194,28 +213,28 @@ static void icl_update_output_csc(struct intel_crtc *crtc, struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_PREOFF_HI(pipe), preoff[0]); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_PREOFF_ME(pipe), preoff[1]); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_PREOFF_LO(pipe), preoff[2]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_PREOFF_HI(pipe), preoff[0]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_PREOFF_ME(pipe), preoff[1]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_PREOFF_LO(pipe), preoff[2]); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_COEFF_RY_GY(pipe), - coeff[0] << 16 | coeff[1]); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_COEFF_BY(pipe), - coeff[2] << 16); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_RY_GY(pipe), + coeff[0] << 16 | coeff[1]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_BY(pipe), + coeff[2] << 16); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_COEFF_RU_GU(pipe), - coeff[3] << 16 | coeff[4]); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_COEFF_BU(pipe), - coeff[5] << 16); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_RU_GU(pipe), + coeff[3] << 16 | coeff[4]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_BU(pipe), + coeff[5] << 16); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_COEFF_RV_GV(pipe), - coeff[6] << 16 | coeff[7]); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_COEFF_BV(pipe), - coeff[8] << 16); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_RV_GV(pipe), + coeff[6] << 16 | coeff[7]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_BV(pipe), + coeff[8] << 16); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_HI(pipe), postoff[0]); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_ME(pipe), postoff[1]); - intel_de_write(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_LO(pipe), postoff[2]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_HI(pipe), postoff[0]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_ME(pipe), postoff[1]); + intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_LO(pipe), postoff[2]); } static bool ilk_csc_limited_range(const struct intel_crtc_state *crtc_state) @@ -319,8 +338,8 @@ static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state) ilk_csc_off_zero); } - intel_de_write(dev_priv, PIPE_CSC_MODE(crtc->pipe), - crtc_state->csc_mode); + intel_de_write_fw(dev_priv, PIPE_CSC_MODE(crtc->pipe), + crtc_state->csc_mode); } static void icl_load_csc_matrix(const struct intel_crtc_state *crtc_state) @@ -346,8 +365,8 @@ static void icl_load_csc_matrix(const struct intel_crtc_state *crtc_state) ilk_csc_postoff_limited_range); } - intel_de_write(dev_priv, PIPE_CSC_MODE(crtc->pipe), - crtc_state->csc_mode); + intel_de_write_fw(dev_priv, PIPE_CSC_MODE(crtc->pipe), + crtc_state->csc_mode); } static void chv_load_cgm_csc(struct intel_crtc *crtc, @@ -377,16 +396,16 @@ static void chv_load_cgm_csc(struct intel_crtc *crtc, coeffs[i] |= (abs_coeff >> 20) & 0xfff; } - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF01(pipe), - coeffs[1] << 16 | coeffs[0]); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF23(pipe), - coeffs[3] << 16 | coeffs[2]); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF45(pipe), - coeffs[5] << 16 | coeffs[4]); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF67(pipe), - coeffs[7] << 16 | coeffs[6]); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF8(pipe), - coeffs[8]); + intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF01(pipe), + coeffs[1] << 16 | coeffs[0]); + intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF23(pipe), + coeffs[3] << 16 | coeffs[2]); + intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF45(pipe), + coeffs[5] << 16 | coeffs[4]); + intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF67(pipe), + coeffs[7] << 16 | coeffs[6]); + intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF8(pipe), + coeffs[8]); } /* convert hw value with given bit_precision to lut property val */ diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy.c b/drivers/gpu/drm/i915/display/intel_combo_phy.c index f628e0542933..4dfe77351b8b 100644 --- a/drivers/gpu/drm/i915/display/intel_combo_phy.c +++ b/drivers/gpu/drm/i915/display/intel_combo_phy.c @@ -4,6 +4,7 @@ */ #include "intel_combo_phy.h" +#include "intel_combo_phy_regs.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h b/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h new file mode 100644 index 000000000000..2ed65193ca19 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_combo_phy_regs.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_COMBO_PHY_REGS__ +#define __INTEL_COMBO_PHY_REGS__ + +#include "i915_reg_defs.h" + +#define _ICL_COMBOPHY_A 0x162000 +#define _ICL_COMBOPHY_B 0x6C000 +#define _EHL_COMBOPHY_C 0x160000 +#define _RKL_COMBOPHY_D 0x161000 +#define _ADL_COMBOPHY_E 0x16B000 + +#define _ICL_COMBOPHY(phy) _PICK(phy, _ICL_COMBOPHY_A, \ + _ICL_COMBOPHY_B, \ + _EHL_COMBOPHY_C, \ + _RKL_COMBOPHY_D, \ + _ADL_COMBOPHY_E) + +/* ICL Port CL_DW registers */ +#define _ICL_PORT_CL_DW(dw, phy) (_ICL_COMBOPHY(phy) + \ + 4 * (dw)) + +#define ICL_PORT_CL_DW5(phy) _MMIO(_ICL_PORT_CL_DW(5, phy)) +#define CL_POWER_DOWN_ENABLE (1 << 4) +#define SUS_CLOCK_CONFIG (3 << 0) + +#define ICL_PORT_CL_DW10(phy) _MMIO(_ICL_PORT_CL_DW(10, phy)) +#define PG_SEQ_DELAY_OVERRIDE_MASK (3 << 25) +#define PG_SEQ_DELAY_OVERRIDE_SHIFT 25 +#define PG_SEQ_DELAY_OVERRIDE_ENABLE (1 << 24) +#define PWR_UP_ALL_LANES (0x0 << 4) +#define PWR_DOWN_LN_3_2_1 (0xe << 4) +#define PWR_DOWN_LN_3_2 (0xc << 4) +#define PWR_DOWN_LN_3 (0x8 << 4) +#define PWR_DOWN_LN_2_1_0 (0x7 << 4) +#define PWR_DOWN_LN_1_0 (0x3 << 4) +#define PWR_DOWN_LN_3_1 (0xa << 4) +#define PWR_DOWN_LN_3_1_0 (0xb << 4) +#define PWR_DOWN_LN_MASK (0xf << 4) +#define PWR_DOWN_LN_SHIFT 4 +#define EDP4K2K_MODE_OVRD_EN (1 << 3) +#define EDP4K2K_MODE_OVRD_OPTIMIZED (1 << 2) + +#define ICL_PORT_CL_DW12(phy) _MMIO(_ICL_PORT_CL_DW(12, phy)) +#define ICL_LANE_ENABLE_AUX (1 << 0) + +/* ICL Port COMP_DW registers */ +#define _ICL_PORT_COMP 0x100 +#define _ICL_PORT_COMP_DW(dw, phy) (_ICL_COMBOPHY(phy) + \ + _ICL_PORT_COMP + 4 * (dw)) + +#define ICL_PORT_COMP_DW0(phy) _MMIO(_ICL_PORT_COMP_DW(0, phy)) +#define COMP_INIT (1 << 31) + +#define ICL_PORT_COMP_DW1(phy) _MMIO(_ICL_PORT_COMP_DW(1, phy)) + +#define ICL_PORT_COMP_DW3(phy) _MMIO(_ICL_PORT_COMP_DW(3, phy)) +#define PROCESS_INFO_DOT_0 (0 << 26) +#define PROCESS_INFO_DOT_1 (1 << 26) +#define PROCESS_INFO_DOT_4 (2 << 26) +#define PROCESS_INFO_MASK (7 << 26) +#define PROCESS_INFO_SHIFT 26 +#define VOLTAGE_INFO_0_85V (0 << 24) +#define VOLTAGE_INFO_0_95V (1 << 24) +#define VOLTAGE_INFO_1_05V (2 << 24) +#define VOLTAGE_INFO_MASK (3 << 24) +#define VOLTAGE_INFO_SHIFT 24 + +#define ICL_PORT_COMP_DW8(phy) _MMIO(_ICL_PORT_COMP_DW(8, phy)) +#define IREFGEN (1 << 24) + +#define ICL_PORT_COMP_DW9(phy) _MMIO(_ICL_PORT_COMP_DW(9, phy)) + +#define ICL_PORT_COMP_DW10(phy) _MMIO(_ICL_PORT_COMP_DW(10, phy)) + +/* ICL Port PCS registers */ +#define _ICL_PORT_PCS_AUX 0x300 +#define _ICL_PORT_PCS_GRP 0x600 +#define _ICL_PORT_PCS_LN(ln) (0x800 + (ln) * 0x100) +#define _ICL_PORT_PCS_DW_AUX(dw, phy) (_ICL_COMBOPHY(phy) + \ + _ICL_PORT_PCS_AUX + 4 * (dw)) +#define _ICL_PORT_PCS_DW_GRP(dw, phy) (_ICL_COMBOPHY(phy) + \ + _ICL_PORT_PCS_GRP + 4 * (dw)) +#define _ICL_PORT_PCS_DW_LN(dw, ln, phy) (_ICL_COMBOPHY(phy) + \ + _ICL_PORT_PCS_LN(ln) + 4 * (dw)) +#define ICL_PORT_PCS_DW1_AUX(phy) _MMIO(_ICL_PORT_PCS_DW_AUX(1, phy)) +#define ICL_PORT_PCS_DW1_GRP(phy) _MMIO(_ICL_PORT_PCS_DW_GRP(1, phy)) +#define ICL_PORT_PCS_DW1_LN(ln, phy) _MMIO(_ICL_PORT_PCS_DW_LN(1, ln, phy)) +#define DCC_MODE_SELECT_MASK (0x3 << 20) +#define DCC_MODE_SELECT_CONTINUOSLY (0x3 << 20) +#define COMMON_KEEPER_EN (1 << 26) +#define LATENCY_OPTIM_MASK (0x3 << 2) +#define LATENCY_OPTIM_VAL(x) ((x) << 2) + +/* ICL Port TX registers */ +#define _ICL_PORT_TX_AUX 0x380 +#define _ICL_PORT_TX_GRP 0x680 +#define _ICL_PORT_TX_LN(ln) (0x880 + (ln) * 0x100) + +#define _ICL_PORT_TX_DW_AUX(dw, phy) (_ICL_COMBOPHY(phy) + \ + _ICL_PORT_TX_AUX + 4 * (dw)) +#define _ICL_PORT_TX_DW_GRP(dw, phy) (_ICL_COMBOPHY(phy) + \ + _ICL_PORT_TX_GRP + 4 * (dw)) +#define _ICL_PORT_TX_DW_LN(dw, ln, phy) (_ICL_COMBOPHY(phy) + \ + _ICL_PORT_TX_LN(ln) + 4 * (dw)) + +#define ICL_PORT_TX_DW2_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(2, phy)) +#define ICL_PORT_TX_DW2_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(2, phy)) +#define ICL_PORT_TX_DW2_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(2, ln, phy)) +#define SWING_SEL_UPPER(x) (((x) >> 3) << 15) +#define SWING_SEL_UPPER_MASK (1 << 15) +#define SWING_SEL_LOWER(x) (((x) & 0x7) << 11) +#define SWING_SEL_LOWER_MASK (0x7 << 11) +#define FRC_LATENCY_OPTIM_MASK (0x7 << 8) +#define FRC_LATENCY_OPTIM_VAL(x) ((x) << 8) +#define RCOMP_SCALAR(x) ((x) << 0) +#define RCOMP_SCALAR_MASK (0xFF << 0) + +#define ICL_PORT_TX_DW4_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(4, phy)) +#define ICL_PORT_TX_DW4_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(4, phy)) +#define ICL_PORT_TX_DW4_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(4, ln, phy)) +#define LOADGEN_SELECT (1 << 31) +#define POST_CURSOR_1(x) ((x) << 12) +#define POST_CURSOR_1_MASK (0x3F << 12) +#define POST_CURSOR_2(x) ((x) << 6) +#define POST_CURSOR_2_MASK (0x3F << 6) +#define CURSOR_COEFF(x) ((x) << 0) +#define CURSOR_COEFF_MASK (0x3F << 0) + +#define ICL_PORT_TX_DW5_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(5, phy)) +#define ICL_PORT_TX_DW5_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(5, phy)) +#define ICL_PORT_TX_DW5_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(5, ln, phy)) +#define TX_TRAINING_EN (1 << 31) +#define TAP2_DISABLE (1 << 30) +#define TAP3_DISABLE (1 << 29) +#define SCALING_MODE_SEL(x) ((x) << 18) +#define SCALING_MODE_SEL_MASK (0x7 << 18) +#define RTERM_SELECT(x) ((x) << 3) +#define RTERM_SELECT_MASK (0x7 << 3) + +#define ICL_PORT_TX_DW7_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(7, phy)) +#define ICL_PORT_TX_DW7_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(7, phy)) +#define ICL_PORT_TX_DW7_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(7, ln, phy)) +#define N_SCALAR(x) ((x) << 24) +#define N_SCALAR_MASK (0x7F << 24) + +#define ICL_PORT_TX_DW8_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(8, phy)) +#define ICL_PORT_TX_DW8_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(8, phy)) +#define ICL_PORT_TX_DW8_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(8, ln, phy)) +#define ICL_PORT_TX_DW8_ODCC_CLK_SEL REG_BIT(31) +#define ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_MASK REG_GENMASK(30, 29) +#define ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_DIV2 REG_FIELD_PREP(ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_MASK, 0x1) + +#define _ICL_DPHY_CHKN_REG 0x194 +#define ICL_DPHY_CHKN(port) _MMIO(_ICL_COMBOPHY(port) + _ICL_DPHY_CHKN_REG) +#define ICL_DPHY_CHKN_AFE_OVER_PPI_STRAP REG_BIT(7) + +#endif /* __INTEL_COMBO_PHY_REGS__ */ diff --git a/drivers/gpu/drm/i915/display/intel_crt.h b/drivers/gpu/drm/i915/display/intel_crt.h index 6c5c44600cbd..c6071efd93ce 100644 --- a/drivers/gpu/drm/i915/display/intel_crt.h +++ b/drivers/gpu/drm/i915/display/intel_crt.h @@ -6,7 +6,7 @@ #ifndef __INTEL_CRT_H__ #define __INTEL_CRT_H__ -#include "i915_reg.h" +#include "i915_reg_defs.h" enum pipe; struct drm_encoder; diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c index 16c3ca66d9f0..65827481c1b1 100644 --- a/drivers/gpu/drm/i915/display/intel_crtc.c +++ b/drivers/gpu/drm/i915/display/intel_crtc.c @@ -12,6 +12,7 @@ #include <drm/drm_plane_helper.h> #include <drm/drm_vblank_work.h> +#include "i915_irq.h" #include "i915_vgpu.h" #include "i9xx_plane.h" #include "icl_dsi.h" @@ -484,7 +485,7 @@ void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state) intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI); DEFINE_WAIT(wait); - if (new_crtc_state->uapi.async_flip) + if (new_crtc_state->do_async_flip) return; if (intel_crtc_needs_vblank_work(new_crtc_state)) @@ -629,7 +630,7 @@ void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state) ktime_t end_vbl_time = ktime_get(); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - if (new_crtc_state->uapi.async_flip) + if (new_crtc_state->do_async_flip) return; trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end); diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c index 16d34685d83f..2ade8fdd9bdd 100644 --- a/drivers/gpu/drm/i915/display/intel_cursor.c +++ b/drivers/gpu/drm/i915/display/intel_cursor.c @@ -51,16 +51,16 @@ static u32 intel_cursor_position(const struct intel_plane_state *plane_state) u32 pos = 0; if (x < 0) { - pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT; + pos |= CURSOR_POS_X_SIGN; x = -x; } - pos |= x << CURSOR_X_SHIFT; + pos |= CURSOR_POS_X(x); if (y < 0) { - pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT; + pos |= CURSOR_POS_Y_SIGN; y = -y; } - pos |= y << CURSOR_Y_SHIFT; + pos |= CURSOR_POS_Y(y); return pos; } @@ -180,7 +180,7 @@ static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state) u32 cntl = 0; if (crtc_state->gamma_enable) - cntl |= CURSOR_GAMMA_ENABLE; + cntl |= CURSOR_PIPE_GAMMA_ENABLE; return cntl; } @@ -264,7 +264,7 @@ static void i845_cursor_update_arm(struct intel_plane *plane, cntl = plane_state->ctl | i845_cursor_ctl_crtc(crtc_state); - size = (height << 12) | width; + size = CURSOR_HEIGHT(height) | CURSOR_WIDTH(width); base = intel_cursor_base(plane_state); pos = intel_cursor_position(plane_state); @@ -280,7 +280,7 @@ static void i845_cursor_update_arm(struct intel_plane *plane, plane->cursor.cntl != cntl) { intel_de_write_fw(dev_priv, CURCNTR(PIPE_A), 0); intel_de_write_fw(dev_priv, CURBASE(PIPE_A), base); - intel_de_write_fw(dev_priv, CURSIZE, size); + intel_de_write_fw(dev_priv, CURSIZE(PIPE_A), size); intel_de_write_fw(dev_priv, CURPOS(PIPE_A), pos); intel_de_write_fw(dev_priv, CURCNTR(PIPE_A), cntl); @@ -340,13 +340,13 @@ static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state) return cntl; if (crtc_state->gamma_enable) - cntl = MCURSOR_GAMMA_ENABLE; + cntl = MCURSOR_PIPE_GAMMA_ENABLE; if (crtc_state->csc_enable) cntl |= MCURSOR_PIPE_CSC_ENABLE; if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) - cntl |= MCURSOR_PIPE_SELECT(crtc->pipe); + cntl |= MCURSOR_PIPE_SEL(crtc->pipe); return cntl; } @@ -502,7 +502,7 @@ static void i9xx_cursor_update_arm(struct intel_plane *plane, i9xx_cursor_ctl_crtc(crtc_state); if (width != height) - fbc_ctl = CUR_FBC_CTL_EN | (height - 1); + fbc_ctl = CUR_FBC_EN | CUR_FBC_HEIGHT(height - 1); base = intel_cursor_base(plane_state); pos = intel_cursor_position(plane_state); @@ -586,13 +586,12 @@ static bool i9xx_cursor_get_hw_state(struct intel_plane *plane, val = intel_de_read(dev_priv, CURCNTR(plane->pipe)); - ret = val & MCURSOR_MODE; + ret = val & MCURSOR_MODE_MASK; if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv)) *pipe = plane->pipe; else - *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >> - MCURSOR_PIPE_SELECT_SHIFT; + *pipe = REG_FIELD_GET(MCURSOR_PIPE_SEL_MASK, val); intel_display_power_put(dev_priv, power_domain, wakeref); diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index cab505277595..e4260806c2a4 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -32,6 +32,7 @@ #include "intel_audio.h" #include "intel_backlight.h" #include "intel_combo_phy.h" +#include "intel_combo_phy_regs.h" #include "intel_connector.h" #include "intel_crtc.h" #include "intel_ddi.h" @@ -56,6 +57,7 @@ #include "intel_snps_phy.h" #include "intel_sprite.h" #include "intel_tc.h" +#include "intel_tc_phy_regs.h" #include "intel_vdsc.h" #include "intel_vrr.h" #include "skl_scaler.h" @@ -2287,116 +2289,6 @@ static void intel_ddi_mso_configure(const struct intel_crtc_state *crtc_state) OVERLAP_PIXELS_MASK, dss1); } -static void dg2_ddi_pre_enable_dp(struct intel_atomic_state *state, - struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) -{ - struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_digital_port *dig_port = enc_to_dig_port(encoder); - bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); - - intel_dp_set_link_params(intel_dp, crtc_state->port_clock, - crtc_state->lane_count); - - /* - * We only configure what the register value will be here. Actual - * enabling happens during link training farther down. - */ - intel_ddi_init_dp_buf_reg(encoder, crtc_state); - - /* - * 1. Enable Power Wells - * - * This was handled at the beginning of intel_atomic_commit_tail(), - * before we called down into this function. - */ - - /* 2. Enable Panel Power if PPS is required */ - intel_pps_on(intel_dp); - - /* - * 3. Enable the port PLL. - */ - intel_ddi_enable_clock(encoder, crtc_state); - - /* 4. Enable IO power */ - if (!intel_tc_port_in_tbt_alt_mode(dig_port)) - dig_port->ddi_io_wakeref = intel_display_power_get(dev_priv, - dig_port->ddi_io_power_domain); - - /* - * 5. The rest of the below are substeps under the bspec's "Enable and - * Train Display Port" step. Note that steps that are specific to - * MST will be handled by intel_mst_pre_enable_dp() before/after it - * calls into this function. Also intel_mst_pre_enable_dp() only calls - * us when active_mst_links==0, so any steps designated for "single - * stream or multi-stream master transcoder" can just be performed - * unconditionally here. - */ - - /* - * 5.a Configure Transcoder Clock Select to direct the Port clock to the - * Transcoder. - */ - intel_ddi_enable_pipe_clock(encoder, crtc_state); - - /* 5.b Configure transcoder for DP 2.0 128b/132b */ - intel_ddi_config_transcoder_dp2(encoder, crtc_state); - - /* - * 5.c Configure TRANS_DDI_FUNC_CTL DDI Select, DDI Mode Select & MST - * Transport Select - */ - intel_ddi_config_transcoder_func(encoder, crtc_state); - - /* - * 5.d Configure & enable DP_TP_CTL with link training pattern 1 - * selected - * - * This will be handled by the intel_dp_start_link_train() farther - * down this function. - */ - - /* 5.e Configure voltage swing and related IO settings */ - encoder->set_signal_levels(encoder, crtc_state); - - if (!is_mst) - intel_dp_set_power(intel_dp, DP_SET_POWER_D0); - - intel_dp_configure_protocol_converter(intel_dp, crtc_state); - intel_dp_sink_set_decompression_state(intel_dp, crtc_state, true); - /* - * DDI FEC: "anticipates enabling FEC encoding sets the FEC_READY bit - * in the FEC_CONFIGURATION register to 1 before initiating link - * training - */ - intel_dp_sink_set_fec_ready(intel_dp, crtc_state); - intel_dp_check_frl_training(intel_dp); - intel_dp_pcon_dsc_configure(intel_dp, crtc_state); - - /* - * 5.h Follow DisplayPort specification training sequence (see notes for - * failure handling) - * 5.i If DisplayPort multi-stream - Set DP_TP_CTL link training to Idle - * Pattern, wait for 5 idle patterns (DP_TP_STATUS Min_Idles_Sent) - * (timeout after 800 us) - */ - intel_dp_start_link_train(intel_dp, crtc_state); - - /* 5.j Set DP_TP_CTL link training to Normal */ - if (!is_trans_port_sync_mode(crtc_state)) - intel_dp_stop_link_train(intel_dp, crtc_state); - - /* 5.k Configure and enable FEC if needed */ - intel_ddi_enable_fec(encoder, crtc_state); - - intel_dsc_dp_pps_write(encoder, crtc_state); - - intel_dsc_enable(crtc_state); -} - static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, @@ -2470,6 +2362,9 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, */ intel_ddi_enable_pipe_clock(encoder, crtc_state); + if (HAS_DP20(dev_priv)) + intel_ddi_config_transcoder_dp2(encoder, crtc_state); + /* * 7.b Configure TRANS_DDI_FUNC_CTL DDI Select, DDI Mode Select & MST * Transport Select @@ -2530,9 +2425,6 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, intel_ddi_enable_fec(encoder, crtc_state); intel_dsc_dp_pps_write(encoder, crtc_state); - - if (!crtc_state->bigjoiner) - intel_dsc_enable(crtc_state); } static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, @@ -2598,9 +2490,6 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, intel_ddi_enable_pipe_clock(encoder, crtc_state); intel_dsc_dp_pps_write(encoder, crtc_state); - - if (!crtc_state->bigjoiner) - intel_dsc_enable(crtc_state); } static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state, @@ -2610,9 +2499,7 @@ static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - if (IS_DG2(dev_priv)) - dg2_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state); - else if (DISPLAY_VER(dev_priv) >= 12) + if (DISPLAY_VER(dev_priv) >= 12) tgl_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state); else hsw_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state); @@ -2620,11 +2507,8 @@ static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state, /* MST will call a setting of MSA after an allocating of Virtual Channel * from MST encoder pre_enable callback. */ - if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)) { + if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)) intel_ddi_set_dp_msa(crtc_state, conn_state); - - intel_dp_set_m_n(crtc_state, M1_N1); - } } static void intel_ddi_pre_enable_hdmi(struct intel_atomic_state *state, @@ -2819,6 +2703,7 @@ static void intel_ddi_post_disable(struct intel_atomic_state *state, struct intel_digital_port *dig_port = enc_to_dig_port(encoder); enum phy phy = intel_port_to_phy(dev_priv, encoder->port); bool is_tc_port = intel_phy_is_tc(dev_priv, phy); + struct intel_crtc *slave_crtc; if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST)) { intel_crtc_vblank_off(old_crtc_state); @@ -2837,9 +2722,8 @@ static void intel_ddi_post_disable(struct intel_atomic_state *state, ilk_pfit_disable(old_crtc_state); } - if (old_crtc_state->bigjoiner_linked_crtc) { - struct intel_crtc *slave_crtc = - old_crtc_state->bigjoiner_linked_crtc; + for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, slave_crtc, + intel_crtc_bigjoiner_slave_pipes(old_crtc_state)) { const struct intel_crtc_state *old_slave_crtc_state = intel_atomic_get_old_crtc_state(state, slave_crtc); @@ -3042,7 +2926,7 @@ static void intel_enable_ddi(struct intel_atomic_state *state, { drm_WARN_ON(state->base.dev, crtc_state->has_pch_encoder); - if (!crtc_state->bigjoiner_slave) + if (!intel_crtc_is_bigjoiner_slave(crtc_state)) intel_ddi_enable_transcoder_func(encoder, crtc_state); intel_vrr_enable(encoder, crtc_state); @@ -3157,6 +3041,7 @@ intel_ddi_update_prepare(struct intel_atomic_state *state, struct intel_encoder *encoder, struct intel_crtc *crtc) { + struct drm_i915_private *i915 = to_i915(state->base.dev); struct intel_crtc_state *crtc_state = crtc ? intel_atomic_get_new_crtc_state(state, crtc) : NULL; int required_lanes = crtc_state ? crtc_state->lane_count : 1; @@ -3166,11 +3051,12 @@ intel_ddi_update_prepare(struct intel_atomic_state *state, intel_tc_port_get_link(enc_to_dig_port(encoder), required_lanes); if (crtc_state && crtc_state->hw.active) { - struct intel_crtc *slave_crtc = crtc_state->bigjoiner_linked_crtc; + struct intel_crtc *slave_crtc; intel_update_active_dpll(state, crtc, encoder); - if (slave_crtc) + for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc, + intel_crtc_bigjoiner_slave_pipes(crtc_state)) intel_update_active_dpll(state, slave_crtc, encoder); } } @@ -3215,10 +3101,23 @@ intel_ddi_pre_pll_enable(struct intel_atomic_state *state, crtc_state->lane_lat_optim_mask); } +static void adlp_tbt_to_dp_alt_switch_wa(struct intel_encoder *encoder) +{ + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + enum tc_port tc_port = intel_port_to_tc(i915, encoder->port); + int ln; + + for (ln = 0; ln < 2; ln++) { + intel_de_write(i915, HIP_INDEX_REG(tc_port), HIP_INDEX_VAL(tc_port, ln)); + intel_de_rmw(i915, DKL_PCS_DW5(tc_port), DKL_PCS_DW5_CORE_SOFTRESET, 0); + } +} + static void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *encoder = &dig_port->base; struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = encoder->port; u32 dp_tp_ctl, ddi_buf_ctl; @@ -3254,6 +3153,10 @@ static void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp, intel_de_write(dev_priv, dp_tp_ctl_reg(encoder, crtc_state), dp_tp_ctl); intel_de_posting_read(dev_priv, dp_tp_ctl_reg(encoder, crtc_state)); + if (IS_ALDERLAKE_P(dev_priv) && + (intel_tc_port_in_dp_alt_mode(dig_port) || intel_tc_port_in_legacy_mode(dig_port))) + adlp_tbt_to_dp_alt_switch_wa(encoder); + intel_dp->DP |= DDI_BUF_CTL_ENABLE; intel_de_write(dev_priv, DDI_BUF_CTL(port), intel_dp->DP); intel_de_posting_read(dev_priv, DDI_BUF_CTL(port)); @@ -3471,7 +3374,11 @@ static void intel_ddi_read_func_ctl(struct intel_encoder *encoder, pipe_config->output_types |= BIT(INTEL_OUTPUT_DP); pipe_config->lane_count = ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1; - intel_dp_get_m_n(crtc, pipe_config); + + intel_cpu_transcoder_get_m1_n1(crtc, cpu_transcoder, + &pipe_config->dp_m_n); + intel_cpu_transcoder_get_m2_n2(crtc, cpu_transcoder, + &pipe_config->dp_m2_n2); if (DISPLAY_VER(dev_priv) >= 11) { i915_reg_t dp_tp_ctl = dp_tp_ctl_reg(encoder, pipe_config); @@ -3508,7 +3415,8 @@ static void intel_ddi_read_func_ctl(struct intel_encoder *encoder, pipe_config->mst_master_transcoder = REG_FIELD_GET(TRANS_DDI_MST_TRANSPORT_SELECT_MASK, temp); - intel_dp_get_m_n(crtc, pipe_config); + intel_cpu_transcoder_get_m1_n1(crtc, cpu_transcoder, + &pipe_config->dp_m_n); pipe_config->infoframes.enable |= intel_hdmi_infoframes_enabled(encoder, pipe_config); @@ -3797,8 +3705,8 @@ static bool m_n_equal(const struct intel_link_m_n *m_n_1, const struct intel_link_m_n *m_n_2) { return m_n_1->tu == m_n_2->tu && - m_n_1->gmch_m == m_n_2->gmch_m && - m_n_1->gmch_n == m_n_2->gmch_n && + m_n_1->data_m == m_n_2->data_m && + m_n_1->data_n == m_n_2->data_n && m_n_1->link_m == m_n_2->link_m && m_n_1->link_n == m_n_2->link_n; } diff --git a/drivers/gpu/drm/i915/display/intel_ddi.h b/drivers/gpu/drm/i915/display/intel_ddi.h index c2fea6562917..d39076facdce 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.h +++ b/drivers/gpu/drm/i915/display/intel_ddi.h @@ -6,7 +6,7 @@ #ifndef __INTEL_DDI_H__ #define __INTEL_DDI_H__ -#include "i915_reg.h" +#include "i915_reg_defs.h" struct drm_connector_state; struct drm_i915_private; diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c index e2dfb93a82bd..934a9f9e7dab 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c +++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c @@ -985,15 +985,15 @@ static const struct intel_ddi_buf_trans adlp_dkl_phy_trans_dp_hbr2_hbr3 = { }; static const union intel_ddi_buf_trans_entry _dg2_snps_trans[] = { - { .snps = { 26, 0, 0 } }, /* VS 0, pre-emph 0 */ - { .snps = { 33, 0, 6 } }, /* VS 0, pre-emph 1 */ - { .snps = { 38, 0, 12 } }, /* VS 0, pre-emph 2 */ - { .snps = { 43, 0, 19 } }, /* VS 0, pre-emph 3 */ - { .snps = { 39, 0, 0 } }, /* VS 1, pre-emph 0 */ - { .snps = { 44, 0, 8 } }, /* VS 1, pre-emph 1 */ - { .snps = { 47, 0, 15 } }, /* VS 1, pre-emph 2 */ - { .snps = { 52, 0, 0 } }, /* VS 2, pre-emph 0 */ - { .snps = { 51, 0, 10 } }, /* VS 2, pre-emph 1 */ + { .snps = { 25, 0, 0 } }, /* VS 0, pre-emph 0 */ + { .snps = { 32, 0, 6 } }, /* VS 0, pre-emph 1 */ + { .snps = { 35, 0, 10 } }, /* VS 0, pre-emph 2 */ + { .snps = { 43, 0, 17 } }, /* VS 0, pre-emph 3 */ + { .snps = { 35, 0, 0 } }, /* VS 1, pre-emph 0 */ + { .snps = { 45, 0, 8 } }, /* VS 1, pre-emph 1 */ + { .snps = { 48, 0, 14 } }, /* VS 1, pre-emph 2 */ + { .snps = { 47, 0, 0 } }, /* VS 2, pre-emph 0 */ + { .snps = { 55, 0, 7 } }, /* VS 2, pre-emph 1 */ { .snps = { 62, 0, 0 } }, /* VS 3, pre-emph 0 */ }; @@ -1005,21 +1005,21 @@ static const struct intel_ddi_buf_trans dg2_snps_trans = { static const union intel_ddi_buf_trans_entry _dg2_snps_trans_uhbr[] = { { .snps = { 62, 0, 0 } }, /* preset 0 */ - { .snps = { 56, 0, 6 } }, /* preset 1 */ - { .snps = { 51, 0, 11 } }, /* preset 2 */ - { .snps = { 48, 0, 14 } }, /* preset 3 */ - { .snps = { 43, 0, 19 } }, /* preset 4 */ + { .snps = { 55, 0, 7 } }, /* preset 1 */ + { .snps = { 50, 0, 12 } }, /* preset 2 */ + { .snps = { 44, 0, 18 } }, /* preset 3 */ + { .snps = { 35, 0, 21 } }, /* preset 4 */ { .snps = { 59, 3, 0 } }, /* preset 5 */ { .snps = { 53, 3, 6 } }, /* preset 6 */ - { .snps = { 49, 3, 10 } }, /* preset 7 */ - { .snps = { 45, 3, 14 } }, /* preset 8 */ - { .snps = { 42, 3, 17 } }, /* preset 9 */ + { .snps = { 48, 3, 11 } }, /* preset 7 */ + { .snps = { 42, 5, 15 } }, /* preset 8 */ + { .snps = { 37, 5, 20 } }, /* preset 9 */ { .snps = { 56, 6, 0 } }, /* preset 10 */ - { .snps = { 50, 6, 6 } }, /* preset 11 */ - { .snps = { 47, 6, 9 } }, /* preset 12 */ - { .snps = { 42, 6, 14 } }, /* preset 13 */ - { .snps = { 46, 8, 8 } }, /* preset 14 */ - { .snps = { 56, 3, 3 } }, /* preset 15 */ + { .snps = { 48, 7, 7 } }, /* preset 11 */ + { .snps = { 45, 7, 10 } }, /* preset 12 */ + { .snps = { 39, 8, 15 } }, /* preset 13 */ + { .snps = { 48, 14, 0 } }, /* preset 14 */ + { .snps = { 45, 4, 4 } }, /* preset 15 */ }; static const struct intel_ddi_buf_trans dg2_snps_trans_uhbr = { @@ -1321,7 +1321,7 @@ tgl_get_combo_buf_trans_dp(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); if (crtc_state->port_clock > 270000) { - if (IS_TGL_U(dev_priv) || IS_TGL_Y(dev_priv)) { + if (IS_TGL_UY(dev_priv)) { return intel_get_buf_trans(&tgl_uy_combo_phy_trans_dp_hbr2, n_entries); } else { diff --git a/drivers/gpu/drm/i915/display/intel_de.h b/drivers/gpu/drm/i915/display/intel_de.h index 9d8c177aa228..9c104f65e4c8 100644 --- a/drivers/gpu/drm/i915/display/intel_de.h +++ b/drivers/gpu/drm/i915/display/intel_de.h @@ -7,7 +7,6 @@ #define __INTEL_DE_H__ #include "i915_drv.h" -#include "i915_reg.h" #include "i915_trace.h" #include "intel_uncore.h" diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index bb4a85445fc6..7dfeb458aa65 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -38,7 +38,7 @@ #include <drm/drm_atomic_helper.h> #include <drm/drm_atomic_uapi.h> #include <drm/drm_damage_helper.h> -#include <drm/drm_dp_helper.h> +#include <drm/dp/drm_dp_helper.h> #include <drm/drm_edid.h> #include <drm/drm_fourcc.h> #include <drm/drm_plane_helper.h> @@ -74,6 +74,7 @@ #include "g4x_dp.h" #include "g4x_hdmi.h" +#include "hsw_ips.h" #include "i915_drv.h" #include "icl_dsi.h" #include "intel_acpi.h" @@ -112,15 +113,13 @@ #include "i9xx_plane.h" #include "skl_scaler.h" #include "skl_universal_plane.h" +#include "vlv_dsi.h" #include "vlv_dsi_pll.h" +#include "vlv_dsi_regs.h" #include "vlv_sideband.h" -#include "vlv_dsi.h" static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state); static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state); -static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, - const struct intel_link_m_n *m_n, - const struct intel_link_m_n *m2_n2); static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state); static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state); static void hsw_set_transconf(const struct intel_crtc_state *crtc_state); @@ -340,10 +339,41 @@ is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state) is_trans_port_sync_slave(crtc_state); } -static struct intel_crtc *intel_master_crtc(const struct intel_crtc_state *crtc_state) +static enum pipe bigjoiner_master_pipe(const struct intel_crtc_state *crtc_state) +{ + return ffs(crtc_state->bigjoiner_pipes) - 1; +} + +u8 intel_crtc_bigjoiner_slave_pipes(const struct intel_crtc_state *crtc_state) { - if (crtc_state->bigjoiner_slave) - return crtc_state->bigjoiner_linked_crtc; + if (crtc_state->bigjoiner_pipes) + return crtc_state->bigjoiner_pipes & ~BIT(bigjoiner_master_pipe(crtc_state)); + else + return 0; +} + +bool intel_crtc_is_bigjoiner_slave(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + return crtc_state->bigjoiner_pipes && + crtc->pipe != bigjoiner_master_pipe(crtc_state); +} + +bool intel_crtc_is_bigjoiner_master(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + return crtc_state->bigjoiner_pipes && + crtc->pipe == bigjoiner_master_pipe(crtc_state); +} + +struct intel_crtc *intel_master_crtc(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + + if (intel_crtc_is_bigjoiner_slave(crtc_state)) + return intel_crtc_for_pipe(i915, bigjoiner_master_pipe(crtc_state)); else return to_intel_crtc(crtc_state->uapi.crtc); } @@ -353,16 +383,10 @@ static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv, { i915_reg_t reg = PIPEDSL(pipe); u32 line1, line2; - u32 line_mask; - - if (DISPLAY_VER(dev_priv) == 2) - line_mask = DSL_LINEMASK_GEN2; - else - line_mask = DSL_LINEMASK_GEN3; - line1 = intel_de_read(dev_priv, reg) & line_mask; + line1 = intel_de_read(dev_priv, reg) & PIPEDSL_LINE_MASK; msleep(5); - line2 = intel_de_read(dev_priv, reg) & line_mask; + line2 = intel_de_read(dev_priv, reg) & PIPEDSL_LINE_MASK; return line1 != line2; } @@ -397,13 +421,11 @@ intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state) if (DISPLAY_VER(dev_priv) >= 4) { enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder; - i915_reg_t reg = PIPECONF(cpu_transcoder); /* Wait for the Pipe State to go off */ - if (intel_de_wait_for_clear(dev_priv, reg, - I965_PIPECONF_ACTIVE, 100)) - drm_WARN(&dev_priv->drm, 1, - "pipe_off wait timed out\n"); + if (intel_de_wait_for_clear(dev_priv, PIPECONF(cpu_transcoder), + PIPECONF_STATE_ENABLE, 100)) + drm_WARN(&dev_priv->drm, 1, "pipe_off wait timed out\n"); } else { intel_wait_for_pipe_scanline_stopped(crtc); } @@ -763,8 +785,11 @@ void intel_plane_disable_noatomic(struct intel_crtc *crtc, crtc_state->data_rate[plane->id] = 0; crtc_state->min_cdclk[plane->id] = 0; - if (plane->id == PLANE_PRIMARY) - hsw_disable_ips(crtc_state); + if ((crtc_state->active_planes & ~BIT(PLANE_CURSOR)) == 0 && + hsw_ips_disable(crtc_state)) { + crtc_state->ips_enabled = false; + intel_crtc_wait_for_next_vblank(crtc); + } /* * Vblank time updates from the shadow to live plane control register @@ -1101,72 +1126,6 @@ static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state) intel_de_write(dev_priv, PF_WIN_SZ(pipe), width << 16 | height); } -void hsw_enable_ips(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - - if (!crtc_state->ips_enabled) - return; - - /* - * We can only enable IPS after we enable a plane and wait for a vblank - * This function is called from post_plane_update, which is run after - * a vblank wait. - */ - drm_WARN_ON(dev, !(crtc_state->active_planes & ~BIT(PLANE_CURSOR))); - - if (IS_BROADWELL(dev_priv)) { - drm_WARN_ON(dev, sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, - IPS_ENABLE | IPS_PCODE_CONTROL)); - /* Quoting Art Runyan: "its not safe to expect any particular - * value in IPS_CTL bit 31 after enabling IPS through the - * mailbox." Moreover, the mailbox may return a bogus state, - * so we need to just enable it and continue on. - */ - } else { - intel_de_write(dev_priv, IPS_CTL, IPS_ENABLE); - /* The bit only becomes 1 in the next vblank, so this wait here - * is essentially intel_wait_for_vblank. If we don't have this - * and don't wait for vblanks until the end of crtc_enable, then - * the HW state readout code will complain that the expected - * IPS_CTL value is not the one we read. */ - if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50)) - drm_err(&dev_priv->drm, - "Timed out waiting for IPS enable\n"); - } -} - -void hsw_disable_ips(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - - if (!crtc_state->ips_enabled) - return; - - if (IS_BROADWELL(dev_priv)) { - drm_WARN_ON(dev, - sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0)); - /* - * Wait for PCODE to finish disabling IPS. The BSpec specified - * 42ms timeout value leads to occasional timeouts so use 100ms - * instead. - */ - if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100)) - drm_err(&dev_priv->drm, - "Timed out waiting for IPS disable\n"); - } else { - intel_de_write(dev_priv, IPS_CTL, 0); - intel_de_posting_read(dev_priv, IPS_CTL); - } - - /* We need to wait for a vblank before we can disable the plane. */ - intel_crtc_wait_for_next_vblank(crtc); -} - static void intel_crtc_dpms_overlay_disable(struct intel_crtc *crtc) { if (crtc->overlay) @@ -1177,67 +1136,6 @@ static void intel_crtc_dpms_overlay_disable(struct intel_crtc *crtc) */ } -static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state, - const struct intel_crtc_state *new_crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - - if (!old_crtc_state->ips_enabled) - return false; - - if (intel_crtc_needs_modeset(new_crtc_state)) - return true; - - /* - * Workaround : Do not read or write the pipe palette/gamma data while - * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. - * - * Disable IPS before we program the LUT. - */ - if (IS_HASWELL(dev_priv) && - (new_crtc_state->uapi.color_mgmt_changed || - new_crtc_state->update_pipe) && - new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) - return true; - - return !new_crtc_state->ips_enabled; -} - -static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state, - const struct intel_crtc_state *new_crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - - if (!new_crtc_state->ips_enabled) - return false; - - if (intel_crtc_needs_modeset(new_crtc_state)) - return true; - - /* - * Workaround : Do not read or write the pipe palette/gamma data while - * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled. - * - * Re-enable IPS after the LUT has been programmed. - */ - if (IS_HASWELL(dev_priv) && - (new_crtc_state->uapi.color_mgmt_changed || - new_crtc_state->update_pipe) && - new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) - return true; - - /* - * We can't read out IPS on broadwell, assume the worst and - * forcibly enable IPS on the first fastset. - */ - if (new_crtc_state->update_pipe && old_crtc_state->inherited) - return true; - - return !old_crtc_state->ips_enabled; -} - static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); @@ -1332,9 +1230,7 @@ static void intel_post_plane_update(struct intel_atomic_state *state, if (new_crtc_state->update_wm_post && new_crtc_state->hw.active) intel_update_watermarks(dev_priv); - if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state)) - hsw_enable_ips(new_crtc_state); - + hsw_ips_post_update(state, crtc); intel_fbc_post_update(state, crtc); intel_drrs_page_flip(state, crtc); @@ -1367,8 +1263,7 @@ static void intel_crtc_enable_flip_done(struct intel_atomic_state *state, int i; for_each_new_intel_plane_in_state(state, plane, plane_state, i) { - if (plane->enable_flip_done && - plane->pipe == crtc->pipe && + if (plane->pipe == crtc->pipe && update_planes & BIT(plane->id)) plane->enable_flip_done(plane); } @@ -1385,8 +1280,7 @@ static void intel_crtc_disable_flip_done(struct intel_atomic_state *state, int i; for_each_new_intel_plane_in_state(state, plane, plane_state, i) { - if (plane->disable_flip_done && - plane->pipe == crtc->pipe && + if (plane->pipe == crtc->pipe && update_planes & BIT(plane->id)) plane->disable_flip_done(plane); } @@ -1435,8 +1329,8 @@ static void intel_pre_plane_update(struct intel_atomic_state *state, intel_psr_pre_plane_update(state, crtc); - if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state)) - hsw_disable_ips(old_crtc_state); + if (hsw_ips_pre_update(state, crtc)) + intel_crtc_wait_for_next_vblank(crtc); if (intel_fbc_pre_update(state, crtc)) intel_crtc_wait_for_next_vblank(crtc); @@ -1817,6 +1711,26 @@ static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_stat plane->disable_arm(plane, crtc_state); } +static void ilk_configure_cpu_transcoder(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + + if (crtc_state->has_pch_encoder) { + intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder, + &crtc_state->fdi_m_n); + } else if (intel_crtc_has_dp_encoder(crtc_state)) { + intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder, + &crtc_state->dp_m_n); + intel_cpu_transcoder_set_m2_n2(crtc, cpu_transcoder, + &crtc_state->dp_m2_n2); + } + + intel_set_transcoder_timings(crtc_state); + + ilk_set_pipeconf(crtc_state); +} + static void ilk_crtc_enable(struct intel_atomic_state *state, struct intel_crtc *crtc) { @@ -1841,27 +1755,16 @@ static void ilk_crtc_enable(struct intel_atomic_state *state, intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false); - if (intel_crtc_has_dp_encoder(new_crtc_state)) - intel_dp_set_m_n(new_crtc_state, M1_N1); + ilk_configure_cpu_transcoder(new_crtc_state); - intel_set_transcoder_timings(new_crtc_state); intel_set_pipe_src_size(new_crtc_state); - if (new_crtc_state->has_pch_encoder) - intel_cpu_transcoder_set_m_n(new_crtc_state, - &new_crtc_state->fdi_m_n, NULL); - - ilk_set_pipeconf(new_crtc_state); - crtc->active = true; intel_encoders_pre_enable(state, crtc); if (new_crtc_state->has_pch_encoder) { - /* Note: FDI PLL enabling _must_ be done before we enable the - * cpu pipes, hence this is separate from all the other fdi/pch - * enabling. */ - ilk_fdi_pll_enable(new_crtc_state); + ilk_pch_pre_enable(state, crtc); } else { assert_fdi_tx_disabled(dev_priv, pipe); assert_fdi_rx_disabled(dev_priv, pipe); @@ -1905,12 +1808,6 @@ static void ilk_crtc_enable(struct intel_atomic_state *state, intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); } -/* IPS only exists on ULT machines and is tied to pipe A. */ -static bool hsw_crtc_supports_ips(struct intel_crtc *crtc) -{ - return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A; -} - static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv, enum pipe pipe, bool apply) { @@ -1974,42 +1871,46 @@ static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state) static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state, const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_crtc_state *master_crtc_state; - struct intel_crtc *master_crtc; - struct drm_connector_state *conn_state; - struct drm_connector *conn; - struct intel_encoder *encoder = NULL; - int i; - - master_crtc = intel_master_crtc(crtc_state); - master_crtc_state = intel_atomic_get_new_crtc_state(state, master_crtc); - - for_each_new_connector_in_state(&state->base, conn, conn_state, i) { - if (conn_state->crtc != &master_crtc->base) - continue; - - encoder = to_intel_encoder(conn_state->best_encoder); - break; - } + struct intel_crtc *master_crtc = intel_master_crtc(crtc_state); /* * Enable sequence steps 1-7 on bigjoiner master */ - if (crtc_state->bigjoiner_slave) + if (intel_crtc_is_bigjoiner_slave(crtc_state)) intel_encoders_pre_pll_enable(state, master_crtc); if (crtc_state->shared_dpll) intel_enable_shared_dpll(crtc_state); - if (crtc_state->bigjoiner_slave) + if (intel_crtc_is_bigjoiner_slave(crtc_state)) intel_encoders_pre_enable(state, master_crtc); +} + +static void hsw_configure_cpu_transcoder(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - /* need to enable VDSC, which we skipped in pre-enable */ - intel_dsc_enable(crtc_state); + if (crtc_state->has_pch_encoder) { + intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder, + &crtc_state->fdi_m_n); + } else if (intel_crtc_has_dp_encoder(crtc_state)) { + intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder, + &crtc_state->dp_m_n); + intel_cpu_transcoder_set_m2_n2(crtc, cpu_transcoder, + &crtc_state->dp_m2_n2); + } - if (DISPLAY_VER(dev_priv) >= 13) - intel_uncompressed_joiner_enable(crtc_state); + intel_set_transcoder_timings(crtc_state); + + if (cpu_transcoder != TRANSCODER_EDP) + intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder), + crtc_state->pixel_multiplier - 1); + + hsw_set_frame_start_delay(crtc_state); + + hsw_set_transconf(crtc_state); } static void hsw_crtc_enable(struct intel_atomic_state *state, @@ -2036,25 +1937,18 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, icl_ddi_bigjoiner_pre_enable(state, new_crtc_state); } + intel_dsc_enable(new_crtc_state); + + if (DISPLAY_VER(dev_priv) >= 13) + intel_uncompressed_joiner_enable(new_crtc_state); + intel_set_pipe_src_size(new_crtc_state); if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) bdw_set_pipemisc(new_crtc_state); - if (!new_crtc_state->bigjoiner_slave && !transcoder_is_dsi(cpu_transcoder)) { - intel_set_transcoder_timings(new_crtc_state); - - if (cpu_transcoder != TRANSCODER_EDP) - intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder), - new_crtc_state->pixel_multiplier - 1); - - if (new_crtc_state->has_pch_encoder) - intel_cpu_transcoder_set_m_n(new_crtc_state, - &new_crtc_state->fdi_m_n, NULL); - - hsw_set_frame_start_delay(new_crtc_state); - - hsw_set_transconf(new_crtc_state); - } + if (!intel_crtc_is_bigjoiner_slave(new_crtc_state) && + !transcoder_is_dsi(cpu_transcoder)) + hsw_configure_cpu_transcoder(new_crtc_state); crtc->active = true; @@ -2093,7 +1987,7 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, icl_pipe_mbus_enable(crtc, dbuf_state->joined_mbus); } - if (new_crtc_state->bigjoiner_slave) + if (intel_crtc_is_bigjoiner_slave(new_crtc_state)) intel_crtc_vblank_on(new_crtc_state); intel_encoders_enable(state, crtc); @@ -2178,7 +2072,7 @@ static void hsw_crtc_disable(struct intel_atomic_state *state, * FIXME collapse everything to one hook. * Need care with mst->ddi interactions. */ - if (!old_crtc_state->bigjoiner_slave) { + if (!intel_crtc_is_bigjoiner_slave(old_crtc_state)) { intel_encoders_disable(state, crtc); intel_encoders_post_disable(state, crtc); } @@ -2441,6 +2335,23 @@ static void modeset_put_crtc_power_domains(struct intel_crtc *crtc, domains); } +static void i9xx_configure_cpu_transcoder(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + + if (intel_crtc_has_dp_encoder(crtc_state)) { + intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder, + &crtc_state->dp_m_n); + intel_cpu_transcoder_set_m2_n2(crtc, cpu_transcoder, + &crtc_state->dp_m2_n2); + } + + intel_set_transcoder_timings(crtc_state); + + i9xx_set_pipeconf(crtc_state); +} + static void valleyview_crtc_enable(struct intel_atomic_state *state, struct intel_crtc *crtc) { @@ -2452,10 +2363,8 @@ static void valleyview_crtc_enable(struct intel_atomic_state *state, if (drm_WARN_ON(&dev_priv->drm, crtc->active)) return; - if (intel_crtc_has_dp_encoder(new_crtc_state)) - intel_dp_set_m_n(new_crtc_state, M1_N1); + i9xx_configure_cpu_transcoder(new_crtc_state); - intel_set_transcoder_timings(new_crtc_state); intel_set_pipe_src_size(new_crtc_state); if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) { @@ -2463,8 +2372,6 @@ static void valleyview_crtc_enable(struct intel_atomic_state *state, intel_de_write(dev_priv, CHV_CANVAS(pipe), 0); } - i9xx_set_pipeconf(new_crtc_state); - crtc->active = true; intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); @@ -2504,14 +2411,10 @@ static void i9xx_crtc_enable(struct intel_atomic_state *state, if (drm_WARN_ON(&dev_priv->drm, crtc->active)) return; - if (intel_crtc_has_dp_encoder(new_crtc_state)) - intel_dp_set_m_n(new_crtc_state, M1_N1); + i9xx_configure_cpu_transcoder(new_crtc_state); - intel_set_transcoder_timings(new_crtc_state); intel_set_pipe_src_size(new_crtc_state); - i9xx_set_pipeconf(new_crtc_state); - crtc->active = true; if (DISPLAY_VER(dev_priv) != 2) @@ -2757,77 +2660,6 @@ static void intel_connector_verify_state(struct intel_crtc_state *crtc_state, } } -bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - - /* IPS only exists on ULT machines and is tied to pipe A. */ - if (!hsw_crtc_supports_ips(crtc)) - return false; - - if (!dev_priv->params.enable_ips) - return false; - - if (crtc_state->pipe_bpp > 24) - return false; - - /* - * We compare against max which means we must take - * the increased cdclk requirement into account when - * calculating the new cdclk. - * - * Should measure whether using a lower cdclk w/o IPS - */ - if (IS_BROADWELL(dev_priv) && - crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100) - return false; - - return true; -} - -static int hsw_compute_ips_config(struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = - to_i915(crtc_state->uapi.crtc->dev); - struct intel_atomic_state *state = - to_intel_atomic_state(crtc_state->uapi.state); - - crtc_state->ips_enabled = false; - - if (!hsw_crtc_state_ips_capable(crtc_state)) - return 0; - - /* - * When IPS gets enabled, the pipe CRC changes. Since IPS gets - * enabled and disabled dynamically based on package C states, - * user space can't make reliable use of the CRCs, so let's just - * completely disable it. - */ - if (crtc_state->crc_enabled) - return 0; - - /* IPS should be fine as long as at least one plane is enabled. */ - if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR))) - return 0; - - if (IS_BROADWELL(dev_priv)) { - const struct intel_cdclk_state *cdclk_state; - - cdclk_state = intel_atomic_get_cdclk_state(state); - if (IS_ERR(cdclk_state)) - return PTR_ERR(cdclk_state); - - /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ - if (crtc_state->pixel_rate > cdclk_state->logical.cdclk * 95 / 100) - return 0; - } - - crtc_state->ips_enabled = true; - - return 0; -} - static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc) { const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -3085,7 +2917,7 @@ intel_link_compute_m_n(u16 bits_per_pixel, int nlanes, m_n->tu = 64; compute_m_n(data_clock, link_clock * nlanes * 8, - &m_n->gmch_m, &m_n->gmch_n, + &m_n->data_m, &m_n->data_n, constant_n); compute_m_n(pixel_clock, link_clock, @@ -3116,99 +2948,66 @@ static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv) } } -static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, - const struct intel_link_m_n *m_n) +void intel_zero_m_n(struct intel_link_m_n *m_n) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; + /* corresponds to 0 register value */ + memset(m_n, 0, sizeof(*m_n)); + m_n->tu = 1; +} - intel_de_write(dev_priv, PCH_TRANS_DATA_M1(pipe), - TU_SIZE(m_n->tu) | m_n->gmch_m); - intel_de_write(dev_priv, PCH_TRANS_DATA_N1(pipe), m_n->gmch_n); - intel_de_write(dev_priv, PCH_TRANS_LINK_M1(pipe), m_n->link_m); - intel_de_write(dev_priv, PCH_TRANS_LINK_N1(pipe), m_n->link_n); +void intel_set_m_n(struct drm_i915_private *i915, + const struct intel_link_m_n *m_n, + i915_reg_t data_m_reg, i915_reg_t data_n_reg, + i915_reg_t link_m_reg, i915_reg_t link_n_reg) +{ + intel_de_write(i915, data_m_reg, TU_SIZE(m_n->tu) | m_n->data_m); + intel_de_write(i915, data_n_reg, m_n->data_n); + intel_de_write(i915, link_m_reg, m_n->link_m); + /* + * On BDW+ writing LINK_N arms the double buffered update + * of all the M/N registers, so it must be written last. + */ + intel_de_write(i915, link_n_reg, m_n->link_n); } -static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv, - enum transcoder transcoder) +bool intel_cpu_transcoder_has_m2_n2(struct drm_i915_private *dev_priv, + enum transcoder transcoder) { if (IS_HASWELL(dev_priv)) return transcoder == TRANSCODER_EDP; - /* - * Strictly speaking some registers are available before - * gen7, but we only support DRRS on gen7+ - */ - return DISPLAY_VER(dev_priv) == 7 || IS_CHERRYVIEW(dev_priv); + return IS_DISPLAY_VER(dev_priv, 5, 7) || IS_CHERRYVIEW(dev_priv); } -static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state, - const struct intel_link_m_n *m_n, - const struct intel_link_m_n *m2_n2) +void intel_cpu_transcoder_set_m1_n1(struct intel_crtc *crtc, + enum transcoder transcoder, + const struct intel_link_m_n *m_n) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - enum transcoder transcoder = crtc_state->cpu_transcoder; - if (DISPLAY_VER(dev_priv) >= 5) { - intel_de_write(dev_priv, PIPE_DATA_M1(transcoder), - TU_SIZE(m_n->tu) | m_n->gmch_m); - intel_de_write(dev_priv, PIPE_DATA_N1(transcoder), - m_n->gmch_n); - intel_de_write(dev_priv, PIPE_LINK_M1(transcoder), - m_n->link_m); - intel_de_write(dev_priv, PIPE_LINK_N1(transcoder), - m_n->link_n); - /* - * M2_N2 registers are set only if DRRS is supported - * (to make sure the registers are not unnecessarily accessed). - */ - if (m2_n2 && crtc_state->has_drrs && - transcoder_has_m2_n2(dev_priv, transcoder)) { - intel_de_write(dev_priv, PIPE_DATA_M2(transcoder), - TU_SIZE(m2_n2->tu) | m2_n2->gmch_m); - intel_de_write(dev_priv, PIPE_DATA_N2(transcoder), - m2_n2->gmch_n); - intel_de_write(dev_priv, PIPE_LINK_M2(transcoder), - m2_n2->link_m); - intel_de_write(dev_priv, PIPE_LINK_N2(transcoder), - m2_n2->link_n); - } - } else { - intel_de_write(dev_priv, PIPE_DATA_M_G4X(pipe), - TU_SIZE(m_n->tu) | m_n->gmch_m); - intel_de_write(dev_priv, PIPE_DATA_N_G4X(pipe), m_n->gmch_n); - intel_de_write(dev_priv, PIPE_LINK_M_G4X(pipe), m_n->link_m); - intel_de_write(dev_priv, PIPE_LINK_N_G4X(pipe), m_n->link_n); - } + if (DISPLAY_VER(dev_priv) >= 5) + intel_set_m_n(dev_priv, m_n, + PIPE_DATA_M1(transcoder), PIPE_DATA_N1(transcoder), + PIPE_LINK_M1(transcoder), PIPE_LINK_N1(transcoder)); + else + intel_set_m_n(dev_priv, m_n, + PIPE_DATA_M_G4X(pipe), PIPE_DATA_N_G4X(pipe), + PIPE_LINK_M_G4X(pipe), PIPE_LINK_N_G4X(pipe)); } -void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n) +void intel_cpu_transcoder_set_m2_n2(struct intel_crtc *crtc, + enum transcoder transcoder, + const struct intel_link_m_n *m_n) { - const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL; - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - - if (m_n == M1_N1) { - dp_m_n = &crtc_state->dp_m_n; - dp_m2_n2 = &crtc_state->dp_m2_n2; - } else if (m_n == M2_N2) { + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - /* - * M2_N2 registers are not supported. Hence m2_n2 divider value - * needs to be programmed into M1_N1. - */ - dp_m_n = &crtc_state->dp_m2_n2; - } else { - drm_err(&i915->drm, "Unsupported divider value\n"); + if (!intel_cpu_transcoder_has_m2_n2(dev_priv, transcoder)) return; - } - if (crtc_state->has_pch_encoder) - intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n); - else - intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2); + intel_set_m_n(dev_priv, m_n, + PIPE_DATA_M2(transcoder), PIPE_DATA_N2(transcoder), + PIPE_LINK_M2(transcoder), PIPE_LINK_N2(transcoder)); } static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state) @@ -3279,7 +3078,8 @@ static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state) * always be the user's requested size. */ intel_de_write(dev_priv, PIPESRC(pipe), - ((crtc_state->pipe_src_w - 1) << 16) | (crtc_state->pipe_src_h - 1)); + PIPESRC_WIDTH(crtc_state->pipe_src_w - 1) | + PIPESRC_HEIGHT(crtc_state->pipe_src_h - 1)); } static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state) @@ -3350,21 +3150,19 @@ static void intel_get_pipe_src_size(struct intel_crtc *crtc, u32 tmp; tmp = intel_de_read(dev_priv, PIPESRC(crtc->pipe)); - pipe_config->pipe_src_h = (tmp & 0xffff) + 1; - pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1; + pipe_config->pipe_src_w = REG_FIELD_GET(PIPESRC_WIDTH_MASK, tmp) + 1; + pipe_config->pipe_src_h = REG_FIELD_GET(PIPESRC_HEIGHT_MASK, tmp) + 1; } static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 pipeconf; - - pipeconf = 0; + u32 pipeconf = 0; /* we keep both pipes enabled on 830 */ if (IS_I830(dev_priv)) - pipeconf |= intel_de_read(dev_priv, PIPECONF(crtc->pipe)) & PIPECONF_ENABLE; + pipeconf |= PIPECONF_ENABLE; if (crtc_state->double_wide) pipeconf |= PIPECONF_DOUBLE_WIDE; @@ -3379,13 +3177,13 @@ static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state) switch (crtc_state->pipe_bpp) { case 18: - pipeconf |= PIPECONF_6BPC; + pipeconf |= PIPECONF_BPC_6; break; case 24: - pipeconf |= PIPECONF_8BPC; + pipeconf |= PIPECONF_BPC_8; break; case 30: - pipeconf |= PIPECONF_10BPC; + pipeconf |= PIPECONF_BPC_10; break; default: /* Case prevented by intel_choose_pipe_bpp_dither. */ @@ -3400,7 +3198,7 @@ static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state) else pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT; } else { - pipeconf |= PIPECONF_PROGRESSIVE; + pipeconf |= PIPECONF_INTERLACE_PROGRESSIVE; } if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && @@ -3543,11 +3341,11 @@ static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state) tmp = intel_de_read(dev_priv, DSPCNTR(i9xx_plane)); - if (tmp & DISPPLANE_GAMMA_ENABLE) + if (tmp & DISP_PIPE_GAMMA_ENABLE) crtc_state->gamma_enable = true; if (!HAS_GMCH(dev_priv) && - tmp & DISPPLANE_PIPE_CSC_ENABLE) + tmp & DISP_PIPE_CSC_ENABLE) crtc_state->csc_enable = true; } @@ -3578,16 +3376,17 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { switch (tmp & PIPECONF_BPC_MASK) { - case PIPECONF_6BPC: + case PIPECONF_BPC_6: pipe_config->pipe_bpp = 18; break; - case PIPECONF_8BPC: + case PIPECONF_BPC_8: pipe_config->pipe_bpp = 24; break; - case PIPECONF_10BPC: + case PIPECONF_BPC_10: pipe_config->pipe_bpp = 30; break; default: + MISSING_CASE(tmp); break; } } @@ -3596,8 +3395,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, (tmp & PIPECONF_COLOR_RANGE_SELECT)) pipe_config->limited_color_range = true; - pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >> - PIPECONF_GAMMA_MODE_SHIFT; + pipe_config->gamma_mode = REG_FIELD_GET(PIPECONF_GAMMA_MODE_MASK_I9XX, tmp); if (IS_CHERRYVIEW(dev_priv)) pipe_config->cgm_mode = intel_de_read(dev_priv, @@ -3684,16 +3482,16 @@ static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state) switch (crtc_state->pipe_bpp) { case 18: - val |= PIPECONF_6BPC; + val |= PIPECONF_BPC_6; break; case 24: - val |= PIPECONF_8BPC; + val |= PIPECONF_BPC_8; break; case 30: - val |= PIPECONF_10BPC; + val |= PIPECONF_BPC_10; break; case 36: - val |= PIPECONF_12BPC; + val |= PIPECONF_BPC_12; break; default: /* Case prevented by intel_choose_pipe_bpp_dither. */ @@ -3701,12 +3499,12 @@ static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state) } if (crtc_state->dither) - val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP); + val |= PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP; if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) - val |= PIPECONF_INTERLACED_ILK; + val |= PIPECONF_INTERLACE_IF_ID_ILK; else - val |= PIPECONF_PROGRESSIVE; + val |= PIPECONF_INTERLACE_PF_PD_ILK; /* * This would end up with an odd purple hue over @@ -3738,12 +3536,12 @@ static void hsw_set_transconf(const struct intel_crtc_state *crtc_state) u32 val = 0; if (IS_HASWELL(dev_priv) && crtc_state->dither) - val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP); + val |= PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP; if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) - val |= PIPECONF_INTERLACED_ILK; + val |= PIPECONF_INTERLACE_IF_ID_ILK; else - val |= PIPECONF_PROGRESSIVE; + val |= PIPECONF_INTERLACE_PF_PD_ILK; if (IS_HASWELL(dev_priv) && crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB) @@ -3765,18 +3563,18 @@ static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state) switch (crtc_state->pipe_bpp) { case 18: - val |= PIPEMISC_6_BPC; + val |= PIPEMISC_BPC_6; break; case 24: - val |= PIPEMISC_8_BPC; + val |= PIPEMISC_BPC_8; break; case 30: - val |= PIPEMISC_10_BPC; + val |= PIPEMISC_BPC_10; break; case 36: /* Port output 12BPC defined for ADLP+ */ if (DISPLAY_VER(dev_priv) > 12) - val |= PIPEMISC_12_BPC_ADLP; + val |= PIPEMISC_BPC_12_ADLP; break; default: MISSING_CASE(crtc_state->pipe_bpp); @@ -3812,7 +3610,7 @@ static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state) } intel_de_rmw(dev_priv, PIPE_MISC2(crtc->pipe), - PIPE_MISC2_UNDERRUN_BUBBLE_COUNTER_MASK, + PIPE_MISC2_BUBBLE_COUNTER_MASK, scaler_in_use ? PIPE_MISC2_BUBBLE_COUNTER_SCALER_EN : PIPE_MISC2_BUBBLE_COUNTER_SCALER_DIS); } @@ -3828,11 +3626,11 @@ int bdw_get_pipemisc_bpp(struct intel_crtc *crtc) tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe)); switch (tmp & PIPEMISC_BPC_MASK) { - case PIPEMISC_6_BPC: + case PIPEMISC_BPC_6: return 18; - case PIPEMISC_8_BPC: + case PIPEMISC_BPC_8: return 24; - case PIPEMISC_10_BPC: + case PIPEMISC_BPC_10: return 30; /* * PORT OUTPUT 12 BPC defined for ADLP+. @@ -3844,7 +3642,7 @@ int bdw_get_pipemisc_bpp(struct intel_crtc *crtc) * on older platforms, need to find a workaround for 12 BPC * MIPI DSI HW readout. */ - case PIPEMISC_12_BPC_ADLP: + case PIPEMISC_BPC_12_ADLP: if (DISPLAY_VER(dev_priv) > 12) return 36; fallthrough; @@ -3865,83 +3663,47 @@ int ilk_get_lanes_required(int target_clock, int link_bw, int bpp) return DIV_ROUND_UP(bps, link_bw * 8); } -static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc, - struct intel_link_m_n *m_n) +void intel_get_m_n(struct drm_i915_private *i915, + struct intel_link_m_n *m_n, + i915_reg_t data_m_reg, i915_reg_t data_n_reg, + i915_reg_t link_m_reg, i915_reg_t link_n_reg) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - enum pipe pipe = crtc->pipe; - - m_n->link_m = intel_de_read(dev_priv, PCH_TRANS_LINK_M1(pipe)); - m_n->link_n = intel_de_read(dev_priv, PCH_TRANS_LINK_N1(pipe)); - m_n->gmch_m = intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe)) - & ~TU_SIZE_MASK; - m_n->gmch_n = intel_de_read(dev_priv, PCH_TRANS_DATA_N1(pipe)); - m_n->tu = ((intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe)) - & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; + m_n->link_m = intel_de_read(i915, link_m_reg) & DATA_LINK_M_N_MASK; + m_n->link_n = intel_de_read(i915, link_n_reg) & DATA_LINK_M_N_MASK; + m_n->data_m = intel_de_read(i915, data_m_reg) & DATA_LINK_M_N_MASK; + m_n->data_n = intel_de_read(i915, data_n_reg) & DATA_LINK_M_N_MASK; + m_n->tu = REG_FIELD_GET(TU_SIZE_MASK, intel_de_read(i915, data_m_reg)) + 1; } -static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc, - enum transcoder transcoder, - struct intel_link_m_n *m_n, - struct intel_link_m_n *m2_n2) +void intel_cpu_transcoder_get_m1_n1(struct intel_crtc *crtc, + enum transcoder transcoder, + struct intel_link_m_n *m_n) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - if (DISPLAY_VER(dev_priv) >= 5) { - m_n->link_m = intel_de_read(dev_priv, - PIPE_LINK_M1(transcoder)); - m_n->link_n = intel_de_read(dev_priv, - PIPE_LINK_N1(transcoder)); - m_n->gmch_m = intel_de_read(dev_priv, - PIPE_DATA_M1(transcoder)) - & ~TU_SIZE_MASK; - m_n->gmch_n = intel_de_read(dev_priv, - PIPE_DATA_N1(transcoder)); - m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M1(transcoder)) - & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; - - if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) { - m2_n2->link_m = intel_de_read(dev_priv, - PIPE_LINK_M2(transcoder)); - m2_n2->link_n = intel_de_read(dev_priv, - PIPE_LINK_N2(transcoder)); - m2_n2->gmch_m = intel_de_read(dev_priv, - PIPE_DATA_M2(transcoder)) - & ~TU_SIZE_MASK; - m2_n2->gmch_n = intel_de_read(dev_priv, - PIPE_DATA_N2(transcoder)); - m2_n2->tu = ((intel_de_read(dev_priv, PIPE_DATA_M2(transcoder)) - & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; - } - } else { - m_n->link_m = intel_de_read(dev_priv, PIPE_LINK_M_G4X(pipe)); - m_n->link_n = intel_de_read(dev_priv, PIPE_LINK_N_G4X(pipe)); - m_n->gmch_m = intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe)) - & ~TU_SIZE_MASK; - m_n->gmch_n = intel_de_read(dev_priv, PIPE_DATA_N_G4X(pipe)); - m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe)) - & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1; - } -} - -void intel_dp_get_m_n(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) -{ - if (pipe_config->has_pch_encoder) - intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n); + if (DISPLAY_VER(dev_priv) >= 5) + intel_get_m_n(dev_priv, m_n, + PIPE_DATA_M1(transcoder), PIPE_DATA_N1(transcoder), + PIPE_LINK_M1(transcoder), PIPE_LINK_N1(transcoder)); else - intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder, - &pipe_config->dp_m_n, - &pipe_config->dp_m2_n2); + intel_get_m_n(dev_priv, m_n, + PIPE_DATA_M_G4X(pipe), PIPE_DATA_N_G4X(pipe), + PIPE_LINK_M_G4X(pipe), PIPE_LINK_N_G4X(pipe)); } -void ilk_get_fdi_m_n_config(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) +void intel_cpu_transcoder_get_m2_n2(struct intel_crtc *crtc, + enum transcoder transcoder, + struct intel_link_m_n *m_n) { - intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder, - &pipe_config->fdi_m_n, NULL); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + + if (!intel_cpu_transcoder_has_m2_n2(dev_priv, transcoder)) + return; + + intel_get_m_n(dev_priv, m_n, + PIPE_DATA_M2(transcoder), PIPE_DATA_N2(transcoder), + PIPE_LINK_M2(transcoder), PIPE_LINK_N2(transcoder)); } static void ilk_get_pfit_pos_size(struct intel_crtc_state *crtc_state, @@ -4037,16 +3799,16 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc, goto out; switch (tmp & PIPECONF_BPC_MASK) { - case PIPECONF_6BPC: + case PIPECONF_BPC_6: pipe_config->pipe_bpp = 18; break; - case PIPECONF_8BPC: + case PIPECONF_BPC_8: pipe_config->pipe_bpp = 24; break; - case PIPECONF_10BPC: + case PIPECONF_BPC_10: pipe_config->pipe_bpp = 30; break; - case PIPECONF_12BPC: + case PIPECONF_BPC_12: pipe_config->pipe_bpp = 36; break; default: @@ -4066,8 +3828,7 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc, break; } - pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >> - PIPECONF_GAMMA_MODE_SHIFT; + pipe_config->gamma_mode = REG_FIELD_GET(PIPECONF_GAMMA_MODE_MASK_ILK, tmp); pipe_config->csc_mode = intel_de_read(dev_priv, PIPE_CSC_MODE(crtc->pipe)); @@ -4117,19 +3878,20 @@ static bool transcoder_ddi_func_is_enabled(struct drm_i915_private *dev_priv, return tmp & TRANS_DDI_FUNC_ENABLE; } -static u8 enabled_bigjoiner_pipes(struct drm_i915_private *dev_priv) +static void enabled_bigjoiner_pipes(struct drm_i915_private *dev_priv, + u8 *master_pipes, u8 *slave_pipes) { - u8 master_pipes = 0, slave_pipes = 0; struct intel_crtc *crtc; - for_each_intel_crtc(&dev_priv->drm, crtc) { + *master_pipes = 0; + *slave_pipes = 0; + + for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, crtc, + bigjoiner_pipes(dev_priv)) { enum intel_display_power_domain power_domain; enum pipe pipe = crtc->pipe; intel_wakeref_t wakeref; - if ((bigjoiner_pipes(dev_priv) & BIT(pipe)) == 0) - continue; - power_domain = intel_dsc_power_domain(crtc, (enum transcoder) pipe); with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref) { u32 tmp = intel_de_read(dev_priv, ICL_PIPE_DSS_CTL1(pipe)); @@ -4138,9 +3900,9 @@ static u8 enabled_bigjoiner_pipes(struct drm_i915_private *dev_priv) continue; if (tmp & MASTER_BIG_JOINER_ENABLE) - master_pipes |= BIT(pipe); + *master_pipes |= BIT(pipe); else - slave_pipes |= BIT(pipe); + *slave_pipes |= BIT(pipe); } if (DISPLAY_VER(dev_priv) < 13) @@ -4151,18 +3913,47 @@ static u8 enabled_bigjoiner_pipes(struct drm_i915_private *dev_priv) u32 tmp = intel_de_read(dev_priv, ICL_PIPE_DSS_CTL1(pipe)); if (tmp & UNCOMPRESSED_JOINER_MASTER) - master_pipes |= BIT(pipe); + *master_pipes |= BIT(pipe); if (tmp & UNCOMPRESSED_JOINER_SLAVE) - slave_pipes |= BIT(pipe); + *slave_pipes |= BIT(pipe); } } /* Bigjoiner pipes should always be consecutive master and slave */ - drm_WARN(&dev_priv->drm, slave_pipes != master_pipes << 1, + drm_WARN(&dev_priv->drm, *slave_pipes != *master_pipes << 1, "Bigjoiner misconfigured (master pipes 0x%x, slave pipes 0x%x)\n", - master_pipes, slave_pipes); + *master_pipes, *slave_pipes); +} + +static enum pipe get_bigjoiner_master_pipe(enum pipe pipe, u8 master_pipes, u8 slave_pipes) +{ + if ((slave_pipes & BIT(pipe)) == 0) + return pipe; + + /* ignore everything above our pipe */ + master_pipes &= ~GENMASK(7, pipe); + + /* highest remaining bit should be our master pipe */ + return fls(master_pipes) - 1; +} + +static u8 get_bigjoiner_slave_pipes(enum pipe pipe, u8 master_pipes, u8 slave_pipes) +{ + enum pipe master_pipe, next_master_pipe; + + master_pipe = get_bigjoiner_master_pipe(pipe, master_pipes, slave_pipes); - return slave_pipes; + if ((master_pipes & BIT(master_pipe)) == 0) + return 0; + + /* ignore our master pipe and everything below it */ + master_pipes &= ~GENMASK(master_pipe, 0); + /* make sure a high bit is set for the ffs() */ + master_pipes |= BIT(7); + /* lowest remaining bit should be the next master pipe */ + next_master_pipe = ffs(master_pipes) - 1; + + return slave_pipes & GENMASK(next_master_pipe - 1, master_pipe); } static u8 hsw_panel_transcoders(struct drm_i915_private *i915) @@ -4181,6 +3972,7 @@ static u8 hsw_enabled_transcoders(struct intel_crtc *crtc) struct drm_i915_private *dev_priv = to_i915(dev); u8 panel_transcoder_mask = hsw_panel_transcoders(dev_priv); enum transcoder cpu_transcoder; + u8 master_pipes, slave_pipes; u8 enabled_transcoders = 0; /* @@ -4232,8 +4024,10 @@ static u8 hsw_enabled_transcoders(struct intel_crtc *crtc) enabled_transcoders |= BIT(cpu_transcoder); /* bigjoiner slave -> consider the master pipe's transcoder as well */ - if (enabled_bigjoiner_pipes(dev_priv) & BIT(crtc->pipe)) { - cpu_transcoder = (enum transcoder) crtc->pipe - 1; + enabled_bigjoiner_pipes(dev_priv, &master_pipes, &slave_pipes); + if (slave_pipes & BIT(crtc->pipe)) { + cpu_transcoder = (enum transcoder) + get_bigjoiner_master_pipe(crtc->pipe, master_pipes, slave_pipes); if (transcoder_ddi_func_is_enabled(dev_priv, cpu_transcoder)) enabled_transcoders |= BIT(cpu_transcoder); } @@ -4358,6 +4152,24 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc, return transcoder_is_dsi(pipe_config->cpu_transcoder); } +static void intel_bigjoiner_get_config(struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + u8 master_pipes, slave_pipes; + enum pipe pipe = crtc->pipe; + + enabled_bigjoiner_pipes(i915, &master_pipes, &slave_pipes); + + if (((master_pipes | slave_pipes) & BIT(pipe)) == 0) + return; + + crtc_state->bigjoiner = true; + crtc_state->bigjoiner_pipes = + BIT(get_bigjoiner_master_pipe(pipe, master_pipes, slave_pipes)) | + get_bigjoiner_slave_pipes(pipe, master_pipes, slave_pipes); +} + static bool hsw_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { @@ -4380,13 +4192,12 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, active = true; } - intel_dsc_get_config(pipe_config); - if (DISPLAY_VER(dev_priv) >= 13 && !pipe_config->dsc.compression_enable) - intel_uncompressed_joiner_get_config(pipe_config); - if (!active) goto out; + intel_dsc_get_config(pipe_config); + intel_bigjoiner_get_config(pipe_config); + if (!transcoder_is_dsi(pipe_config->cpu_transcoder) || DISPLAY_VER(dev_priv) >= 11) intel_get_transcoder_timings(crtc, pipe_config); @@ -4443,19 +4254,7 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, ilk_get_pfit_config(pipe_config); } - if (hsw_crtc_supports_ips(crtc)) { - if (IS_HASWELL(dev_priv)) - pipe_config->ips_enabled = intel_de_read(dev_priv, - IPS_CTL) & IPS_ENABLE; - else { - /* - * We cannot readout IPS state on broadwell, set to - * true so we can set it to a defined state on first - * commit. - */ - pipe_config->ips_enabled = true; - } - } + hsw_ips_get_config(pipe_config); if (pipe_config->cpu_transcoder != TRANSCODER_EDP && !transcoder_is_dsi(pipe_config->cpu_transcoder)) { @@ -4867,169 +4666,6 @@ intel_encoder_current_mode(struct intel_encoder *encoder) return mode; } -/** - * intel_wm_need_update - Check whether watermarks need updating - * @cur: current plane state - * @new: new plane state - * - * Check current plane state versus the new one to determine whether - * watermarks need to be recalculated. - * - * Returns true or false. - */ -static bool intel_wm_need_update(const struct intel_plane_state *cur, - struct intel_plane_state *new) -{ - /* Update watermarks on tiling or size changes. */ - if (new->uapi.visible != cur->uapi.visible) - return true; - - if (!cur->hw.fb || !new->hw.fb) - return false; - - if (cur->hw.fb->modifier != new->hw.fb->modifier || - cur->hw.rotation != new->hw.rotation || - drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) || - drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) || - drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) || - drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst)) - return true; - - return false; -} - -static bool needs_scaling(const struct intel_plane_state *state) -{ - int src_w = drm_rect_width(&state->uapi.src) >> 16; - int src_h = drm_rect_height(&state->uapi.src) >> 16; - int dst_w = drm_rect_width(&state->uapi.dst); - int dst_h = drm_rect_height(&state->uapi.dst); - - return (src_w != dst_w || src_h != dst_h); -} - -int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *new_crtc_state, - const struct intel_plane_state *old_plane_state, - struct intel_plane_state *new_plane_state) -{ - struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); - struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - bool mode_changed = intel_crtc_needs_modeset(new_crtc_state); - bool was_crtc_enabled = old_crtc_state->hw.active; - bool is_crtc_enabled = new_crtc_state->hw.active; - bool turn_off, turn_on, visible, was_visible; - int ret; - - if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_CURSOR) { - ret = skl_update_scaler_plane(new_crtc_state, new_plane_state); - if (ret) - return ret; - } - - was_visible = old_plane_state->uapi.visible; - visible = new_plane_state->uapi.visible; - - if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible)) - was_visible = false; - - /* - * Visibility is calculated as if the crtc was on, but - * after scaler setup everything depends on it being off - * when the crtc isn't active. - * - * FIXME this is wrong for watermarks. Watermarks should also - * be computed as if the pipe would be active. Perhaps move - * per-plane wm computation to the .check_plane() hook, and - * only combine the results from all planes in the current place? - */ - if (!is_crtc_enabled) { - intel_plane_set_invisible(new_crtc_state, new_plane_state); - visible = false; - } - - if (!was_visible && !visible) - return 0; - - turn_off = was_visible && (!visible || mode_changed); - turn_on = visible && (!was_visible || mode_changed); - - drm_dbg_atomic(&dev_priv->drm, - "[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n", - crtc->base.base.id, crtc->base.name, - plane->base.base.id, plane->base.name, - was_visible, visible, - turn_off, turn_on, mode_changed); - - if (turn_on) { - if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) - new_crtc_state->update_wm_pre = true; - - /* must disable cxsr around plane enable/disable */ - if (plane->id != PLANE_CURSOR) - new_crtc_state->disable_cxsr = true; - } else if (turn_off) { - if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) - new_crtc_state->update_wm_post = true; - - /* must disable cxsr around plane enable/disable */ - if (plane->id != PLANE_CURSOR) - new_crtc_state->disable_cxsr = true; - } else if (intel_wm_need_update(old_plane_state, new_plane_state)) { - if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) { - /* FIXME bollocks */ - new_crtc_state->update_wm_pre = true; - new_crtc_state->update_wm_post = true; - } - } - - if (visible || was_visible) - new_crtc_state->fb_bits |= plane->frontbuffer_bit; - - /* - * ILK/SNB DVSACNTR/Sprite Enable - * IVB SPR_CTL/Sprite Enable - * "When in Self Refresh Big FIFO mode, a write to enable the - * plane will be internally buffered and delayed while Big FIFO - * mode is exiting." - * - * Which means that enabling the sprite can take an extra frame - * when we start in big FIFO mode (LP1+). Thus we need to drop - * down to LP0 and wait for vblank in order to make sure the - * sprite gets enabled on the next vblank after the register write. - * Doing otherwise would risk enabling the sprite one frame after - * we've already signalled flip completion. We can resume LP1+ - * once the sprite has been enabled. - * - * - * WaCxSRDisabledForSpriteScaling:ivb - * IVB SPR_SCALE/Scaling Enable - * "Low Power watermarks must be disabled for at least one - * frame before enabling sprite scaling, and kept disabled - * until sprite scaling is disabled." - * - * ILK/SNB DVSASCALE/Scaling Enable - * "When in Self Refresh Big FIFO mode, scaling enable will be - * masked off while Big FIFO mode is exiting." - * - * Despite the w/a only being listed for IVB we assume that - * the ILK/SNB note has similar ramifications, hence we apply - * the w/a on all three platforms. - * - * With experimental results seems this is needed also for primary - * plane, not only sprite plane. - */ - if (plane->id != PLANE_CURSOR && - (IS_IRONLAKE(dev_priv) || IS_SANDYBRIDGE(dev_priv) || - IS_IVYBRIDGE(dev_priv)) && - (turn_on || (!needs_scaling(old_plane_state) && - needs_scaling(new_plane_state)))) - new_crtc_state->disable_lp_wm = true; - - return 0; -} - static bool encoders_cloneable(const struct intel_encoder *a, const struct intel_encoder *b) { @@ -5289,7 +4925,7 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, if (mode_changed && crtc_state->hw.enable && !drm_WARN_ON(&dev_priv->drm, crtc_state->shared_dpll)) { - ret = dev_priv->dpll_funcs->crtc_compute_clock(crtc_state); + ret = intel_dpll_crtc_compute_clock(crtc_state); if (ret) return ret; } @@ -5340,7 +4976,7 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, } if (HAS_IPS(dev_priv)) { - ret = hsw_compute_ips_config(crtc_state); + ret = hsw_ips_compute_config(state, crtc); if (ret) return ret; } @@ -5491,9 +5127,9 @@ intel_dump_m_n_config(const struct intel_crtc_state *pipe_config, struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev); drm_dbg_kms(&i915->drm, - "%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", + "%s: lanes: %i; data_m: %u, data_n: %u, link_m: %u, link_n: %u, tu: %u\n", id, lane_count, - m_n->gmch_m, m_n->gmch_n, + m_n->data_m, m_n->data_n, m_n->link_m, m_n->link_n, m_n->tu); } @@ -5642,9 +5278,10 @@ static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config, transcoder_name(pipe_config->master_transcoder), pipe_config->sync_mode_slaves_mask); - drm_dbg_kms(&dev_priv->drm, "bigjoiner: %s\n", - pipe_config->bigjoiner_slave ? "slave" : - pipe_config->bigjoiner ? "master" : "no"); + drm_dbg_kms(&dev_priv->drm, "bigjoiner: %s, pipes: 0x%x\n", + intel_crtc_is_bigjoiner_slave(pipe_config) ? "slave" : + intel_crtc_is_bigjoiner_master(pipe_config) ? "master" : "no", + pipe_config->bigjoiner_pipes); drm_dbg_kms(&dev_priv->drm, "splitter: %s, link count %d, overlap %d\n", enableddisabled(pipe_config->splitter.enable), @@ -5658,11 +5295,11 @@ static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config, if (intel_crtc_has_dp_encoder(pipe_config)) { intel_dump_m_n_config(pipe_config, "dp m_n", - pipe_config->lane_count, &pipe_config->dp_m_n); - if (pipe_config->has_drrs) - intel_dump_m_n_config(pipe_config, "dp m2_n2", - pipe_config->lane_count, - &pipe_config->dp_m2_n2); + pipe_config->lane_count, + &pipe_config->dp_m_n); + intel_dump_m_n_config(pipe_config, "dp m2_n2", + pipe_config->lane_count, + &pipe_config->dp_m2_n2); } drm_dbg_kms(&dev_priv->drm, @@ -5841,35 +5478,42 @@ static bool check_digital_port_conflicts(struct intel_atomic_state *state) static void intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state) + struct intel_crtc *crtc) { - const struct intel_crtc_state *master_crtc_state; - struct intel_crtc *master_crtc; + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); - master_crtc = intel_master_crtc(crtc_state); - master_crtc_state = intel_atomic_get_new_crtc_state(state, master_crtc); + WARN_ON(intel_crtc_is_bigjoiner_slave(crtc_state)); - /* No need to copy state if the master state is unchanged */ - if (master_crtc_state) - intel_crtc_copy_color_blobs(crtc_state, master_crtc_state); + drm_property_replace_blob(&crtc_state->hw.degamma_lut, + crtc_state->uapi.degamma_lut); + drm_property_replace_blob(&crtc_state->hw.gamma_lut, + crtc_state->uapi.gamma_lut); + drm_property_replace_blob(&crtc_state->hw.ctm, + crtc_state->uapi.ctm); } static void -intel_crtc_copy_uapi_to_hw_state(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state) +intel_crtc_copy_uapi_to_hw_state_modeset(struct intel_atomic_state *state, + struct intel_crtc *crtc) { + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + WARN_ON(intel_crtc_is_bigjoiner_slave(crtc_state)); + crtc_state->hw.enable = crtc_state->uapi.enable; crtc_state->hw.active = crtc_state->uapi.active; crtc_state->hw.mode = crtc_state->uapi.mode; crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode; crtc_state->hw.scaling_filter = crtc_state->uapi.scaling_filter; - intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc_state); + intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc); } static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state) { - if (crtc_state->bigjoiner_slave) + if (intel_crtc_is_bigjoiner_slave(crtc_state)) return; crtc_state->uapi.enable = crtc_state->hw.enable; @@ -5880,7 +5524,6 @@ static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode; crtc_state->uapi.scaling_filter = crtc_state->hw.scaling_filter; - /* copy color blobs to uapi */ drm_property_replace_blob(&crtc_state->uapi.degamma_lut, crtc_state->hw.degamma_lut); drm_property_replace_blob(&crtc_state->uapi.gamma_lut, @@ -5889,51 +5532,79 @@ static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state crtc_state->hw.ctm); } +static void +copy_bigjoiner_crtc_state_nomodeset(struct intel_atomic_state *state, + struct intel_crtc *slave_crtc) +{ + struct intel_crtc_state *slave_crtc_state = + intel_atomic_get_new_crtc_state(state, slave_crtc); + struct intel_crtc *master_crtc = intel_master_crtc(slave_crtc_state); + const struct intel_crtc_state *master_crtc_state = + intel_atomic_get_new_crtc_state(state, master_crtc); + + drm_property_replace_blob(&slave_crtc_state->hw.degamma_lut, + master_crtc_state->hw.degamma_lut); + drm_property_replace_blob(&slave_crtc_state->hw.gamma_lut, + master_crtc_state->hw.gamma_lut); + drm_property_replace_blob(&slave_crtc_state->hw.ctm, + master_crtc_state->hw.ctm); + + slave_crtc_state->uapi.color_mgmt_changed = master_crtc_state->uapi.color_mgmt_changed; +} + static int -copy_bigjoiner_crtc_state(struct intel_crtc_state *crtc_state, - const struct intel_crtc_state *from_crtc_state) +copy_bigjoiner_crtc_state_modeset(struct intel_atomic_state *state, + struct intel_crtc *slave_crtc) { + struct intel_crtc_state *slave_crtc_state = + intel_atomic_get_new_crtc_state(state, slave_crtc); + struct intel_crtc *master_crtc = intel_master_crtc(slave_crtc_state); + const struct intel_crtc_state *master_crtc_state = + intel_atomic_get_new_crtc_state(state, master_crtc); struct intel_crtc_state *saved_state; - saved_state = kmemdup(from_crtc_state, sizeof(*saved_state), GFP_KERNEL); + saved_state = kmemdup(master_crtc_state, sizeof(*saved_state), GFP_KERNEL); if (!saved_state) return -ENOMEM; - saved_state->uapi = crtc_state->uapi; - saved_state->scaler_state = crtc_state->scaler_state; - saved_state->shared_dpll = crtc_state->shared_dpll; - saved_state->dpll_hw_state = crtc_state->dpll_hw_state; - saved_state->crc_enabled = crtc_state->crc_enabled; + /* preserve some things from the slave's original crtc state */ + saved_state->uapi = slave_crtc_state->uapi; + saved_state->scaler_state = slave_crtc_state->scaler_state; + saved_state->shared_dpll = slave_crtc_state->shared_dpll; + saved_state->dpll_hw_state = slave_crtc_state->dpll_hw_state; + saved_state->crc_enabled = slave_crtc_state->crc_enabled; - intel_crtc_free_hw_state(crtc_state); - memcpy(crtc_state, saved_state, sizeof(*crtc_state)); + intel_crtc_free_hw_state(slave_crtc_state); + memcpy(slave_crtc_state, saved_state, sizeof(*slave_crtc_state)); kfree(saved_state); /* Re-init hw state */ - memset(&crtc_state->hw, 0, sizeof(saved_state->hw)); - crtc_state->hw.enable = from_crtc_state->hw.enable; - crtc_state->hw.active = from_crtc_state->hw.active; - crtc_state->hw.pipe_mode = from_crtc_state->hw.pipe_mode; - crtc_state->hw.adjusted_mode = from_crtc_state->hw.adjusted_mode; + memset(&slave_crtc_state->hw, 0, sizeof(slave_crtc_state->hw)); + slave_crtc_state->hw.enable = master_crtc_state->hw.enable; + slave_crtc_state->hw.active = master_crtc_state->hw.active; + slave_crtc_state->hw.mode = master_crtc_state->hw.mode; + slave_crtc_state->hw.pipe_mode = master_crtc_state->hw.pipe_mode; + slave_crtc_state->hw.adjusted_mode = master_crtc_state->hw.adjusted_mode; + slave_crtc_state->hw.scaling_filter = master_crtc_state->hw.scaling_filter; + + copy_bigjoiner_crtc_state_nomodeset(state, slave_crtc); /* Some fixups */ - crtc_state->uapi.mode_changed = from_crtc_state->uapi.mode_changed; - crtc_state->uapi.connectors_changed = from_crtc_state->uapi.connectors_changed; - crtc_state->uapi.active_changed = from_crtc_state->uapi.active_changed; - crtc_state->nv12_planes = crtc_state->c8_planes = crtc_state->update_planes = 0; - crtc_state->bigjoiner_linked_crtc = to_intel_crtc(from_crtc_state->uapi.crtc); - crtc_state->bigjoiner_slave = true; - crtc_state->cpu_transcoder = from_crtc_state->cpu_transcoder; - crtc_state->has_audio = from_crtc_state->has_audio; + slave_crtc_state->uapi.mode_changed = master_crtc_state->uapi.mode_changed; + slave_crtc_state->uapi.connectors_changed = master_crtc_state->uapi.connectors_changed; + slave_crtc_state->uapi.active_changed = master_crtc_state->uapi.active_changed; + slave_crtc_state->cpu_transcoder = master_crtc_state->cpu_transcoder; + slave_crtc_state->has_audio = master_crtc_state->has_audio; return 0; } static int intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, - struct intel_crtc_state *crtc_state) + struct intel_crtc *crtc) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_crtc_state *saved_state; @@ -5963,7 +5634,7 @@ intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, memcpy(crtc_state, saved_state, sizeof(*crtc_state)); kfree(saved_state); - intel_crtc_copy_uapi_to_hw_state(state, crtc_state); + intel_crtc_copy_uapi_to_hw_state_modeset(state, crtc); return 0; } @@ -6189,8 +5860,8 @@ intel_compare_link_m_n(const struct intel_link_m_n *m_n, bool exact) { return m_n->tu == m2_n2->tu && - intel_compare_m_n(m_n->gmch_m, m_n->gmch_n, - m2_n2->gmch_m, m2_n2->gmch_n, exact) && + intel_compare_m_n(m_n->data_m, m_n->data_n, + m2_n2->data_m, m2_n2->data_n, exact) && intel_compare_m_n(m_n->link_m, m_n->link_n, m2_n2->link_m, m2_n2->link_n, exact); } @@ -6389,16 +6060,16 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, &pipe_config->name,\ !fastset)) { \ pipe_config_mismatch(fastset, crtc, __stringify(name), \ - "(expected tu %i gmch %i/%i link %i/%i, " \ - "found tu %i, gmch %i/%i link %i/%i)", \ + "(expected tu %i data %i/%i link %i/%i, " \ + "found tu %i, data %i/%i link %i/%i)", \ current_config->name.tu, \ - current_config->name.gmch_m, \ - current_config->name.gmch_n, \ + current_config->name.data_m, \ + current_config->name.data_n, \ current_config->name.link_m, \ current_config->name.link_n, \ pipe_config->name.tu, \ - pipe_config->name.gmch_m, \ - pipe_config->name.gmch_n, \ + pipe_config->name.data_m, \ + pipe_config->name.data_n, \ pipe_config->name.link_m, \ pipe_config->name.link_n); \ ret = false; \ @@ -6416,22 +6087,22 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, !intel_compare_link_m_n(¤t_config->alt_name, \ &pipe_config->name, !fastset)) { \ pipe_config_mismatch(fastset, crtc, __stringify(name), \ - "(expected tu %i gmch %i/%i link %i/%i, " \ - "or tu %i gmch %i/%i link %i/%i, " \ - "found tu %i, gmch %i/%i link %i/%i)", \ + "(expected tu %i data %i/%i link %i/%i, " \ + "or tu %i data %i/%i link %i/%i, " \ + "found tu %i, data %i/%i link %i/%i)", \ current_config->name.tu, \ - current_config->name.gmch_m, \ - current_config->name.gmch_n, \ + current_config->name.data_m, \ + current_config->name.data_n, \ current_config->name.link_m, \ current_config->name.link_n, \ current_config->alt_name.tu, \ - current_config->alt_name.gmch_m, \ - current_config->alt_name.gmch_n, \ + current_config->alt_name.data_m, \ + current_config->alt_name.data_n, \ current_config->alt_name.link_m, \ current_config->alt_name.link_n, \ pipe_config->name.tu, \ - pipe_config->name.gmch_m, \ - pipe_config->name.gmch_n, \ + pipe_config->name.data_m, \ + pipe_config->name.data_n, \ pipe_config->name.link_m, \ pipe_config->name.link_n); \ ret = false; \ @@ -6510,13 +6181,12 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_I(lane_count); PIPE_CONF_CHECK_X(lane_lat_optim_mask); - if (DISPLAY_VER(dev_priv) < 8) { - PIPE_CONF_CHECK_M_N(dp_m_n); - - if (current_config->has_drrs) - PIPE_CONF_CHECK_M_N(dp_m2_n2); - } else + if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) { PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2); + } else { + PIPE_CONF_CHECK_M_N(dp_m_n); + PIPE_CONF_CHECK_M_N(dp_m2_n2); + } PIPE_CONF_CHECK_X(output_types); @@ -6642,6 +6312,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1); PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2); PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0); + PIPE_CONF_CHECK_X(dpll_hw_state.div0); PIPE_CONF_CHECK_X(dpll_hw_state.ebb0); PIPE_CONF_CHECK_X(dpll_hw_state.ebb4); PIPE_CONF_CHECK_X(dpll_hw_state.pll0); @@ -6693,8 +6364,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_X(sync_mode_slaves_mask); PIPE_CONF_CHECK_I(master_transcoder); PIPE_CONF_CHECK_BOOL(bigjoiner); - PIPE_CONF_CHECK_BOOL(bigjoiner_slave); - PIPE_CONF_CHECK_P(bigjoiner_linked_crtc); + PIPE_CONF_CHECK_X(bigjoiner_pipes); PIPE_CONF_CHECK_I(dsc.compression_enable); PIPE_CONF_CHECK_I(dsc.dsc_split); @@ -7480,20 +7150,25 @@ static int intel_crtc_add_bigjoiner_planes(struct intel_atomic_state *state, static int intel_bigjoiner_add_affected_planes(struct intel_atomic_state *state) { + struct drm_i915_private *i915 = to_i915(state->base.dev); const struct intel_crtc_state *crtc_state; struct intel_crtc *crtc; int i; for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { - int ret; + struct intel_crtc *other; - if (!crtc_state->bigjoiner) - continue; + for_each_intel_crtc_in_pipe_mask(&i915->drm, other, + crtc_state->bigjoiner_pipes) { + int ret; - ret = intel_crtc_add_bigjoiner_planes(state, crtc, - crtc_state->bigjoiner_linked_crtc); - if (ret) - return ret; + if (crtc == other) + continue; + + ret = intel_crtc_add_bigjoiner_planes(state, crtc, other); + if (ret) + return ret; + } } return 0; @@ -7595,67 +7270,123 @@ static bool intel_cpu_transcoders_need_modeset(struct intel_atomic_state *state, return false; } -static int intel_atomic_check_bigjoiner(struct intel_atomic_state *state, - struct intel_crtc *crtc, - struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *new_crtc_state) +static bool intel_pipes_need_modeset(struct intel_atomic_state *state, + u8 pipes) { - struct intel_crtc_state *slave_crtc_state, *master_crtc_state; - struct intel_crtc *slave_crtc, *master_crtc; + const struct intel_crtc_state *new_crtc_state; + struct intel_crtc *crtc; + int i; - /* slave being enabled, is master is still claiming this crtc? */ - if (old_crtc_state->bigjoiner_slave) { - slave_crtc = crtc; - master_crtc = old_crtc_state->bigjoiner_linked_crtc; - master_crtc_state = intel_atomic_get_new_crtc_state(state, master_crtc); - if (!master_crtc_state || !intel_crtc_needs_modeset(master_crtc_state)) - goto claimed; + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + if (new_crtc_state->hw.enable && + pipes & BIT(crtc->pipe) && + intel_crtc_needs_modeset(new_crtc_state)) + return true; } - if (!new_crtc_state->bigjoiner) + return false; +} + +static int intel_atomic_check_bigjoiner(struct intel_atomic_state *state, + struct intel_crtc *master_crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + struct intel_crtc_state *master_crtc_state = + intel_atomic_get_new_crtc_state(state, master_crtc); + struct intel_crtc *slave_crtc; + u8 slave_pipes; + + /* + * TODO: encoder.compute_config() may be the best + * place to populate the bitmask for the master crtc. + * For now encoder.compute_config() just flags things + * as needing bigjoiner and we populate the bitmask + * here. + */ + WARN_ON(master_crtc_state->bigjoiner_pipes); + + if (!master_crtc_state->bigjoiner) return 0; - slave_crtc = intel_dsc_get_bigjoiner_secondary(crtc); - if (!slave_crtc) { - DRM_DEBUG_KMS("[CRTC:%d:%s] Big joiner configuration requires " - "CRTC + 1 to be used, doesn't exist\n", - crtc->base.base.id, crtc->base.name); + slave_pipes = BIT(master_crtc->pipe + 1); + + if (slave_pipes & ~bigjoiner_pipes(i915)) { + drm_dbg_kms(&i915->drm, + "[CRTC:%d:%s] Cannot act as big joiner master " + "(need 0x%x as slave pipes, only 0x%x possible)\n", + master_crtc->base.base.id, master_crtc->base.name, + slave_pipes, bigjoiner_pipes(i915)); return -EINVAL; } - new_crtc_state->bigjoiner_linked_crtc = slave_crtc; - slave_crtc_state = intel_atomic_get_crtc_state(&state->base, slave_crtc); - master_crtc = crtc; - if (IS_ERR(slave_crtc_state)) - return PTR_ERR(slave_crtc_state); + for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc, slave_pipes) { + struct intel_crtc_state *slave_crtc_state; + int ret; - /* master being enabled, slave was already configured? */ - if (slave_crtc_state->uapi.enable) - goto claimed; + slave_crtc_state = intel_atomic_get_crtc_state(&state->base, slave_crtc); + if (IS_ERR(slave_crtc_state)) + return PTR_ERR(slave_crtc_state); - DRM_DEBUG_KMS("[CRTC:%d:%s] Used as slave for big joiner\n", - slave_crtc->base.base.id, slave_crtc->base.name); + /* master being enabled, slave was already configured? */ + if (slave_crtc_state->uapi.enable) { + drm_dbg_kms(&i915->drm, + "[CRTC:%d:%s] Slave is enabled as normal CRTC, but " + "[CRTC:%d:%s] claiming this CRTC for bigjoiner.\n", + slave_crtc->base.base.id, slave_crtc->base.name, + master_crtc->base.base.id, master_crtc->base.name); + return -EINVAL; + } - return copy_bigjoiner_crtc_state(slave_crtc_state, new_crtc_state); + /* + * The state copy logic assumes the master crtc gets processed + * before the slave crtc during the main compute_config loop. + * This works because the crtcs are created in pipe order, + * and the hardware requires master pipe < slave pipe as well. + * Should that change we need to rethink the logic. + */ + if (WARN_ON(drm_crtc_index(&master_crtc->base) > + drm_crtc_index(&slave_crtc->base))) + return -EINVAL; + + drm_dbg_kms(&i915->drm, + "[CRTC:%d:%s] Used as slave for big joiner master [CRTC:%d:%s]\n", + slave_crtc->base.base.id, slave_crtc->base.name, + master_crtc->base.base.id, master_crtc->base.name); -claimed: - DRM_DEBUG_KMS("[CRTC:%d:%s] Slave is enabled as normal CRTC, but " - "[CRTC:%d:%s] claiming this CRTC for bigjoiner.\n", - slave_crtc->base.base.id, slave_crtc->base.name, - master_crtc->base.base.id, master_crtc->base.name); - return -EINVAL; + master_crtc_state->bigjoiner_pipes = + BIT(master_crtc->pipe) | BIT(slave_crtc->pipe); + slave_crtc_state->bigjoiner_pipes = + BIT(master_crtc->pipe) | BIT(slave_crtc->pipe); + + ret = copy_bigjoiner_crtc_state_modeset(state, slave_crtc); + if (ret) + return ret; + } + + return 0; } static void kill_bigjoiner_slave(struct intel_atomic_state *state, - struct intel_crtc_state *master_crtc_state) + struct intel_crtc *master_crtc) { - struct intel_crtc_state *slave_crtc_state = - intel_atomic_get_new_crtc_state(state, master_crtc_state->bigjoiner_linked_crtc); + struct drm_i915_private *i915 = to_i915(state->base.dev); + struct intel_crtc_state *master_crtc_state = + intel_atomic_get_new_crtc_state(state, master_crtc); + struct intel_crtc *slave_crtc; + + for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc, + intel_crtc_bigjoiner_slave_pipes(master_crtc_state)) { + struct intel_crtc_state *slave_crtc_state = + intel_atomic_get_new_crtc_state(state, slave_crtc); + + slave_crtc_state->bigjoiner = false; + slave_crtc_state->bigjoiner_pipes = 0; + + intel_crtc_copy_uapi_to_hw_state_modeset(state, slave_crtc); + } - slave_crtc_state->bigjoiner = master_crtc_state->bigjoiner = false; - slave_crtc_state->bigjoiner_slave = master_crtc_state->bigjoiner_slave = false; - slave_crtc_state->bigjoiner_linked_crtc = master_crtc_state->bigjoiner_linked_crtc = NULL; - intel_crtc_copy_uapi_to_hw_state(state, slave_crtc_state); + master_crtc_state->bigjoiner = false; + master_crtc_state->bigjoiner_pipes = 0; } /** @@ -7666,7 +7397,7 @@ static void kill_bigjoiner_slave(struct intel_atomic_state *state, * Correspondingly, support is currently added for primary plane only. * * Async flip can only change the plane surface address, so anything else - * changing is rejected from the intel_atomic_check_async() function. + * changing is rejected from the intel_async_flip_check_hw() function. * Once this check is cleared, flip done interrupt is enabled using * the intel_crtc_enable_flip_done() function. * @@ -7676,7 +7407,65 @@ static void kill_bigjoiner_slave(struct intel_atomic_state *state, * correspond to the last vblank and have no relation to the actual time when * the flip done event was sent. */ -static int intel_atomic_check_async(struct intel_atomic_state *state, struct intel_crtc *crtc) +static int intel_async_flip_check_uapi(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + const struct intel_plane_state *old_plane_state; + struct intel_plane_state *new_plane_state; + struct intel_plane *plane; + int i; + + if (!new_crtc_state->uapi.async_flip) + return 0; + + if (!new_crtc_state->uapi.active) { + drm_dbg_kms(&i915->drm, + "[CRTC:%d:%s] not active\n", + crtc->base.base.id, crtc->base.name); + return -EINVAL; + } + + if (intel_crtc_needs_modeset(new_crtc_state)) { + drm_dbg_kms(&i915->drm, + "[CRTC:%d:%s] modeset required\n", + crtc->base.base.id, crtc->base.name); + return -EINVAL; + } + + for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state, + new_plane_state, i) { + if (plane->pipe != crtc->pipe) + continue; + + /* + * TODO: Async flip is only supported through the page flip IOCTL + * as of now. So support currently added for primary plane only. + * Support for other planes on platforms on which supports + * this(vlv/chv and icl+) should be added when async flip is + * enabled in the atomic IOCTL path. + */ + if (!plane->async_flip) { + drm_dbg_kms(&i915->drm, + "[PLANE:%d:%s] async flip not supported\n", + plane->base.base.id, plane->base.name); + return -EINVAL; + } + + if (!old_plane_state->uapi.fb || !new_plane_state->uapi.fb) { + drm_dbg_kms(&i915->drm, + "[PLANE:%d:%s] no old or new framebuffer\n", + plane->base.base.id, plane->base.name); + return -EINVAL; + } + } + + return 0; +} + +static int intel_async_flip_check_hw(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *i915 = to_i915(state->base.dev); const struct intel_crtc_state *old_crtc_state, *new_crtc_state; @@ -7687,6 +7476,9 @@ static int intel_atomic_check_async(struct intel_atomic_state *state, struct int old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc); new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); + if (!new_crtc_state->uapi.async_flip) + return 0; + if (intel_crtc_needs_modeset(new_crtc_state)) { drm_dbg_kms(&i915->drm, "Modeset Required. Async flip not supported\n"); return -EINVAL; @@ -7708,16 +7500,26 @@ static int intel_atomic_check_async(struct intel_atomic_state *state, struct int continue; /* - * TODO: Async flip is only supported through the page flip IOCTL - * as of now. So support currently added for primary plane only. - * Support for other planes on platforms on which supports - * this(vlv/chv and icl+) should be added when async flip is - * enabled in the atomic IOCTL path. + * Only async flip capable planes should be in the state + * if we're really about to ask the hardware to perform + * an async flip. We should never get this far otherwise. */ - if (!plane->async_flip) + if (drm_WARN_ON(&i915->drm, + new_crtc_state->do_async_flip && !plane->async_flip)) return -EINVAL; /* + * Only check async flip capable planes other planes + * may be involved in the initial commit due to + * the wm0/ddb optimization. + * + * TODO maybe should track which planes actually + * were requested to do the async flip... + */ + if (!plane->async_flip) + continue; + + /* * FIXME: This check is kept generic for all platforms. * Need to verify this for all gen9 platforms to enable * this selectively if required. @@ -7805,34 +7607,37 @@ static int intel_atomic_check_async(struct intel_atomic_state *state, struct int static int intel_bigjoiner_add_affected_crtcs(struct intel_atomic_state *state) { + struct drm_i915_private *i915 = to_i915(state->base.dev); struct intel_crtc_state *crtc_state; struct intel_crtc *crtc; + u8 affected_pipes = 0; + u8 modeset_pipes = 0; int i; for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { - struct intel_crtc_state *linked_crtc_state; - struct intel_crtc *linked_crtc; - int ret; + affected_pipes |= crtc_state->bigjoiner_pipes; + if (intel_crtc_needs_modeset(crtc_state)) + modeset_pipes |= crtc_state->bigjoiner_pipes; + } - if (!crtc_state->bigjoiner) - continue; + for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, affected_pipes) { + crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + } - linked_crtc = crtc_state->bigjoiner_linked_crtc; - linked_crtc_state = intel_atomic_get_crtc_state(&state->base, linked_crtc); - if (IS_ERR(linked_crtc_state)) - return PTR_ERR(linked_crtc_state); + for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, modeset_pipes) { + int ret; - if (!intel_crtc_needs_modeset(crtc_state)) - continue; + crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - linked_crtc_state->uapi.mode_changed = true; + crtc_state->uapi.mode_changed = true; - ret = drm_atomic_add_affected_connectors(&state->base, - &linked_crtc->base); + ret = drm_atomic_add_affected_connectors(&state->base, &crtc->base); if (ret) return ret; - ret = intel_atomic_add_affected_planes(state, linked_crtc); + ret = intel_atomic_add_affected_planes(state, crtc); if (ret) return ret; } @@ -7840,8 +7645,8 @@ static int intel_bigjoiner_add_affected_crtcs(struct intel_atomic_state *state) for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { /* Kill old bigjoiner link, we may re-establish afterwards */ if (intel_crtc_needs_modeset(crtc_state) && - crtc_state->bigjoiner && !crtc_state->bigjoiner_slave) - kill_bigjoiner_slave(state, crtc_state); + intel_crtc_is_bigjoiner_master(crtc_state)) + kill_bigjoiner_slave(state, crtc); } return 0; @@ -7866,6 +7671,10 @@ static int intel_atomic_check(struct drm_device *dev, new_crtc_state, i) { if (new_crtc_state->inherited != old_crtc_state->inherited) new_crtc_state->uapi.mode_changed = true; + + if (new_crtc_state->uapi.scaling_filter != + old_crtc_state->uapi.scaling_filter) + new_crtc_state->uapi.mode_changed = true; } intel_vrr_check_modeset(state); @@ -7874,6 +7683,12 @@ static int intel_atomic_check(struct drm_device *dev, if (ret) goto fail; + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + ret = intel_async_flip_check_uapi(state, crtc); + if (ret) + return ret; + } + ret = intel_bigjoiner_add_affected_crtcs(state); if (ret) goto fail; @@ -7881,30 +7696,30 @@ static int intel_atomic_check(struct drm_device *dev, for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { if (!intel_crtc_needs_modeset(new_crtc_state)) { - /* Light copy */ - intel_crtc_copy_uapi_to_hw_state_nomodeset(state, new_crtc_state); - + if (intel_crtc_is_bigjoiner_slave(new_crtc_state)) + copy_bigjoiner_crtc_state_nomodeset(state, crtc); + else + intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc); continue; } - if (!new_crtc_state->uapi.enable) { - if (!new_crtc_state->bigjoiner_slave) { - intel_crtc_copy_uapi_to_hw_state(state, new_crtc_state); - any_ms = true; - } + if (intel_crtc_is_bigjoiner_slave(new_crtc_state)) { + drm_WARN_ON(&dev_priv->drm, new_crtc_state->uapi.enable); continue; } - ret = intel_crtc_prepare_cleared_state(state, new_crtc_state); + ret = intel_crtc_prepare_cleared_state(state, crtc); if (ret) goto fail; + if (!new_crtc_state->hw.enable) + continue; + ret = intel_modeset_pipe_config(state, new_crtc_state); if (ret) goto fail; - ret = intel_atomic_check_bigjoiner(state, crtc, old_crtc_state, - new_crtc_state); + ret = intel_atomic_check_bigjoiner(state, crtc); if (ret) goto fail; } @@ -7958,10 +7773,7 @@ static int intel_atomic_check(struct drm_device *dev, } if (new_crtc_state->bigjoiner) { - struct intel_crtc_state *linked_crtc_state = - intel_atomic_get_new_crtc_state(state, new_crtc_state->bigjoiner_linked_crtc); - - if (intel_crtc_needs_modeset(linked_crtc_state)) { + if (intel_pipes_need_modeset(state, new_crtc_state->bigjoiner_pipes)) { new_crtc_state->uapi.mode_changed = true; new_crtc_state->update_pipe = false; } @@ -8033,11 +7845,9 @@ static int intel_atomic_check(struct drm_device *dev, for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - if (new_crtc_state->uapi.async_flip) { - ret = intel_atomic_check_async(state, crtc); - if (ret) - goto fail; - } + ret = intel_async_flip_check_hw(state, crtc); + if (ret) + goto fail; if (!intel_crtc_needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe) @@ -8141,9 +7951,6 @@ static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state, if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) hsw_set_linetime_wm(new_crtc_state); - - if (DISPLAY_VER(dev_priv) >= 11) - icl_set_pipe_chicken(new_crtc_state); } static void commit_pipe_pre_planes(struct intel_atomic_state *state, @@ -8208,7 +8015,7 @@ static void intel_enable_crtc(struct intel_atomic_state *state, dev_priv->display->crtc_enable(state, crtc); - if (new_crtc_state->bigjoiner_slave) + if (intel_crtc_is_bigjoiner_slave(new_crtc_state)) return; /* vblanks work again, re-enable pipe CRC. */ @@ -8218,7 +8025,7 @@ static void intel_enable_crtc(struct intel_atomic_state *state, static void intel_update_crtc(struct intel_atomic_state *state, struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct drm_i915_private *i915 = to_i915(state->base.dev); const struct intel_crtc_state *old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc); struct intel_crtc_state *new_crtc_state = @@ -8235,21 +8042,22 @@ static void intel_update_crtc(struct intel_atomic_state *state, if (new_crtc_state->update_pipe) intel_encoders_update_pipe(state, crtc); + + if (DISPLAY_VER(i915) >= 11 && + new_crtc_state->update_pipe) + icl_set_pipe_chicken(new_crtc_state); } intel_fbc_update(state, crtc); - intel_update_planes_on_crtc(state, crtc); + intel_crtc_planes_update_noarm(state, crtc); /* Perform vblank evasion around commit operation */ intel_pipe_update_start(new_crtc_state); commit_pipe_pre_planes(state, crtc); - if (DISPLAY_VER(dev_priv) >= 9) - skl_arm_planes_on_crtc(state, crtc); - else - i9xx_arm_planes_on_crtc(state, crtc); + intel_crtc_planes_update_arm(state, crtc); commit_pipe_post_planes(state, crtc); @@ -8325,7 +8133,7 @@ static void intel_commit_modeset_disables(struct intel_atomic_state *state) */ if (!is_trans_port_sync_slave(old_crtc_state) && !intel_dp_mst_is_slave_trans(old_crtc_state) && - !old_crtc_state->bigjoiner_slave) + !intel_crtc_is_bigjoiner_slave(old_crtc_state)) continue; intel_old_crtc_state_disables(state, old_crtc_state, @@ -8440,7 +8248,7 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state) if (intel_dp_mst_is_slave_trans(new_crtc_state) || is_trans_port_sync_master(new_crtc_state) || - (new_crtc_state->bigjoiner && !new_crtc_state->bigjoiner_slave)) + intel_crtc_is_bigjoiner_master(new_crtc_state)) continue; modeset_pipes &= ~BIT(pipe); @@ -8661,7 +8469,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state) intel_dbuf_pre_plane_update(state); for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { - if (new_crtc_state->uapi.async_flip) + if (new_crtc_state->do_async_flip) intel_crtc_enable_flip_done(state, crtc); } @@ -8687,7 +8495,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state) drm_atomic_helper_wait_for_flip_done(dev, &state->base); for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { - if (new_crtc_state->uapi.async_flip) + if (new_crtc_state->do_async_flip) intel_crtc_disable_flip_done(state, crtc); } @@ -8967,10 +8775,8 @@ static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder) struct intel_crtc *crtc; u32 possible_crtcs = 0; - for_each_intel_crtc(dev, crtc) { - if (encoder->pipe_mask & BIT(crtc->pipe)) - possible_crtcs |= drm_crtc_mask(&crtc->base); - } + for_each_intel_crtc_in_pipe_mask(dev, crtc, encoder->pipe_mask) + possible_crtcs |= drm_crtc_mask(&crtc->base); return possible_crtcs; } @@ -9026,6 +8832,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) intel_ddi_init(dev_priv, PORT_B); intel_ddi_init(dev_priv, PORT_C); intel_ddi_init(dev_priv, PORT_D_XELPD); + intel_ddi_init(dev_priv, PORT_TC1); } else if (IS_ALDERLAKE_P(dev_priv)) { intel_ddi_init(dev_priv, PORT_A); intel_ddi_init(dev_priv, PORT_B); @@ -9478,7 +9285,7 @@ void intel_modeset_init_hw(struct drm_i915_private *i915) cdclk_state = to_intel_cdclk_state(i915->cdclk.obj.state); intel_update_cdclk(i915); - intel_dump_cdclk_config(&i915->cdclk.hw, "Current CDCLK"); + intel_cdclk_dump_config(i915, &i915->cdclk.hw, "Current CDCLK"); cdclk_state->logical = cdclk_state->actual = i915->cdclk.hw; } @@ -9980,8 +9787,7 @@ void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) udelay(150); /* wait for warmup */ } - intel_de_write(dev_priv, PIPECONF(pipe), - PIPECONF_ENABLE | PIPECONF_PROGRESSIVE); + intel_de_write(dev_priv, PIPECONF(pipe), PIPECONF_ENABLE); intel_de_posting_read(dev_priv, PIPECONF(pipe)); intel_wait_for_pipe_scanline_moving(crtc); @@ -9995,18 +9801,15 @@ void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) pipe_name(pipe)); drm_WARN_ON(&dev_priv->drm, - intel_de_read(dev_priv, DSPCNTR(PLANE_A)) & - DISPLAY_PLANE_ENABLE); + intel_de_read(dev_priv, DSPCNTR(PLANE_A)) & DISP_ENABLE); drm_WARN_ON(&dev_priv->drm, - intel_de_read(dev_priv, DSPCNTR(PLANE_B)) & - DISPLAY_PLANE_ENABLE); + intel_de_read(dev_priv, DSPCNTR(PLANE_B)) & DISP_ENABLE); drm_WARN_ON(&dev_priv->drm, - intel_de_read(dev_priv, DSPCNTR(PLANE_C)) & - DISPLAY_PLANE_ENABLE); + intel_de_read(dev_priv, DSPCNTR(PLANE_C)) & DISP_ENABLE); drm_WARN_ON(&dev_priv->drm, - intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE); + intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE_MASK); drm_WARN_ON(&dev_priv->drm, - intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE); + intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE_MASK); intel_de_write(dev_priv, PIPECONF(pipe), 0); intel_de_posting_read(dev_priv, PIPECONF(pipe)); @@ -10156,7 +9959,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc, /* Adjust the state of the output pipe according to whether we * have active connectors/encoders. */ if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc) && - !crtc_state->bigjoiner_slave) + !intel_crtc_is_bigjoiner_slave(crtc_state)) intel_crtc_disable_noatomic(crtc, ctx); if (crtc_state->hw.active || HAS_GMCH(dev_priv)) { @@ -10369,12 +10172,18 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) /* read out to slave crtc as well for bigjoiner */ if (crtc_state->bigjoiner) { + struct intel_crtc *slave_crtc; + /* encoder should read be linked to bigjoiner master */ - WARN_ON(crtc_state->bigjoiner_slave); + WARN_ON(intel_crtc_is_bigjoiner_slave(crtc_state)); + + for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, slave_crtc, + intel_crtc_bigjoiner_slave_pipes(crtc_state)) { + struct intel_crtc_state *slave_crtc_state; - crtc = crtc_state->bigjoiner_linked_crtc; - crtc_state = to_intel_crtc_state(crtc->base.state); - intel_encoder_get_config(encoder, crtc_state); + slave_crtc_state = to_intel_crtc_state(slave_crtc->base.state); + intel_encoder_get_config(encoder, slave_crtc_state); + } } } else { encoder->base.crtc = NULL; @@ -10689,6 +10498,8 @@ intel_modeset_setup_hw_state(struct drm_device *dev, } intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); + + intel_power_domains_sanitize_state(dev_priv); } void intel_display_resume(struct drm_device *dev) diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index b61b75248ded..11d6134c53c8 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -27,7 +27,8 @@ #include <drm/drm_util.h> -enum link_m_n_set; +#include "i915_reg_defs.h" + enum drm_scaling_filter; struct dpll; struct drm_connector; @@ -317,8 +318,8 @@ enum aux_ch { /* Used by dp and fdi links */ struct intel_link_m_n { u32 tu; - u32 gmch_m; - u32 gmch_n; + u32 data_m; + u32 data_n; u32 link_m; u32 link_n; }; @@ -429,11 +430,11 @@ enum hpd_pin { &(dev)->mode_config.crtc_list, \ base.head) -#define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask) \ +#define for_each_intel_crtc_in_pipe_mask(dev, intel_crtc, pipe_mask) \ list_for_each_entry(intel_crtc, \ &(dev)->mode_config.crtc_list, \ base.head) \ - for_each_if((crtc_mask) & drm_crtc_mask(&intel_crtc->base)) + for_each_if((pipe_mask) & BIT(intel_crtc->pipe)) #define for_each_intel_encoder(dev, intel_encoder) \ list_for_each_entry(intel_encoder, \ @@ -554,6 +555,10 @@ intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv, bool bigjoiner); enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port); bool is_trans_port_sync_mode(const struct intel_crtc_state *state); +bool intel_crtc_is_bigjoiner_slave(const struct intel_crtc_state *crtc_state); +bool intel_crtc_is_bigjoiner_master(const struct intel_crtc_state *crtc_state); +u8 intel_crtc_bigjoiner_slave_pipes(const struct intel_crtc_state *crtc_state); +struct intel_crtc *intel_master_crtc(const struct intel_crtc_state *crtc_state); void intel_plane_destroy(struct drm_plane *plane); void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state); @@ -605,18 +610,32 @@ bool intel_fuzzy_clock_check(int clock1, int clock2); void intel_display_prepare_reset(struct drm_i915_private *dev_priv); void intel_display_finish_reset(struct drm_i915_private *dev_priv); -void intel_dp_get_m_n(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config); -void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, - enum link_m_n_set m_n); -void ilk_get_fdi_m_n_config(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config); +void intel_zero_m_n(struct intel_link_m_n *m_n); +void intel_set_m_n(struct drm_i915_private *i915, + const struct intel_link_m_n *m_n, + i915_reg_t data_m_reg, i915_reg_t data_n_reg, + i915_reg_t link_m_reg, i915_reg_t link_n_reg); +void intel_get_m_n(struct drm_i915_private *i915, + struct intel_link_m_n *m_n, + i915_reg_t data_m_reg, i915_reg_t data_n_reg, + i915_reg_t link_m_reg, i915_reg_t link_n_reg); +bool intel_cpu_transcoder_has_m2_n2(struct drm_i915_private *dev_priv, + enum transcoder transcoder); +void intel_cpu_transcoder_set_m1_n1(struct intel_crtc *crtc, + enum transcoder cpu_transcoder, + const struct intel_link_m_n *m_n); +void intel_cpu_transcoder_set_m2_n2(struct intel_crtc *crtc, + enum transcoder cpu_transcoder, + const struct intel_link_m_n *m_n); +void intel_cpu_transcoder_get_m1_n1(struct intel_crtc *crtc, + enum transcoder cpu_transcoder, + struct intel_link_m_n *m_n); +void intel_cpu_transcoder_get_m2_n2(struct intel_crtc *crtc, + enum transcoder cpu_transcoder, + struct intel_link_m_n *m_n); void i9xx_crtc_clock_get(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config); int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n); -bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state); -void hsw_enable_ips(const struct intel_crtc_state *crtc_state); -void hsw_disable_ips(const struct intel_crtc_state *crtc_state); enum intel_display_power_domain intel_port_to_power_domain(enum port port); enum intel_display_power_domain intel_aux_power_domain(struct intel_digital_port *dig_port); diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index 572445299b04..ffe6822d7414 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -16,6 +16,7 @@ #include "intel_dp_mst.h" #include "intel_drrs.h" #include "intel_fbc.h" +#include "intel_fbdev.h" #include "intel_hdcp.h" #include "intel_hdmi.h" #include "intel_pm.h" @@ -78,7 +79,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) if (DISPLAY_VER(dev_priv) >= 9) /* no global SR status; inspect per-plane WM */; else if (HAS_PCH_SPLIT(dev_priv)) - sr_enabled = intel_de_read(dev_priv, WM1_LP_ILK) & WM1_LP_SR_EN; + sr_enabled = intel_de_read(dev_priv, WM1_LP_ILK) & WM_LP_ENABLE; else if (IS_I965GM(dev_priv) || IS_G4X(dev_priv) || IS_I945G(dev_priv) || IS_I945GM(dev_priv)) sr_enabled = intel_de_read(dev_priv, FW_BLC_SELF) & FW_BLC_SELF_EN; @@ -124,9 +125,8 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) struct drm_framebuffer *drm_fb; #ifdef CONFIG_DRM_FBDEV_EMULATION - if (dev_priv->fbdev && dev_priv->fbdev->helper.fb) { - fbdev_fb = to_intel_framebuffer(dev_priv->fbdev->helper.fb); - + fbdev_fb = intel_fbdev_framebuffer(dev_priv->fbdev); + if (fbdev_fb) { seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", fbdev_fb->base.width, fbdev_fb->base.height, @@ -474,8 +474,8 @@ static int i915_dmc_info(struct seq_file *m, void *unused) * reg for DC3CO debugging and validation, * but TGL DMC f/w is using DMC_DEBUG3 reg for DC3CO counter. */ - seq_printf(m, "DC3CO count: %d\n", - intel_de_read(dev_priv, DMC_DEBUG3)); + seq_printf(m, "DC3CO count: %d\n", intel_de_read(dev_priv, IS_DGFX(dev_priv) ? + DG1_DMC_DEBUG3 : TGL_DMC_DEBUG3)); } else { dc5_reg = IS_BROXTON(dev_priv) ? BXT_DMC_DC3_DC5_COUNT : SKL_DMC_DC3_DC5_COUNT; @@ -923,23 +923,23 @@ static void intel_crtc_info(struct seq_file *m, struct intel_crtc *crtc) yesno(crtc_state->uapi.active), DRM_MODE_ARG(&crtc_state->uapi.mode)); - if (crtc_state->hw.enable) { - seq_printf(m, "\thw: active=%s, adjusted_mode=" DRM_MODE_FMT "\n", - yesno(crtc_state->hw.active), - DRM_MODE_ARG(&crtc_state->hw.adjusted_mode)); + seq_printf(m, "\thw: enable=%s, active=%s\n", + yesno(crtc_state->hw.enable), yesno(crtc_state->hw.active)); + seq_printf(m, "\tadjusted_mode=" DRM_MODE_FMT "\n", + DRM_MODE_ARG(&crtc_state->hw.adjusted_mode)); + seq_printf(m, "\tpipe__mode=" DRM_MODE_FMT "\n", + DRM_MODE_ARG(&crtc_state->hw.pipe_mode)); - seq_printf(m, "\tpipe src size=%dx%d, dither=%s, bpp=%d\n", - crtc_state->pipe_src_w, crtc_state->pipe_src_h, - yesno(crtc_state->dither), crtc_state->pipe_bpp); + seq_printf(m, "\tpipe src size=%dx%d, dither=%s, bpp=%d\n", + crtc_state->pipe_src_w, crtc_state->pipe_src_h, + yesno(crtc_state->dither), crtc_state->pipe_bpp); - intel_scaler_info(m, crtc); - } + intel_scaler_info(m, crtc); if (crtc_state->bigjoiner) - seq_printf(m, "\tLinked to [CRTC:%d:%s] as a %s\n", - crtc_state->bigjoiner_linked_crtc->base.base.id, - crtc_state->bigjoiner_linked_crtc->base.name, - crtc_state->bigjoiner_slave ? "slave" : "master"); + seq_printf(m, "\tLinked to 0x%x pipes as a %s\n", + crtc_state->bigjoiner_pipes, + intel_crtc_is_bigjoiner_slave(crtc_state) ? "slave" : "master"); for_each_intel_encoder_mask(&dev_priv->drm, encoder, crtc_state->uapi.encoder_mask) @@ -1015,6 +1015,7 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused) seq_printf(m, " wrpll: 0x%08x\n", pll->state.hw_state.wrpll); seq_printf(m, " cfgcr0: 0x%08x\n", pll->state.hw_state.cfgcr0); seq_printf(m, " cfgcr1: 0x%08x\n", pll->state.hw_state.cfgcr1); + seq_printf(m, " div0: 0x%08x\n", pll->state.hw_state.div0); seq_printf(m, " mg_refclkin_ctl: 0x%08x\n", pll->state.hw_state.mg_refclkin_ctl); seq_printf(m, " mg_clktop2_coreclkctl1: 0x%08x\n", @@ -2402,6 +2403,9 @@ void intel_connector_debugfs_add(struct intel_connector *intel_connector) */ void intel_crtc_debugfs_add(struct drm_crtc *crtc) { - if (crtc->debugfs_entry) - crtc_updates_add(crtc); + if (!crtc->debugfs_entry) + return; + + crtc_updates_add(crtc); + intel_fbc_crtc_debugfs_add(to_intel_crtc(crtc)); } diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 05babdcf5f2e..9ebae7ac3235 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -7,6 +7,7 @@ #include "i915_irq.h" #include "intel_cdclk.h" #include "intel_combo_phy.h" +#include "intel_combo_phy_regs.h" #include "intel_crt.h" #include "intel_de.h" #include "intel_display_power.h" @@ -15,6 +16,7 @@ #include "intel_dpio_phy.h" #include "intel_dpll.h" #include "intel_hotplug.h" +#include "intel_mchbar_regs.h" #include "intel_pch_refclk.h" #include "intel_pcode.h" #include "intel_pm.h" @@ -682,9 +684,8 @@ static void icl_tc_cold_exit(struct drm_i915_private *i915) int ret, tries = 0; while (1) { - ret = sandybridge_pcode_write_timeout(i915, - ICL_PCODE_EXIT_TCCOLD, - 0, 250, 1); + ret = snb_pcode_write_timeout(i915, ICL_PCODE_EXIT_TCCOLD, 0, + 250, 1); if (ret != -EAGAIN || ++tries == 3) break; msleep(1); @@ -4052,8 +4053,7 @@ tgl_tc_cold_request(struct drm_i915_private *i915, bool block) * Spec states that we should timeout the request after 200us * but the function below will timeout after 500us */ - ret = sandybridge_pcode_read(i915, TGL_PCODE_TCCOLD, &low_val, - &high_val); + ret = snb_pcode_read(i915, TGL_PCODE_TCCOLD, &low_val, &high_val); if (ret == 0) { if (block && (low_val & TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED)) @@ -5468,8 +5468,7 @@ static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv) static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val) { if (IS_HASWELL(dev_priv)) { - if (sandybridge_pcode_write(dev_priv, - GEN6_PCODE_WRITE_D_COMP, val)) + if (snb_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP, val)) drm_dbg_kms(&dev_priv->drm, "Failed to write to D_COMP\n"); } else { @@ -5582,7 +5581,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); intel_update_cdclk(dev_priv); - intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK"); + intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK"); } /* @@ -6216,6 +6215,37 @@ void intel_power_domains_driver_remove(struct drm_i915_private *i915) } /** + * intel_power_domains_sanitize_state - sanitize power domains state + * @i915: i915 device instance + * + * Sanitize the power domains state during driver loading and system resume. + * The function will disable all display power wells that BIOS has enabled + * without a user for it (any user for a power well has taken a reference + * on it by the time this function is called, after the state of all the + * pipe, encoder, etc. HW resources have been sanitized). + */ +void intel_power_domains_sanitize_state(struct drm_i915_private *i915) +{ + struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_well *power_well; + + mutex_lock(&power_domains->lock); + + for_each_power_well_reverse(i915, power_well) { + if (power_well->desc->always_on || power_well->count || + !power_well->desc->ops->is_enabled(i915, power_well)) + continue; + + drm_dbg_kms(&i915->drm, + "BIOS left unused %s power well enabled, disabling it\n", + power_well->desc->name); + intel_power_well_disable(i915, power_well); + } + + mutex_unlock(&power_domains->lock); +} + +/** * intel_power_domains_enable - enable toggling of display power wells * @i915: i915 device instance * diff --git a/drivers/gpu/drm/i915/display/intel_display_power.h b/drivers/gpu/drm/i915/display/intel_display_power.h index 686d18eaa24c..f6d0e6e73c6d 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.h +++ b/drivers/gpu/drm/i915/display/intel_display_power.h @@ -7,7 +7,6 @@ #define __INTEL_DISPLAY_POWER_H__ #include "intel_runtime_pm.h" -#include "i915_reg.h" enum dpio_channel; enum dpio_phy; @@ -219,6 +218,7 @@ void intel_power_domains_disable(struct drm_i915_private *dev_priv); void intel_power_domains_suspend(struct drm_i915_private *dev_priv, enum i915_drm_suspend_mode); void intel_power_domains_resume(struct drm_i915_private *dev_priv); +void intel_power_domains_sanitize_state(struct drm_i915_private *dev_priv); void intel_display_power_suspend_late(struct drm_i915_private *i915); void intel_display_power_resume_early(struct drm_i915_private *i915); diff --git a/drivers/gpu/drm/i915/display/intel_display_trace.h b/drivers/gpu/drm/i915/display/intel_display_trace.h index 4043e1276383..f05f0f9b5103 100644 --- a/drivers/gpu/drm/i915/display/intel_display_trace.h +++ b/drivers/gpu/drm/i915/display/intel_display_trace.h @@ -13,6 +13,7 @@ #include <linux/tracepoint.h> #include "i915_drv.h" +#include "i915_irq.h" #include "intel_crtc.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index c9c6efadf8b4..776b3e6662f2 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -26,19 +26,17 @@ #ifndef __INTEL_DISPLAY_TYPES_H__ #define __INTEL_DISPLAY_TYPES_H__ -#include <linux/async.h> #include <linux/i2c.h> #include <linux/pm_qos.h> #include <linux/pwm.h> #include <linux/sched/clock.h> +#include <drm/dp/drm_dp_dual_mode_helper.h> +#include <drm/dp/drm_dp_mst_helper.h> #include <drm/drm_atomic.h> #include <drm/drm_crtc.h> -#include <drm/drm_dp_dual_mode_helper.h> -#include <drm/drm_dp_mst_helper.h> #include <drm/drm_dsc.h> #include <drm/drm_encoder.h> -#include <drm/drm_fb_helper.h> #include <drm/drm_fourcc.h> #include <drm/drm_probe_helper.h> #include <drm/drm_rect.h> @@ -145,25 +143,6 @@ struct intel_framebuffer { struct i915_address_space *dpt_vm; }; -struct intel_fbdev { - struct drm_fb_helper helper; - struct intel_framebuffer *fb; - struct i915_vma *vma; - unsigned long vma_flags; - async_cookie_t cookie; - int preferred_bpp; - - /* Whether or not fbdev hpd processing is temporarily suspended */ - bool hpd_suspended : 1; - /* Set when a hotplug was received while HPD processing was - * suspended - */ - bool hpd_waiting : 1; - - /* Protects hpd_suspended */ - struct mutex hpd_lock; -}; - enum intel_hotplug_state { INTEL_HOTPLUG_UNCHANGED, INTEL_HOTPLUG_CHANGED, @@ -969,6 +948,9 @@ struct intel_crtc_state { bool preload_luts; bool inherited; /* state inherited from BIOS? */ + /* Ask the hardware to actually async flip? */ + bool do_async_flip; + /* Pipe source size (ie. panel fitter input size) * All planes will be positioned inside this space, * and get clipped at the edges. */ @@ -1165,6 +1147,7 @@ struct intel_crtc_state { /* bitmask of actually visible planes (enum plane_id) */ u8 active_planes; + u8 scaled_planes; u8 nv12_planes; u8 c8_planes; @@ -1199,11 +1182,8 @@ struct intel_crtc_state { /* enable pipe big joiner? */ bool bigjoiner; - /* big joiner slave crtc? */ - bool bigjoiner_slave; - - /* linked crtc for bigjoiner, either slave or master */ - struct intel_crtc *bigjoiner_linked_crtc; + /* big joiner pipe bitmask */ + u8 bigjoiner_pipes; /* Display Stream compression state */ struct { @@ -1442,25 +1422,6 @@ struct intel_hdmi { }; struct intel_dp_mst_encoder; -/* - * enum link_m_n_set: - * When platform provides two set of M_N registers for dp, we can - * program them and switch between them incase of DRRS. - * But When only one such register is provided, we have to program the - * required divider value on that registers itself based on the DRRS state. - * - * M1_N1 : Program dp_m_n on M1_N1 registers - * dp_m2_n2 on M2_N2 registers (If supported) - * - * M2_N2 : Program dp_m2_n2 on M1_N1 registers - * M2_N2 registers are not supported - */ - -enum link_m_n_set { - /* Sets the m1_n1 and m2_n2 */ - M1_N1 = 0, - M2_N2 -}; struct intel_dp_compliance_data { unsigned long edid; diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c index a69b28d65a9b..7616a3906b9e 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc.c +++ b/drivers/gpu/drm/i915/display/intel_dmc.c @@ -43,9 +43,9 @@ __stringify(major) "_" \ __stringify(minor) ".bin" -#define GEN12_DMC_MAX_FW_SIZE ICL_DMC_MAX_FW_SIZE +#define DISPLAY_VER13_DMC_MAX_FW_SIZE 0x20000 -#define GEN13_DMC_MAX_FW_SIZE 0x20000 +#define DISPLAY_VER12_DMC_MAX_FW_SIZE ICL_DMC_MAX_FW_SIZE #define ADLP_DMC_PATH DMC_PATH(adlp, 2, 14) #define ADLP_DMC_VERSION_REQUIRED DMC_VERSION(2, 14) @@ -684,23 +684,23 @@ void intel_dmc_ucode_init(struct drm_i915_private *dev_priv) if (IS_ALDERLAKE_P(dev_priv)) { dmc->fw_path = ADLP_DMC_PATH; dmc->required_version = ADLP_DMC_VERSION_REQUIRED; - dmc->max_fw_size = GEN13_DMC_MAX_FW_SIZE; + dmc->max_fw_size = DISPLAY_VER13_DMC_MAX_FW_SIZE; } else if (IS_ALDERLAKE_S(dev_priv)) { dmc->fw_path = ADLS_DMC_PATH; dmc->required_version = ADLS_DMC_VERSION_REQUIRED; - dmc->max_fw_size = GEN12_DMC_MAX_FW_SIZE; + dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE; } else if (IS_DG1(dev_priv)) { dmc->fw_path = DG1_DMC_PATH; dmc->required_version = DG1_DMC_VERSION_REQUIRED; - dmc->max_fw_size = GEN12_DMC_MAX_FW_SIZE; + dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE; } else if (IS_ROCKETLAKE(dev_priv)) { dmc->fw_path = RKL_DMC_PATH; dmc->required_version = RKL_DMC_VERSION_REQUIRED; - dmc->max_fw_size = GEN12_DMC_MAX_FW_SIZE; + dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE; } else if (DISPLAY_VER(dev_priv) >= 12) { dmc->fw_path = TGL_DMC_PATH; dmc->required_version = TGL_DMC_VERSION_REQUIRED; - dmc->max_fw_size = GEN12_DMC_MAX_FW_SIZE; + dmc->max_fw_size = DISPLAY_VER12_DMC_MAX_FW_SIZE; } else if (DISPLAY_VER(dev_priv) == 11) { dmc->fw_path = ICL_DMC_PATH; dmc->required_version = ICL_DMC_VERSION_REQUIRED; diff --git a/drivers/gpu/drm/i915/display/intel_dmc.h b/drivers/gpu/drm/i915/display/intel_dmc.h index b20f3441ca60..7c590309a3a9 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc.h +++ b/drivers/gpu/drm/i915/display/intel_dmc.h @@ -6,7 +6,7 @@ #ifndef __INTEL_DMC_H__ #define __INTEL_DMC_H__ -#include "i915_reg.h" +#include "i915_reg_defs.h" #include "intel_wakeref.h" #include <linux/workqueue.h> diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index b5e2508db1cf..d667657e3606 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -36,7 +36,7 @@ #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> -#include <drm/drm_dp_helper.h> +#include <drm/dp/drm_dp_helper.h> #include <drm/drm_edid.h> #include <drm/drm_probe_helper.h> @@ -46,6 +46,7 @@ #include "intel_atomic.h" #include "intel_audio.h" #include "intel_backlight.h" +#include "intel_combo_phy_regs.h" #include "intel_connector.h" #include "intel_crtc.h" #include "intel_ddi.h" @@ -72,8 +73,6 @@ #include "intel_vdsc.h" #include "intel_vrr.h" -#define DP_DPRX_ESI_LEN 14 - /* DP DSC throughput values used for slice count calculations KPixels/s */ #define DP_DSC_PEAK_PIXEL_RATE 2720000 #define DP_DSC_MAX_ENC_THROUGHPUT_0 340000 @@ -705,7 +704,7 @@ static u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915, i915->max_cdclk_freq * 48 / intel_dp_mode_to_fec_clock(mode_clock); - DRM_DEBUG_KMS("Max big joiner bpp: %u\n", max_bpp_bigjoiner); + drm_dbg_kms(&i915->drm, "Max big joiner bpp: %u\n", max_bpp_bigjoiner); bits_per_pixel = min(bits_per_pixel, max_bpp_bigjoiner); } @@ -887,9 +886,8 @@ intel_dp_mode_valid_downstream(struct intel_connector *connector, return MODE_CLOCK_HIGH; /* Assume 8bpc for the DP++/HDMI/DVI TMDS clock check */ - tmds_clock = target_clock; - if (drm_mode_is_420_only(info, mode)) - tmds_clock /= 2; + tmds_clock = intel_hdmi_tmds_clock(target_clock, 8, + drm_mode_is_420_only(info, mode)); if (intel_dp->dfp.min_tmds_clock && tmds_clock < intel_dp->dfp.min_tmds_clock) @@ -1140,21 +1138,12 @@ static bool intel_dp_hdmi_ycbcr420(struct intel_dp *intel_dp, intel_dp->dfp.ycbcr_444_to_420); } -static int intel_dp_hdmi_tmds_clock(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, int bpc) -{ - int clock = crtc_state->hw.adjusted_mode.crtc_clock * bpc / 8; - - if (intel_dp_hdmi_ycbcr420(intel_dp, crtc_state)) - clock /= 2; - - return clock; -} - static bool intel_dp_hdmi_tmds_clock_valid(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, int bpc) { - int tmds_clock = intel_dp_hdmi_tmds_clock(intel_dp, crtc_state, bpc); + int clock = crtc_state->hw.adjusted_mode.crtc_clock; + int tmds_clock = intel_hdmi_tmds_clock(clock, bpc, + intel_dp_hdmi_ycbcr420(intel_dp, crtc_state)); if (intel_dp->dfp.min_tmds_clock && tmds_clock < intel_dp->dfp.min_tmds_clock) @@ -1167,14 +1156,13 @@ static bool intel_dp_hdmi_tmds_clock_valid(struct intel_dp *intel_dp, return true; } -static bool intel_dp_hdmi_deep_color_possible(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - int bpc) +static bool intel_dp_hdmi_bpc_possible(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + int bpc) { - return intel_hdmi_deep_color_possible(crtc_state, bpc, - intel_dp->has_hdmi_sink, - intel_dp_hdmi_ycbcr420(intel_dp, crtc_state)) && + return intel_hdmi_bpc_possible(crtc_state, bpc, intel_dp->has_hdmi_sink, + intel_dp_hdmi_ycbcr420(intel_dp, crtc_state)) && intel_dp_hdmi_tmds_clock_valid(intel_dp, crtc_state, bpc); } @@ -1192,7 +1180,7 @@ static int intel_dp_max_bpp(struct intel_dp *intel_dp, if (intel_dp->dfp.min_tmds_clock) { for (; bpc >= 10; bpc -= 2) { - if (intel_dp_hdmi_deep_color_possible(intel_dp, crtc_state, bpc)) + if (intel_dp_hdmi_bpc_possible(intel_dp, crtc_state, bpc)) break; } } @@ -1897,7 +1885,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, /* FIXME: abstract this better */ if (pipe_config->splitter.enable) - pipe_config->dp_m_n.gmch_m *= pipe_config->splitter.link_count; + pipe_config->dp_m_n.data_m *= pipe_config->splitter.link_count; if (!HAS_DDI(dev_priv)) g4x_dp_set_clock(encoder, pipe_config); @@ -2813,11 +2801,22 @@ intel_dp_configure_mst(struct intel_dp *intel_dp) } static bool -intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *sink_irq_vector) +intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *esi) +{ + return drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT_ESI, esi, 4) == 4; +} + +static bool intel_dp_ack_sink_irq_esi(struct intel_dp *intel_dp, u8 esi[4]) { - return drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT_ESI, - sink_irq_vector, DP_DPRX_ESI_LEN) == - DP_DPRX_ESI_LEN; + int retry; + + for (retry = 0; retry < 3; retry++) { + if (drm_dp_dpcd_write(&intel_dp->aux, DP_SINK_COUNT_ESI + 1, + &esi[1], 3) == 3) + return true; + } + + return false; } bool @@ -2909,7 +2908,8 @@ out: } static ssize_t -intel_dp_hdr_metadata_infoframe_sdp_pack(const struct hdmi_drm_infoframe *drm_infoframe, +intel_dp_hdr_metadata_infoframe_sdp_pack(struct drm_i915_private *i915, + const struct hdmi_drm_infoframe *drm_infoframe, struct dp_sdp *sdp, size_t size) { @@ -2925,12 +2925,12 @@ intel_dp_hdr_metadata_infoframe_sdp_pack(const struct hdmi_drm_infoframe *drm_in len = hdmi_drm_infoframe_pack_only(drm_infoframe, buf, sizeof(buf)); if (len < 0) { - DRM_DEBUG_KMS("buffer size is smaller than hdr metadata infoframe\n"); + drm_dbg_kms(&i915->drm, "buffer size is smaller than hdr metadata infoframe\n"); return -ENOSPC; } if (len != infoframe_size) { - DRM_DEBUG_KMS("wrong static hdr metadata size\n"); + drm_dbg_kms(&i915->drm, "wrong static hdr metadata size\n"); return -ENOSPC; } @@ -3003,7 +3003,8 @@ static void intel_write_dp_sdp(struct intel_encoder *encoder, sizeof(sdp)); break; case HDMI_PACKET_TYPE_GAMUT_METADATA: - len = intel_dp_hdr_metadata_infoframe_sdp_pack(&crtc_state->infoframes.drm.drm, + len = intel_dp_hdr_metadata_infoframe_sdp_pack(dev_priv, + &crtc_state->infoframes.drm.drm, &sdp, sizeof(sdp)); break; default: @@ -3411,22 +3412,22 @@ static void intel_dp_phy_pattern_update(struct intel_dp *intel_dp, switch (data->phy_pattern) { case DP_PHY_TEST_PATTERN_NONE: - DRM_DEBUG_KMS("Disable Phy Test Pattern\n"); + drm_dbg_kms(&dev_priv->drm, "Disable Phy Test Pattern\n"); intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), 0x0); break; case DP_PHY_TEST_PATTERN_D10_2: - DRM_DEBUG_KMS("Set D10.2 Phy Test Pattern\n"); + drm_dbg_kms(&dev_priv->drm, "Set D10.2 Phy Test Pattern\n"); intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), DDI_DP_COMP_CTL_ENABLE | DDI_DP_COMP_CTL_D10_2); break; case DP_PHY_TEST_PATTERN_ERROR_COUNT: - DRM_DEBUG_KMS("Set Error Count Phy Test Pattern\n"); + drm_dbg_kms(&dev_priv->drm, "Set Error Count Phy Test Pattern\n"); intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), DDI_DP_COMP_CTL_ENABLE | DDI_DP_COMP_CTL_SCRAMBLED_0); break; case DP_PHY_TEST_PATTERN_PRBS7: - DRM_DEBUG_KMS("Set PRBS7 Phy Test Pattern\n"); + drm_dbg_kms(&dev_priv->drm, "Set PRBS7 Phy Test Pattern\n"); intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), DDI_DP_COMP_CTL_ENABLE | DDI_DP_COMP_CTL_PRBS7); break; @@ -3436,7 +3437,8 @@ static void intel_dp_phy_pattern_update(struct intel_dp *intel_dp, * current firmware of DPR-100 could not set it, so hardcoding * now for complaince test. */ - DRM_DEBUG_KMS("Set 80Bit Custom Phy Test Pattern 0x3e0f83e0 0x0f83e0f8 0x0000f83e\n"); + drm_dbg_kms(&dev_priv->drm, + "Set 80Bit Custom Phy Test Pattern 0x3e0f83e0 0x0f83e0f8 0x0000f83e\n"); pattern_val = 0x3e0f83e0; intel_de_write(dev_priv, DDI_DP_COMP_PAT(pipe, 0), pattern_val); pattern_val = 0x0f83e0f8; @@ -3453,7 +3455,7 @@ static void intel_dp_phy_pattern_update(struct intel_dp *intel_dp, * current firmware of DPR-100 could not set it, so hardcoding * now for complaince test. */ - DRM_DEBUG_KMS("Set HBR2 compliance Phy Test Pattern\n"); + drm_dbg_kms(&dev_priv->drm, "Set HBR2 compliance Phy Test Pattern\n"); pattern_val = 0xFB; intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), DDI_DP_COMP_CTL_ENABLE | DDI_DP_COMP_CTL_HBR2 | @@ -3522,13 +3524,14 @@ intel_dp_autotest_phy_ddi_enable(struct intel_dp *intel_dp, static void intel_dp_process_phy_request(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct drm_dp_phy_test_params *data = &intel_dp->compliance.test_data.phytest; u8 link_status[DP_LINK_STATUS_SIZE]; if (drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, DP_PHY_DPRX, link_status) < 0) { - DRM_DEBUG_KMS("failed to get link status\n"); + drm_dbg_kms(&i915->drm, "failed to get link status\n"); return; } @@ -3553,11 +3556,12 @@ static void intel_dp_process_phy_request(struct intel_dp *intel_dp, static u8 intel_dp_autotest_phy_pattern(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct drm_dp_phy_test_params *data = &intel_dp->compliance.test_data.phytest; if (drm_dp_get_phy_test_pattern(&intel_dp->aux, data)) { - DRM_DEBUG_KMS("DP Phy Test pattern AUX read failure\n"); + drm_dbg_kms(&i915->drm, "DP Phy Test pattern AUX read failure\n"); return DP_TEST_NAK; } @@ -3614,15 +3618,63 @@ update_status: "Could not write test response to sink\n"); } +static bool intel_dp_link_ok(struct intel_dp *intel_dp, + u8 link_status[DP_LINK_STATUS_SIZE]) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + bool uhbr = intel_dp->link_rate >= 1000000; + bool ok; + + if (uhbr) + ok = drm_dp_128b132b_lane_channel_eq_done(link_status, + intel_dp->lane_count); + else + ok = drm_dp_channel_eq_ok(link_status, intel_dp->lane_count); + + if (ok) + return true; + + intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status); + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s] %s link not ok, retraining\n", + encoder->base.base.id, encoder->base.name, + uhbr ? "128b/132b" : "8b/10b"); + + return false; +} + static void -intel_dp_mst_hpd_irq(struct intel_dp *intel_dp, u8 *esi, bool *handled) +intel_dp_mst_hpd_irq(struct intel_dp *intel_dp, u8 *esi, u8 *ack) { - drm_dp_mst_hpd_irq(&intel_dp->mst_mgr, esi, handled); + bool handled = false; - if (esi[1] & DP_CP_IRQ) { - intel_hdcp_handle_cp_irq(intel_dp->attached_connector); - *handled = true; - } + drm_dp_mst_hpd_irq(&intel_dp->mst_mgr, esi, &handled); + if (handled) + ack[1] |= esi[1] & (DP_DOWN_REP_MSG_RDY | DP_UP_REQ_MSG_RDY); + + if (esi[1] & DP_CP_IRQ) { + intel_hdcp_handle_cp_irq(intel_dp->attached_connector); + ack[1] |= DP_CP_IRQ; + } +} + +static bool intel_dp_mst_link_status(struct intel_dp *intel_dp) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + u8 link_status[DP_LINK_STATUS_SIZE] = {}; + const size_t esi_link_status_size = DP_LINK_STATUS_SIZE - 2; + + if (drm_dp_dpcd_read(&intel_dp->aux, DP_LANE0_1_STATUS_ESI, link_status, + esi_link_status_size) != esi_link_status_size) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to read link status\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + return intel_dp_link_ok(intel_dp, link_status); } /** @@ -3647,20 +3699,8 @@ intel_dp_check_mst_status(struct intel_dp *intel_dp) drm_WARN_ON_ONCE(&i915->drm, intel_dp->active_mst_links < 0); for (;;) { - /* - * The +2 is because DP_DPRX_ESI_LEN is 14, but we then - * pass in "esi+10" to drm_dp_channel_eq_ok(), which - * takes a 6-byte array. So we actually need 16 bytes - * here. - * - * Somebody who knows what the limits actually are - * should check this, but for now this is at least - * harmless and avoids a valid compiler warning about - * using more of the array than we have allocated. - */ - u8 esi[DP_DPRX_ESI_LEN+2] = {}; - bool handled; - int retry; + u8 esi[4] = {}; + u8 ack[4] = {}; if (!intel_dp_get_sink_irq_esi(intel_dp, esi)) { drm_dbg_kms(&i915->drm, @@ -3670,30 +3710,22 @@ intel_dp_check_mst_status(struct intel_dp *intel_dp) break; } - /* check link status - esi[10] = 0x200c */ + drm_dbg_kms(&i915->drm, "DPRX ESI: %4ph\n", esi); + if (intel_dp->active_mst_links > 0 && link_ok && - !drm_dp_channel_eq_ok(&esi[10], intel_dp->lane_count)) { - drm_dbg_kms(&i915->drm, - "channel EQ not ok, retraining\n"); - link_ok = false; + esi[3] & LINK_STATUS_CHANGED) { + if (!intel_dp_mst_link_status(intel_dp)) + link_ok = false; + ack[3] |= LINK_STATUS_CHANGED; } - drm_dbg_kms(&i915->drm, "got esi %3ph\n", esi); + intel_dp_mst_hpd_irq(intel_dp, esi, ack); - intel_dp_mst_hpd_irq(intel_dp, esi, &handled); - - if (!handled) + if (!memchr_inv(ack, 0, sizeof(ack))) break; - for (retry = 0; retry < 3; retry++) { - int wret; - - wret = drm_dp_dpcd_write(&intel_dp->aux, - DP_SINK_COUNT_ESI+1, - &esi[1], 3); - if (wret == 3) - break; - } + if (!intel_dp_ack_sink_irq_esi(intel_dp, ack)) + drm_dbg_kms(&i915->drm, "Failed to ack ESI\n"); } return link_ok; @@ -3756,8 +3788,8 @@ intel_dp_needs_link_retrain(struct intel_dp *intel_dp) intel_dp->lane_count)) return false; - /* Retrain if Channel EQ or CR not ok */ - return !drm_dp_channel_eq_ok(link_status, intel_dp->lane_count); + /* Retrain if link not ok */ + return !intel_dp_link_ok(intel_dp, link_status); } static bool intel_dp_has_connector(struct intel_dp *intel_dp, @@ -3787,14 +3819,14 @@ static bool intel_dp_has_connector(struct intel_dp *intel_dp, static int intel_dp_prep_link_retrain(struct intel_dp *intel_dp, struct drm_modeset_acquire_ctx *ctx, - u32 *crtc_mask) + u8 *pipe_mask) { struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct drm_connector_list_iter conn_iter; struct intel_connector *connector; int ret = 0; - *crtc_mask = 0; + *pipe_mask = 0; if (!intel_dp_needs_link_retrain(intel_dp)) return 0; @@ -3828,12 +3860,12 @@ static int intel_dp_prep_link_retrain(struct intel_dp *intel_dp, !try_wait_for_completion(&conn_state->commit->hw_done)) continue; - *crtc_mask |= drm_crtc_mask(&crtc->base); + *pipe_mask |= BIT(crtc->pipe); } drm_connector_list_iter_end(&conn_iter); if (!intel_dp_needs_link_retrain(intel_dp)) - *crtc_mask = 0; + *pipe_mask = 0; return ret; } @@ -3852,7 +3884,7 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_crtc *crtc; - u32 crtc_mask; + u8 pipe_mask; int ret; if (!intel_dp_is_connected(intel_dp)) @@ -3863,17 +3895,17 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, if (ret) return ret; - ret = intel_dp_prep_link_retrain(intel_dp, ctx, &crtc_mask); + ret = intel_dp_prep_link_retrain(intel_dp, ctx, &pipe_mask); if (ret) return ret; - if (crtc_mask == 0) + if (pipe_mask == 0) return 0; drm_dbg_kms(&dev_priv->drm, "[ENCODER:%d:%s] retraining link\n", encoder->base.base.id, encoder->base.name); - for_each_intel_crtc_mask(&dev_priv->drm, crtc, crtc_mask) { + for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, crtc, pipe_mask) { const struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); @@ -3884,7 +3916,7 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, intel_crtc_pch_transcoder(crtc), false); } - for_each_intel_crtc_mask(&dev_priv->drm, crtc, crtc_mask) { + for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, crtc, pipe_mask) { const struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); @@ -3901,7 +3933,7 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, break; } - for_each_intel_crtc_mask(&dev_priv->drm, crtc, crtc_mask) { + for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, crtc, pipe_mask) { const struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); @@ -3919,14 +3951,14 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, static int intel_dp_prep_phy_test(struct intel_dp *intel_dp, struct drm_modeset_acquire_ctx *ctx, - u32 *crtc_mask) + u8 *pipe_mask) { struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct drm_connector_list_iter conn_iter; struct intel_connector *connector; int ret = 0; - *crtc_mask = 0; + *pipe_mask = 0; drm_connector_list_iter_begin(&i915->drm, &conn_iter); for_each_intel_connector_iter(connector, &conn_iter) { @@ -3957,7 +3989,7 @@ static int intel_dp_prep_phy_test(struct intel_dp *intel_dp, !try_wait_for_completion(&conn_state->commit->hw_done)) continue; - *crtc_mask |= drm_crtc_mask(&crtc->base); + *pipe_mask |= BIT(crtc->pipe); } drm_connector_list_iter_end(&conn_iter); @@ -3970,7 +4002,7 @@ static int intel_dp_do_phy_test(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_crtc *crtc; - u32 crtc_mask; + u8 pipe_mask; int ret; ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, @@ -3978,17 +4010,17 @@ static int intel_dp_do_phy_test(struct intel_encoder *encoder, if (ret) return ret; - ret = intel_dp_prep_phy_test(intel_dp, ctx, &crtc_mask); + ret = intel_dp_prep_phy_test(intel_dp, ctx, &pipe_mask); if (ret) return ret; - if (crtc_mask == 0) + if (pipe_mask == 0) return 0; drm_dbg_kms(&dev_priv->drm, "[ENCODER:%d:%s] PHY test\n", encoder->base.base.id, encoder->base.name); - for_each_intel_crtc_mask(&dev_priv->drm, crtc, crtc_mask) { + for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, crtc, pipe_mask) { const struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); @@ -4831,7 +4863,7 @@ intel_dp_hpd_pulse(struct intel_digital_port *dig_port, bool long_hpd) struct intel_dp *intel_dp = &dig_port->dp; if (dig_port->base.type == INTEL_OUTPUT_EDP && - (long_hpd || !intel_pps_have_power(intel_dp))) { + (long_hpd || !intel_pps_have_panel_power_or_vdd(intel_dp))) { /* * vdd off can generate a long/short pulse on eDP which * would require vdd on to handle it, and thus we @@ -4974,6 +5006,14 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, mutex_lock(&dev->mode_config.mutex); edid = drm_get_edid(connector, &intel_dp->aux.ddc); + if (!edid) { + /* Fallback to EDID from ACPI OpRegion, if any */ + edid = intel_opregion_get_edid(intel_connector); + if (edid) + drm_dbg_kms(&dev_priv->drm, + "[CONNECTOR:%d:%s] Using OpRegion EDID\n", + connector->base.id, connector->name); + } if (edid) { if (drm_add_edid_modes(connector, edid)) { drm_connector_update_edid_property(connector, edid); @@ -5048,8 +5088,8 @@ static void intel_dp_modeset_retry_work_fn(struct work_struct *work) intel_connector = container_of(work, typeof(*intel_connector), modeset_retry_work); connector = &intel_connector->base; - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, - connector->name); + drm_dbg_kms(connector->dev, "[CONNECTOR:%d:%s]\n", connector->base.id, + connector->name); /* Grab the locks before changing connector property*/ mutex_lock(&connector->dev->mode_config.mutex); diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h index b64145a3869a..d457e17bdc57 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.h +++ b/drivers/gpu/drm/i915/display/intel_dp.h @@ -8,8 +8,6 @@ #include <linux/types.h> -#include "i915_reg.h" - enum intel_output_format; enum pipe; enum port; diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c index 5fbb767fcd63..2bc119374555 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux.c +++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c @@ -10,7 +10,7 @@ #include "intel_pps.h" #include "intel_tc.h" -u32 intel_dp_pack_aux(const u8 *src, int src_bytes) +static u32 intel_dp_aux_pack(const u8 *src, int src_bytes) { int i; u32 v = 0; @@ -22,7 +22,7 @@ u32 intel_dp_pack_aux(const u8 *src, int src_bytes) return v; } -static void intel_dp_unpack_aux(u32 src, u8 *dst, int dst_bytes) +static void intel_dp_aux_unpack(u32 src, u8 *dst, int dst_bytes) { int i; @@ -267,7 +267,7 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp, for (i = 0; i < send_bytes; i += 4) intel_uncore_write(uncore, ch_data[i >> 2], - intel_dp_pack_aux(send + i, + intel_dp_aux_pack(send + i, send_bytes - i)); /* Send the command and wait for it to complete */ @@ -352,7 +352,7 @@ done: recv_bytes = recv_size; for (i = 0; i < recv_bytes; i += 4) - intel_dp_unpack_aux(intel_uncore_read(uncore, ch_data[i >> 2]), + intel_dp_aux_unpack(intel_uncore_read(uncore, ch_data[i >> 2]), recv + i, recv_bytes - i); ret = recv_bytes; diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.h b/drivers/gpu/drm/i915/display/intel_dp_aux.h index 4afbe76217b9..738577537bc7 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux.h +++ b/drivers/gpu/drm/i915/display/intel_dp_aux.h @@ -6,12 +6,8 @@ #ifndef __INTEL_DP_AUX_H__ #define __INTEL_DP_AUX_H__ -#include <linux/types.h> - struct intel_dp; -u32 intel_dp_pack_aux(const u8 *src, int src_bytes); - void intel_dp_aux_fini(struct intel_dp *intel_dp); void intel_dp_aux_init(struct intel_dp *intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c index 540a669e01dd..82d024dafe7b 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c @@ -6,8 +6,8 @@ * Sean Paul <seanpaul@chromium.org> */ -#include <drm/drm_dp_helper.h> -#include <drm/drm_dp_mst_helper.h> +#include <drm/dp/drm_dp_helper.h> +#include <drm/dp/drm_dp_mst_helper.h> #include <drm/drm_hdcp.h> #include <drm/drm_print.h> diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index 9451f336f28f..5d98773efd1b 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -712,7 +712,7 @@ static bool intel_dp_adjust_request_changed(const struct intel_crtc_state *crtc_ return false; } -static void +void intel_dp_dump_link_status(struct intel_dp *intel_dp, enum drm_dp_phy dp_phy, const u8 link_status[DP_LINK_STATUS_SIZE]) { @@ -996,6 +996,23 @@ static bool intel_dp_disable_dpcd_training_pattern(struct intel_dp *intel_dp, return drm_dp_dpcd_write(&intel_dp->aux, reg, &val, 1) == 1; } +static int +intel_dp_128b132b_intra_hop(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + u8 sink_status; + int ret; + + ret = drm_dp_dpcd_readb(&intel_dp->aux, DP_SINK_STATUS, &sink_status); + if (ret != 1) { + drm_dbg_kms(&i915->drm, "Failed to read sink status\n"); + return ret < 0 ? ret : -EIO; + } + + return sink_status & DP_INTRA_HOP_AUX_REPLY_INDICATION ? 1 : 0; +} + /** * intel_dp_stop_link_train - stop link training * @intel_dp: DP struct @@ -1015,11 +1032,21 @@ static bool intel_dp_disable_dpcd_training_pattern(struct intel_dp *intel_dp, void intel_dp_stop_link_train(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + intel_dp->link_trained = true; intel_dp_disable_dpcd_training_pattern(intel_dp, DP_PHY_DPRX); intel_dp_program_link_training_pattern(intel_dp, crtc_state, DP_PHY_DPRX, DP_TRAINING_PATTERN_DISABLE); + + if (intel_dp_is_uhbr(crtc_state) && + wait_for(intel_dp_128b132b_intra_hop(intel_dp, crtc_state) == 0, 500)) { + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s] 128b/132b intra-hop not clearing\n", + encoder->base.base.id, encoder->base.name); + } } static bool @@ -1083,8 +1110,6 @@ intel_dp_link_train_all_phys(struct intel_dp *intel_dp, bool ret = true; int i; - intel_dp_prepare_link_train(intel_dp, crtc_state); - for (i = lttpr_count - 1; i >= 0; i--) { enum drm_dp_phy dp_phy = DP_PHY_LTTPR(i); @@ -1104,6 +1129,272 @@ intel_dp_link_train_all_phys(struct intel_dp *intel_dp, return ret; } +/* + * 128b/132b DP LANEx_EQ_DONE Sequence (DP 2.0 E11 3.5.2.16.1) + */ +static bool +intel_dp_128b132b_lane_eq(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + u8 link_status[DP_LINK_STATUS_SIZE]; + int delay_us; + int try, max_tries = 20; + unsigned long deadline; + bool timeout = false; + + /* + * Reset signal levels. Start transmitting 128b/132b TPS1. + * + * Put DPRX and LTTPRs (if any) into intra-hop AUX mode by writing TPS1 + * in DP_TRAINING_PATTERN_SET. + */ + if (!intel_dp_reset_link_train(intel_dp, crtc_state, DP_PHY_DPRX, + DP_TRAINING_PATTERN_1)) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to start 128b/132b TPS1\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + delay_us = drm_dp_128b132b_read_aux_rd_interval(&intel_dp->aux); + + /* Read the initial TX FFE settings. */ + if (drm_dp_dpcd_read_link_status(&intel_dp->aux, link_status) < 0) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to read TX FFE presets\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + /* Update signal levels and training set as requested. */ + intel_dp_get_adjust_train(intel_dp, crtc_state, DP_PHY_DPRX, link_status); + if (!intel_dp_update_link_train(intel_dp, crtc_state, DP_PHY_DPRX)) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to set initial TX FFE settings\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + /* Start transmitting 128b/132b TPS2. */ + if (!intel_dp_set_link_train(intel_dp, crtc_state, DP_PHY_DPRX, + DP_TRAINING_PATTERN_2)) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to start 128b/132b TPS2\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + /* Time budget for the LANEx_EQ_DONE Sequence */ + deadline = jiffies + msecs_to_jiffies_timeout(400); + + for (try = 0; try < max_tries; try++) { + usleep_range(delay_us, 2 * delay_us); + + /* + * The delay may get updated. The transmitter shall read the + * delay before link status during link training. + */ + delay_us = drm_dp_128b132b_read_aux_rd_interval(&intel_dp->aux); + + if (drm_dp_dpcd_read_link_status(&intel_dp->aux, link_status) < 0) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to read link status\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + if (drm_dp_128b132b_link_training_failed(link_status)) { + intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status); + drm_err(&i915->drm, + "[ENCODER:%d:%s] Downstream link training failure\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + if (drm_dp_128b132b_lane_channel_eq_done(link_status, crtc_state->lane_count)) { + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s] Lane channel eq done\n", + encoder->base.base.id, encoder->base.name); + break; + } + + if (timeout) { + intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status); + drm_err(&i915->drm, + "[ENCODER:%d:%s] Lane channel eq timeout\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + if (time_after(jiffies, deadline)) + timeout = true; /* try one last time after deadline */ + + /* Update signal levels and training set as requested. */ + intel_dp_get_adjust_train(intel_dp, crtc_state, DP_PHY_DPRX, link_status); + if (!intel_dp_update_link_train(intel_dp, crtc_state, DP_PHY_DPRX)) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to update TX FFE settings\n", + encoder->base.base.id, encoder->base.name); + return false; + } + } + + if (try == max_tries) { + intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status); + drm_err(&i915->drm, + "[ENCODER:%d:%s] Max loop count reached\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + for (;;) { + if (time_after(jiffies, deadline)) + timeout = true; /* try one last time after deadline */ + + if (drm_dp_dpcd_read_link_status(&intel_dp->aux, link_status) < 0) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to read link status\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + if (drm_dp_128b132b_link_training_failed(link_status)) { + intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status); + drm_err(&i915->drm, + "[ENCODER:%d:%s] Downstream link training failure\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + if (drm_dp_128b132b_eq_interlane_align_done(link_status)) { + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s] Interlane align done\n", + encoder->base.base.id, encoder->base.name); + break; + } + + if (timeout) { + intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status); + drm_err(&i915->drm, + "[ENCODER:%d:%s] Interlane align timeout\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + usleep_range(2000, 3000); + } + + return true; +} + +/* + * 128b/132b DP LANEx_CDS_DONE Sequence (DP 2.0 E11 3.5.2.16.2) + */ +static bool +intel_dp_128b132b_lane_cds(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + int lttpr_count) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + u8 link_status[DP_LINK_STATUS_SIZE]; + unsigned long deadline; + + if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TRAINING_PATTERN_SET, + DP_TRAINING_PATTERN_2_CDS) != 1) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to start 128b/132b TPS2 CDS\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + /* Time budget for the LANEx_CDS_DONE Sequence */ + deadline = jiffies + msecs_to_jiffies_timeout((lttpr_count + 1) * 20); + + for (;;) { + bool timeout = false; + + if (time_after(jiffies, deadline)) + timeout = true; /* try one last time after deadline */ + + usleep_range(2000, 3000); + + if (drm_dp_dpcd_read_link_status(&intel_dp->aux, link_status) < 0) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] Failed to read link status\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + if (drm_dp_128b132b_eq_interlane_align_done(link_status) && + drm_dp_128b132b_cds_interlane_align_done(link_status) && + drm_dp_128b132b_lane_symbol_locked(link_status, crtc_state->lane_count)) { + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s] CDS interlane align done\n", + encoder->base.base.id, encoder->base.name); + break; + } + + if (drm_dp_128b132b_link_training_failed(link_status)) { + intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status); + drm_err(&i915->drm, + "[ENCODER:%d:%s] Downstream link training failure\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + if (timeout) { + intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status); + drm_err(&i915->drm, + "[ENCODER:%d:%s] CDS timeout\n", + encoder->base.base.id, encoder->base.name); + return false; + } + } + + /* FIXME: Should DP_TRAINING_PATTERN_DISABLE be written first? */ + if (intel_dp->set_idle_link_train) + intel_dp->set_idle_link_train(intel_dp, crtc_state); + + return true; +} + +/* + * 128b/132b link training sequence. (DP 2.0 E11 SCR on link training.) + */ +static bool +intel_dp_128b132b_link_train(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + int lttpr_count) +{ + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct intel_connector *connector = intel_dp->attached_connector; + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + bool passed = false; + + if (wait_for(intel_dp_128b132b_intra_hop(intel_dp, crtc_state) == 0, 500)) { + drm_err(&i915->drm, + "[ENCODER:%d:%s] 128b/132b intra-hop not clear\n", + encoder->base.base.id, encoder->base.name); + return false; + } + + if (intel_dp_128b132b_lane_eq(intel_dp, crtc_state) && + intel_dp_128b132b_lane_cds(intel_dp, crtc_state, lttpr_count)) + passed = true; + + drm_dbg_kms(&i915->drm, + "[CONNECTOR:%d:%s][ENCODER:%d:%s] 128b/132b Link Training %s at link rate = %d, lane count = %d\n", + connector->base.base.id, connector->base.name, + encoder->base.base.id, encoder->base.name, + passed ? "passed" : "failed", + crtc_state->port_clock, crtc_state->lane_count); + + return passed; +} + /** * intel_dp_start_link_train - start link training * @intel_dp: DP struct @@ -1117,6 +1408,7 @@ intel_dp_link_train_all_phys(struct intel_dp *intel_dp, void intel_dp_start_link_train(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { + bool passed; /* * TODO: Reiniting LTTPRs here won't be needed once proper connector * HW state readout is added. @@ -1127,6 +1419,13 @@ void intel_dp_start_link_train(struct intel_dp *intel_dp, /* Still continue with enabling the port and link training. */ lttpr_count = 0; - if (!intel_dp_link_train_all_phys(intel_dp, crtc_state, lttpr_count)) + intel_dp_prepare_link_train(intel_dp, crtc_state); + + if (intel_dp_is_uhbr(crtc_state)) + passed = intel_dp_128b132b_link_train(intel_dp, crtc_state, lttpr_count); + else + passed = intel_dp_link_train_all_phys(intel_dp, crtc_state, lttpr_count); + + if (!passed) intel_dp_schedule_fallback_link_training(intel_dp, crtc_state); } diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.h b/drivers/gpu/drm/i915/display/intel_dp_link_training.h index 6a3a7b37349a..dc1556b46b85 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.h +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.h @@ -6,7 +6,7 @@ #ifndef __INTEL_DP_LINK_TRAINING_H__ #define __INTEL_DP_LINK_TRAINING_H__ -#include <drm/drm_dp_helper.h> +#include <drm/dp/drm_dp_helper.h> struct intel_crtc_state; struct intel_dp; @@ -29,6 +29,10 @@ void intel_dp_start_link_train(struct intel_dp *intel_dp, void intel_dp_stop_link_train(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state); +void +intel_dp_dump_link_status(struct intel_dp *intel_dp, enum drm_dp_phy dp_phy, + const u8 link_status[DP_LINK_STATUS_SIZE]); + /* Get the TPSx symbol type of the value programmed to DP_TRAINING_PATTERN_SET */ static inline u8 intel_dp_training_pattern_symbol(u8 pattern) { diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index b8bc7d397c81..e30e698aa684 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -99,6 +99,29 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, return 0; } +static int intel_dp_mst_update_slots(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state, + struct drm_connector_state *conn_state) +{ + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); + struct intel_dp *intel_dp = &intel_mst->primary->dp; + struct drm_dp_mst_topology_mgr *mgr = &intel_dp->mst_mgr; + struct drm_dp_mst_topology_state *topology_state; + u8 link_coding_cap = intel_dp_is_uhbr(crtc_state) ? + DP_CAP_ANSI_128B132B : DP_CAP_ANSI_8B10B; + + topology_state = drm_atomic_get_mst_topology_state(conn_state->state, mgr); + if (IS_ERR(topology_state)) { + drm_dbg_kms(&i915->drm, "slot update failed\n"); + return PTR_ERR(topology_state); + } + + drm_dp_mst_update_slots(topology_state, link_coding_cap); + + return 0; +} + static int intel_dp_mst_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) @@ -155,6 +178,10 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder, if (ret) return ret; + ret = intel_dp_mst_update_slots(encoder, pipe_config, conn_state); + if (ret) + return ret; + pipe_config->limited_color_range = intel_dp_limited_color_range(pipe_config, conn_state); @@ -357,6 +384,7 @@ static void intel_mst_disable_dp(struct intel_atomic_state *state, struct intel_connector *connector = to_intel_connector(old_conn_state->connector); struct drm_i915_private *i915 = to_i915(connector->base.dev); + int start_slot = intel_dp_is_uhbr(old_crtc_state) ? 0 : 1; int ret; drm_dbg_kms(&i915->drm, "active links %d\n", @@ -366,7 +394,7 @@ static void intel_mst_disable_dp(struct intel_atomic_state *state, drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, connector->port); - ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr, 1); + ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr, start_slot); if (ret) { drm_dbg_kms(&i915->drm, "failed to update payload %d\n", ret); } @@ -475,6 +503,7 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state, struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_connector *connector = to_intel_connector(conn_state->connector); + int start_slot = intel_dp_is_uhbr(pipe_config) ? 0 : 1; int ret; bool first_mst_stream; @@ -509,7 +538,7 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state, intel_dp->active_mst_links++; - ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr, 1); + ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr, start_slot); /* * Before Gen 12 this is not done as part of @@ -522,8 +551,6 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state, intel_ddi_enable_pipe_clock(encoder, pipe_config); intel_ddi_set_dp_msa(pipe_config, conn_state); - - intel_dp_set_m_n(pipe_config, M1_N1); } static void intel_mst_enable_dp(struct intel_atomic_state *state, diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c index 1ce0c171f4fb..14f5ffe27d05 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll.c +++ b/drivers/gpu/drm/i915/display/intel_dpll.c @@ -16,6 +16,10 @@ #include "intel_snps_phy.h" #include "vlv_sideband.h" +struct intel_dpll_funcs { + int (*crtc_compute_clock)(struct intel_crtc_state *crtc_state); +}; + struct intel_limit { struct { int min, max; @@ -1400,6 +1404,14 @@ static const struct intel_dpll_funcs i8xx_dpll_funcs = { .crtc_compute_clock = i8xx_crtc_compute_clock, }; +int intel_dpll_crtc_compute_clock(struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + + return i915->dpll_funcs->crtc_compute_clock(crtc_state); +} + void intel_dpll_init_clock_hook(struct drm_i915_private *dev_priv) { diff --git a/drivers/gpu/drm/i915/display/intel_dpll.h b/drivers/gpu/drm/i915/display/intel_dpll.h index 1af0ac43cca4..69b06a9e473e 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll.h +++ b/drivers/gpu/drm/i915/display/intel_dpll.h @@ -15,6 +15,7 @@ struct intel_crtc_state; enum pipe; void intel_dpll_init_clock_hook(struct drm_i915_private *dev_priv); +int intel_dpll_crtc_compute_clock(struct intel_crtc_state *crtc_state); int vlv_calc_dpll_params(int refclk, struct dpll *clock); int pnv_calc_dpll_params(int refclk, struct dpll *clock); int i9xx_calc_dpll_params(int refclk, struct dpll *clock); diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index fc8fda77483a..569903d47aea 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -28,6 +28,7 @@ #include "intel_dpll_mgr.h" #include "intel_pch_refclk.h" #include "intel_tc.h" +#include "intel_tc_phy_regs.h" /** * DOC: Display PLLs @@ -49,6 +50,41 @@ * commit phase. */ +/* platform specific hooks for managing DPLLs */ +struct intel_shared_dpll_funcs { + /* + * Hook for enabling the pll, called from intel_enable_shared_dpll() if + * the pll is not already enabled. + */ + void (*enable)(struct drm_i915_private *i915, + struct intel_shared_dpll *pll); + + /* + * Hook for disabling the pll, called from intel_disable_shared_dpll() + * only when it is safe to disable the pll, i.e., there are no more + * tracked users for it. + */ + void (*disable)(struct drm_i915_private *i915, + struct intel_shared_dpll *pll); + + /* + * Hook for reading the values currently programmed to the DPLL + * registers. This is used for initial hw state readout and state + * verification after a mode set. + */ + bool (*get_hw_state)(struct drm_i915_private *i915, + struct intel_shared_dpll *pll, + struct intel_dpll_hw_state *hw_state); + + /* + * Hook for calculating the pll's output frequency based on its passed + * in state. + */ + int (*get_freq)(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state); +}; + struct intel_dpll_mgr { const struct dpll_info *dpll_info; @@ -2712,6 +2748,9 @@ static void icl_calc_dpll_state(struct drm_i915_private *i915, pll_state->cfgcr1 |= TGL_DPLL_CFGCR1_CFSELOVRD_NORMAL_XTAL; else pll_state->cfgcr1 |= DPLL_CFGCR1_CENTRAL_FREQ_8400; + + if (i915->vbt.override_afc_startup) + pll_state->div0 = TGL_DPLL0_DIV0_AFC_STARTUP(i915->vbt.override_afc_startup_val); } static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc, @@ -2913,6 +2952,11 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, DKL_PLL_DIV0_PROP_COEFF(prop_coeff) | DKL_PLL_DIV0_FBPREDIV(m1div) | DKL_PLL_DIV0_FBDIV_INT(m2div_int); + if (dev_priv->vbt.override_afc_startup) { + u8 val = dev_priv->vbt.override_afc_startup_val; + + pll_state->mg_pll_div0 |= DKL_PLL_DIV0_AFC_STARTUP(val); + } pll_state->mg_pll_div1 = DKL_PLL_DIV1_IREF_TRIM(iref_trim) | DKL_PLL_DIV1_TDC_TARGET_CNT(tdc_targetcnt); @@ -3412,10 +3456,10 @@ static bool dkl_pll_get_hw_state(struct drm_i915_private *dev_priv, MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK; hw_state->mg_pll_div0 = intel_de_read(dev_priv, DKL_PLL_DIV0(tc_port)); - hw_state->mg_pll_div0 &= (DKL_PLL_DIV0_INTEG_COEFF_MASK | - DKL_PLL_DIV0_PROP_COEFF_MASK | - DKL_PLL_DIV0_FBPREDIV_MASK | - DKL_PLL_DIV0_FBDIV_INT_MASK); + val = DKL_PLL_DIV0_MASK; + if (dev_priv->vbt.override_afc_startup) + val |= DKL_PLL_DIV0_AFC_STARTUP_MASK; + hw_state->mg_pll_div0 &= val; hw_state->mg_pll_div1 = intel_de_read(dev_priv, DKL_PLL_DIV1(tc_port)); hw_state->mg_pll_div1 &= (DKL_PLL_DIV1_IREF_TRIM_MASK | @@ -3477,6 +3521,10 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv, TGL_DPLL_CFGCR0(id)); hw_state->cfgcr1 = intel_de_read(dev_priv, TGL_DPLL_CFGCR1(id)); + if (dev_priv->vbt.override_afc_startup) { + hw_state->div0 = intel_de_read(dev_priv, TGL_DPLL0_DIV0(id)); + hw_state->div0 &= TGL_DPLL0_DIV0_AFC_STARTUP_MASK; + } } else { if (IS_JSL_EHL(dev_priv) && id == DPLL_ID_EHL_DPLL4) { hw_state->cfgcr0 = intel_de_read(dev_priv, @@ -3518,7 +3566,7 @@ static void icl_dpll_write(struct drm_i915_private *dev_priv, { struct intel_dpll_hw_state *hw_state = &pll->state.hw_state; const enum intel_dpll_id id = pll->info->id; - i915_reg_t cfgcr0_reg, cfgcr1_reg; + i915_reg_t cfgcr0_reg, cfgcr1_reg, div0_reg = INVALID_MMIO_REG; if (IS_ALDERLAKE_S(dev_priv)) { cfgcr0_reg = ADLS_DPLL_CFGCR0(id); @@ -3532,6 +3580,7 @@ static void icl_dpll_write(struct drm_i915_private *dev_priv, } else if (DISPLAY_VER(dev_priv) >= 12) { cfgcr0_reg = TGL_DPLL_CFGCR0(id); cfgcr1_reg = TGL_DPLL_CFGCR1(id); + div0_reg = TGL_DPLL0_DIV0(id); } else { if (IS_JSL_EHL(dev_priv) && id == DPLL_ID_EHL_DPLL4) { cfgcr0_reg = ICL_DPLL_CFGCR0(4); @@ -3544,6 +3593,12 @@ static void icl_dpll_write(struct drm_i915_private *dev_priv, intel_de_write(dev_priv, cfgcr0_reg, hw_state->cfgcr0); intel_de_write(dev_priv, cfgcr1_reg, hw_state->cfgcr1); + drm_WARN_ON_ONCE(&dev_priv->drm, dev_priv->vbt.override_afc_startup && + !i915_mmio_reg_valid(div0_reg)); + if (dev_priv->vbt.override_afc_startup && + i915_mmio_reg_valid(div0_reg)) + intel_de_rmw(dev_priv, div0_reg, TGL_DPLL0_DIV0_AFC_STARTUP_MASK, + hw_state->div0); intel_de_posting_read(dev_priv, cfgcr1_reg); } @@ -3631,13 +3686,11 @@ static void dkl_pll_write(struct drm_i915_private *dev_priv, val |= hw_state->mg_clktop2_hsclkctl; intel_de_write(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port), val); - val = intel_de_read(dev_priv, DKL_PLL_DIV0(tc_port)); - val &= ~(DKL_PLL_DIV0_INTEG_COEFF_MASK | - DKL_PLL_DIV0_PROP_COEFF_MASK | - DKL_PLL_DIV0_FBPREDIV_MASK | - DKL_PLL_DIV0_FBDIV_INT_MASK); - val |= hw_state->mg_pll_div0; - intel_de_write(dev_priv, DKL_PLL_DIV0(tc_port), val); + val = DKL_PLL_DIV0_MASK; + if (dev_priv->vbt.override_afc_startup) + val |= DKL_PLL_DIV0_AFC_STARTUP_MASK; + intel_de_rmw(dev_priv, DKL_PLL_DIV0(tc_port), val, + hw_state->mg_pll_div0); val = intel_de_read(dev_priv, DKL_PLL_DIV1(tc_port)); val &= ~(DKL_PLL_DIV1_IREF_TRIM_MASK | @@ -3876,13 +3929,14 @@ static void icl_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state) { drm_dbg_kms(&dev_priv->drm, - "dpll_hw_state: cfgcr0: 0x%x, cfgcr1: 0x%x, " + "dpll_hw_state: cfgcr0: 0x%x, cfgcr1: 0x%x, div0: 0x%x, " "mg_refclkin_ctl: 0x%x, hg_clktop2_coreclkctl1: 0x%x, " "mg_clktop2_hsclkctl: 0x%x, mg_pll_div0: 0x%x, " "mg_pll_div2: 0x%x, mg_pll_lf: 0x%x, " "mg_pll_frac_lock: 0x%x, mg_pll_ssc: 0x%x, " "mg_pll_bias: 0x%x, mg_pll_tdc_coldst_bias: 0x%x\n", hw_state->cfgcr0, hw_state->cfgcr1, + hw_state->div0, hw_state->mg_refclkin_ctl, hw_state->mg_clktop2_coreclkctl1, hw_state->mg_clktop2_hsclkctl, diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h index ef2889753807..ba2fdfce1579 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h @@ -44,6 +44,7 @@ struct intel_crtc; struct intel_crtc_state; struct intel_encoder; struct intel_shared_dpll; +struct intel_shared_dpll_funcs; /** * enum intel_dpll_id - possible DPLL ids @@ -207,6 +208,9 @@ struct intel_dpll_hw_state { /* icl */ u32 cfgcr0; + /* tgl */ + u32 div0; + /* bxt */ u32 ebb0, ebb4, pll0, pll1, pll2, pll3, pll6, pll8, pll9, pll10, pcsdw12; @@ -252,51 +256,6 @@ struct intel_shared_dpll_state { }; /** - * struct intel_shared_dpll_funcs - platform specific hooks for managing DPLLs - */ -struct intel_shared_dpll_funcs { - /** - * @enable: - * - * Hook for enabling the pll, called from intel_enable_shared_dpll() - * if the pll is not already enabled. - */ - void (*enable)(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll); - - /** - * @disable: - * - * Hook for disabling the pll, called from intel_disable_shared_dpll() - * only when it is safe to disable the pll, i.e., there are no more - * tracked users for it. - */ - void (*disable)(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll); - - /** - * @get_hw_state: - * - * Hook for reading the values currently programmed to the DPLL - * registers. This is used for initial hw state readout and state - * verification after a mode set. - */ - bool (*get_hw_state)(struct drm_i915_private *dev_priv, - struct intel_shared_dpll *pll, - struct intel_dpll_hw_state *hw_state); - - /** - * @get_freq: - * - * Hook for calculating the pll's output frequency based on its - * passed in state. - */ - int (*get_freq)(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll, - const struct intel_dpll_hw_state *pll_state); -}; - -/** * struct dpll_info - display PLL platform specific info */ struct dpll_info { diff --git a/drivers/gpu/drm/i915/display/intel_dpt.c b/drivers/gpu/drm/i915/display/intel_dpt.c index 8f674745e7e0..05dd7dba3a5c 100644 --- a/drivers/gpu/drm/i915/display/intel_dpt.c +++ b/drivers/gpu/drm/i915/display/intel_dpt.c @@ -3,11 +3,13 @@ * Copyright © 2021 Intel Corporation */ +#include "gem/i915_gem_domain.h" +#include "gt/gen8_ppgtt.h" + #include "i915_drv.h" #include "intel_display_types.h" #include "intel_dpt.h" #include "intel_fb.h" -#include "gt/gen8_ppgtt.h" struct i915_dpt { struct i915_address_space vm; @@ -48,7 +50,7 @@ static void dpt_insert_page(struct i915_address_space *vm, } static void dpt_insert_entries(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level level, u32 flags) { @@ -64,8 +66,8 @@ static void dpt_insert_entries(struct i915_address_space *vm, * not to allow the user to override access to a read only page. */ - i = vma->node.start / I915_GTT_PAGE_SIZE; - for_each_sgt_daddr(addr, sgt_iter, vma->pages) + i = vma_res->start / I915_GTT_PAGE_SIZE; + for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages) gen8_set_pte(&base[i++], pte_encode | addr); } @@ -76,35 +78,38 @@ static void dpt_clear_range(struct i915_address_space *vm, static void dpt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags) { - struct drm_i915_gem_object *obj = vma->obj; u32 pte_flags; + if (vma_res->bound_flags) + return; + /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ pte_flags = 0; - if (vma->vm->has_read_only && i915_gem_object_is_readonly(obj)) + if (vm->has_read_only && vma_res->bi.readonly) pte_flags |= PTE_READ_ONLY; - if (i915_gem_object_is_lmem(obj)) + if (vma_res->bi.lmem) pte_flags |= PTE_LM; - vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags); + vm->insert_entries(vm, vma_res, cache_level, pte_flags); - vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; + vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; /* * Without aliasing PPGTT there's no difference between * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally * upgrade to both bound if we bind either to avoid double-binding. */ - atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags); + vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; } -static void dpt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) +static void dpt_unbind_vma(struct i915_address_space *vm, + struct i915_vma_resource *vma_res) { - vm->clear_range(vm, vma->node.start, vma->size); + vm->clear_range(vm, vma_res->start, vma_res->vma_size); } static void dpt_cleanup(struct i915_address_space *vm) @@ -250,7 +255,11 @@ intel_dpt_create(struct intel_framebuffer *fb) if (IS_ERR(dpt_obj)) return ERR_CAST(dpt_obj); - ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); + ret = i915_gem_object_lock_interruptible(dpt_obj, NULL); + if (!ret) { + ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); + i915_gem_object_unlock(dpt_obj); + } if (ret) { i915_gem_object_put(dpt_obj); return ERR_PTR(ret); diff --git a/drivers/gpu/drm/i915/display/intel_drrs.c b/drivers/gpu/drm/i915/display/intel_drrs.c index 3ff149df4a77..fa715b8ea310 100644 --- a/drivers/gpu/drm/i915/display/intel_drrs.c +++ b/drivers/gpu/drm/i915/display/intel_drrs.c @@ -47,17 +47,13 @@ * requested by userspace. */ -void -intel_drrs_compute_config(struct intel_dp *intel_dp, - struct intel_crtc_state *pipe_config, - int output_bpp, bool constant_n) +static bool can_enable_drrs(struct intel_connector *connector, + const struct intel_crtc_state *pipe_config) { - struct intel_connector *intel_connector = intel_dp->attached_connector; - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - int pixel_clock; + const struct drm_i915_private *i915 = to_i915(connector->base.dev); if (pipe_config->vrr.enable) - return; + return false; /* * DRRS and PSR can't be enable together, so giving preference to PSR @@ -66,15 +62,30 @@ intel_drrs_compute_config(struct intel_dp *intel_dp, * after intel_psr_compute_config(). */ if (pipe_config->has_psr) - return; + return false; + + return connector->panel.downclock_mode && + i915->drrs.type == SEAMLESS_DRRS_SUPPORT; +} - if (!intel_connector->panel.downclock_mode || - dev_priv->drrs.type != SEAMLESS_DRRS_SUPPORT) +void +intel_drrs_compute_config(struct intel_dp *intel_dp, + struct intel_crtc_state *pipe_config, + int output_bpp, bool constant_n) +{ + struct intel_connector *connector = intel_dp->attached_connector; + struct drm_i915_private *i915 = to_i915(connector->base.dev); + int pixel_clock; + + if (!can_enable_drrs(connector, pipe_config)) { + if (intel_cpu_transcoder_has_m2_n2(i915, pipe_config->cpu_transcoder)) + intel_zero_m_n(&pipe_config->dp_m2_n2); return; + } pipe_config->has_drrs = true; - pixel_clock = intel_connector->panel.downclock_mode->clock; + pixel_clock = connector->panel.downclock_mode->clock; if (pipe_config->splitter.enable) pixel_clock /= pipe_config->splitter.link_count; @@ -84,7 +95,42 @@ intel_drrs_compute_config(struct intel_dp *intel_dp, /* FIXME: abstract this better */ if (pipe_config->splitter.enable) - pipe_config->dp_m2_n2.gmch_m *= pipe_config->splitter.link_count; + pipe_config->dp_m2_n2.data_m *= pipe_config->splitter.link_count; +} + +static void +intel_drrs_set_refresh_rate_pipeconf(const struct intel_crtc_state *crtc_state, + enum drrs_refresh_rate_type refresh_type) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; + u32 val, bit; + + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + bit = PIPECONF_EDP_RR_MODE_SWITCH_VLV; + else + bit = PIPECONF_EDP_RR_MODE_SWITCH; + + val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder)); + + if (refresh_type == DRRS_LOW_RR) + val |= bit; + else + val &= ~bit; + + intel_de_write(dev_priv, PIPECONF(cpu_transcoder), val); +} + +static void +intel_drrs_set_refresh_rate_m_n(const struct intel_crtc_state *crtc_state, + enum drrs_refresh_rate_type refresh_type) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + intel_cpu_transcoder_set_m1_n1(crtc, crtc_state->cpu_transcoder, + refresh_type == DRRS_LOW_RR ? + &crtc_state->dp_m2_n2 : &crtc_state->dp_m_n); } static void intel_drrs_set_state(struct drm_i915_private *dev_priv, @@ -120,37 +166,10 @@ static void intel_drrs_set_state(struct drm_i915_private *dev_priv, return; } - if (DISPLAY_VER(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) { - switch (refresh_type) { - case DRRS_HIGH_RR: - intel_dp_set_m_n(crtc_state, M1_N1); - break; - case DRRS_LOW_RR: - intel_dp_set_m_n(crtc_state, M2_N2); - break; - case DRRS_MAX_RR: - default: - drm_err(&dev_priv->drm, - "Unsupported refreshrate type\n"); - } - } else if (DISPLAY_VER(dev_priv) > 6) { - i915_reg_t reg = PIPECONF(crtc_state->cpu_transcoder); - u32 val; - - val = intel_de_read(dev_priv, reg); - if (refresh_type == DRRS_LOW_RR) { - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - val |= PIPECONF_EDP_RR_MODE_SWITCH_VLV; - else - val |= PIPECONF_EDP_RR_MODE_SWITCH; - } else { - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - val &= ~PIPECONF_EDP_RR_MODE_SWITCH_VLV; - else - val &= ~PIPECONF_EDP_RR_MODE_SWITCH; - } - intel_de_write(dev_priv, reg, val); - } + if (DISPLAY_VER(dev_priv) >= 8 && !IS_CHERRYVIEW(dev_priv)) + intel_drrs_set_refresh_rate_m_n(crtc_state, refresh_type); + else if (DISPLAY_VER(dev_priv) > 6) + intel_drrs_set_refresh_rate_pipeconf(crtc_state, refresh_type); dev_priv->drrs.refresh_rate_type = refresh_type; diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c index 83a69a4a4fea..b34a67309976 100644 --- a/drivers/gpu/drm/i915/display/intel_dsb.c +++ b/drivers/gpu/drm/i915/display/intel_dsb.c @@ -4,6 +4,8 @@ * */ +#include "gem/i915_gem_internal.h" + #include "i915_drv.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_dsb.h b/drivers/gpu/drm/i915/display/intel_dsb.h index 654a11f24b80..6cb9c580cdca 100644 --- a/drivers/gpu/drm/i915/display/intel_dsb.h +++ b/drivers/gpu/drm/i915/display/intel_dsb.h @@ -8,7 +8,7 @@ #include <linux/types.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" struct intel_crtc_state; struct i915_vma; diff --git a/drivers/gpu/drm/i915/display/intel_dsi.h b/drivers/gpu/drm/i915/display/intel_dsi.h index a3a906cb097e..eafef0a87fea 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi.h +++ b/drivers/gpu/drm/i915/display/intel_dsi.h @@ -79,8 +79,8 @@ struct intel_dsi { */ enum mipi_dsi_pixel_format pixel_format; - /* video mode format for MIPI_VIDEO_MODE_FORMAT register */ - u32 video_mode_format; + /* NON_BURST_SYNC_PULSE, NON_BURST_SYNC_EVENTS, or BURST_MODE */ + int video_mode; /* eot for MIPI_EOT_DISABLE register */ u8 eotp_pkt; diff --git a/drivers/gpu/drm/i915/display/intel_dsi_vbt.c b/drivers/gpu/drm/i915/display/intel_dsi_vbt.c index 0da91849efde..6b4a27372c82 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi_vbt.c +++ b/drivers/gpu/drm/i915/display/intel_dsi_vbt.c @@ -39,10 +39,12 @@ #include <video/mipi_display.h> #include "i915_drv.h" +#include "i915_reg.h" #include "intel_display_types.h" #include "intel_dsi.h" #include "intel_dsi_vbt.h" #include "vlv_dsi.h" +#include "vlv_dsi_regs.h" #include "vlv_sideband.h" #define MIPI_TRANSFER_MODE_SHIFT 0 @@ -426,24 +428,16 @@ static void i2c_acpi_find_adapter(struct intel_dsi *intel_dsi, const u16 slave_addr) { struct drm_device *drm_dev = intel_dsi->base.base.dev; - struct device *dev = drm_dev->dev; - struct acpi_device *acpi_dev; - struct list_head resource_list; - struct i2c_adapter_lookup lookup; - - acpi_dev = ACPI_COMPANION(dev); - if (acpi_dev) { - memset(&lookup, 0, sizeof(lookup)); - lookup.slave_addr = slave_addr; - lookup.intel_dsi = intel_dsi; - lookup.dev_handle = acpi_device_handle(acpi_dev); - - INIT_LIST_HEAD(&resource_list); - acpi_dev_get_resources(acpi_dev, &resource_list, - i2c_adapter_lookup, - &lookup); - acpi_dev_free_resource_list(&resource_list); - } + struct acpi_device *adev = ACPI_COMPANION(drm_dev->dev); + struct i2c_adapter_lookup lookup = { + .slave_addr = slave_addr, + .intel_dsi = intel_dsi, + .dev_handle = acpi_device_handle(adev), + }; + LIST_HEAD(resource_list); + + acpi_dev_get_resources(adev, &resource_list, i2c_adapter_lookup, &lookup); + acpi_dev_free_resource_list(&resource_list); } #else static inline void i2c_acpi_find_adapter(struct intel_dsi *intel_dsi, @@ -682,11 +676,11 @@ void intel_dsi_log_params(struct intel_dsi *intel_dsi) drm_dbg_kms(&i915->drm, "Lane count %d\n", intel_dsi->lane_count); drm_dbg_kms(&i915->drm, "DPHY param reg 0x%x\n", intel_dsi->dphy_reg); drm_dbg_kms(&i915->drm, "Video mode format %s\n", - intel_dsi->video_mode_format == VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE ? + intel_dsi->video_mode == NON_BURST_SYNC_PULSE ? "non-burst with sync pulse" : - intel_dsi->video_mode_format == VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS ? + intel_dsi->video_mode == NON_BURST_SYNC_EVENTS ? "non-burst with sync events" : - intel_dsi->video_mode_format == VIDEO_MODE_BURST ? + intel_dsi->video_mode == BURST_MODE ? "burst" : "<unknown>"); drm_dbg_kms(&i915->drm, "Burst mode ratio %d\n", intel_dsi->burst_mode_ratio); @@ -746,7 +740,7 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id) intel_dsi->dual_link = mipi_config->dual_link; intel_dsi->pixel_overlap = mipi_config->pixel_overlap; intel_dsi->operation_mode = mipi_config->is_cmd_mode; - intel_dsi->video_mode_format = mipi_config->video_transfer_mode; + intel_dsi->video_mode = mipi_config->video_transfer_mode; intel_dsi->escape_clk_div = mipi_config->byte_clk_sel; intel_dsi->lp_rx_timeout = mipi_config->lp_rx_timeout; intel_dsi->hs_tx_timeout = mipi_config->hs_tx_timeout; @@ -777,7 +771,7 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id) * Target ddr frequency from VBT / non burst ddr freq * multiply by 100 to preserve remainder */ - if (intel_dsi->video_mode_format == VIDEO_MODE_BURST) { + if (intel_dsi->video_mode == BURST_MODE) { if (mipi_config->target_burst_mode_freq) { u32 bitrate = intel_dsi_bitrate(intel_dsi); diff --git a/drivers/gpu/drm/i915/display/intel_dvo_dev.h b/drivers/gpu/drm/i915/display/intel_dvo_dev.h index 94a6ae1e0292..d96c3cc46e50 100644 --- a/drivers/gpu/drm/i915/display/intel_dvo_dev.h +++ b/drivers/gpu/drm/i915/display/intel_dvo_dev.h @@ -27,7 +27,7 @@ #include <drm/drm_crtc.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" struct intel_dvo_device { const char *name; diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.c b/drivers/gpu/drm/i915/display/intel_fb_pin.c index 31c15e5fca95..a307b4993bcf 100644 --- a/drivers/gpu/drm/i915/display/intel_fb_pin.c +++ b/drivers/gpu/drm/i915/display/intel_fb_pin.c @@ -7,6 +7,7 @@ * DOC: display pinning helpers */ +#include "gem/i915_gem_domain.h" #include "gem/i915_gem_object.h" #include "i915_drv.h" @@ -36,7 +37,11 @@ intel_pin_fb_obj_dpt(struct drm_framebuffer *fb, atomic_inc(&dev_priv->gpu_error.pending_fb_pin); - ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); + ret = i915_gem_object_lock_interruptible(obj, NULL); + if (!ret) { + ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); + i915_gem_object_unlock(obj); + } if (ret) { vma = ERR_PTR(ret); goto err; @@ -47,7 +52,7 @@ intel_pin_fb_obj_dpt(struct drm_framebuffer *fb, goto err; if (i915_vma_misplaced(vma, 0, alignment, 0)) { - ret = i915_vma_unbind(vma); + ret = i915_vma_unbind_unlocked(vma); if (ret) { vma = ERR_PTR(ret); goto err; diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index 957feeccff3f..87f4af3fd523 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -49,6 +49,14 @@ #include "intel_fbc.h" #include "intel_frontbuffer.h" +#define for_each_fbc_id(__dev_priv, __fbc_id) \ + for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \ + for_each_if(INTEL_INFO(__dev_priv)->display.fbc_mask & BIT(__fbc_id)) + +#define for_each_intel_fbc(__dev_priv, __fbc, __fbc_id) \ + for_each_fbc_id((__dev_priv), (__fbc_id)) \ + for_each_if((__fbc) = (__dev_priv)->fbc[(__fbc_id)]) + struct intel_fbc_funcs { void (*activate)(struct intel_fbc *fbc); void (*deactivate)(struct intel_fbc *fbc); @@ -85,6 +93,8 @@ struct intel_fbc { struct drm_mm_node compressed_fb; struct drm_mm_node compressed_llb; + enum intel_fbc_id id; + u8 limit; bool false_color; @@ -454,10 +464,10 @@ static void ilk_fbc_activate(struct intel_fbc *fbc) struct intel_fbc_state *fbc_state = &fbc->state; struct drm_i915_private *i915 = fbc->i915; - intel_de_write(i915, ILK_DPFC_FENCE_YOFF, + intel_de_write(i915, ILK_DPFC_FENCE_YOFF(fbc->id), fbc_state->fence_y_offset); - intel_de_write(i915, ILK_DPFC_CONTROL, + intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), DPFC_CTL_EN | g4x_dpfc_ctl(fbc)); } @@ -467,28 +477,28 @@ static void ilk_fbc_deactivate(struct intel_fbc *fbc) u32 dpfc_ctl; /* Disable compression */ - dpfc_ctl = intel_de_read(i915, ILK_DPFC_CONTROL); + dpfc_ctl = intel_de_read(i915, ILK_DPFC_CONTROL(fbc->id)); if (dpfc_ctl & DPFC_CTL_EN) { dpfc_ctl &= ~DPFC_CTL_EN; - intel_de_write(i915, ILK_DPFC_CONTROL, dpfc_ctl); + intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl); } } static bool ilk_fbc_is_active(struct intel_fbc *fbc) { - return intel_de_read(fbc->i915, ILK_DPFC_CONTROL) & DPFC_CTL_EN; + return intel_de_read(fbc->i915, ILK_DPFC_CONTROL(fbc->id)) & DPFC_CTL_EN; } static bool ilk_fbc_is_compressing(struct intel_fbc *fbc) { - return intel_de_read(fbc->i915, ILK_DPFC_STATUS) & DPFC_COMP_SEG_MASK; + return intel_de_read(fbc->i915, ILK_DPFC_STATUS(fbc->id)) & DPFC_COMP_SEG_MASK; } static void ilk_fbc_program_cfb(struct intel_fbc *fbc) { struct drm_i915_private *i915 = fbc->i915; - intel_de_write(i915, ILK_DPFC_CB_BASE, fbc->compressed_fb.start); + intel_de_write(i915, ILK_DPFC_CB_BASE(fbc->id), fbc->compressed_fb.start); } static const struct intel_fbc_funcs ilk_fbc_funcs = { @@ -524,8 +534,8 @@ static void snb_fbc_nuke(struct intel_fbc *fbc) { struct drm_i915_private *i915 = fbc->i915; - intel_de_write(i915, MSG_FBC_REND_STATE, FBC_REND_NUKE); - intel_de_posting_read(i915, MSG_FBC_REND_STATE); + intel_de_write(i915, MSG_FBC_REND_STATE(fbc->id), FBC_REND_NUKE); + intel_de_posting_read(i915, MSG_FBC_REND_STATE(fbc->id)); } static const struct intel_fbc_funcs snb_fbc_funcs = { @@ -547,7 +557,7 @@ static void glk_fbc_program_cfb_stride(struct intel_fbc *fbc) val |= FBC_STRIDE_OVERRIDE | FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit); - intel_de_write(i915, GLK_FBC_STRIDE, val); + intel_de_write(i915, GLK_FBC_STRIDE(fbc->id), val); } static void skl_fbc_program_cfb_stride(struct intel_fbc *fbc) @@ -595,22 +605,22 @@ static void ivb_fbc_activate(struct intel_fbc *fbc) else if (DISPLAY_VER(i915) == 9) skl_fbc_program_cfb_stride(fbc); - if (i915->ggtt.num_fences) + if (to_gt(i915)->ggtt->num_fences) snb_fbc_program_fence(fbc); - intel_de_write(i915, ILK_DPFC_CONTROL, + intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), DPFC_CTL_EN | ivb_dpfc_ctl(fbc)); } static bool ivb_fbc_is_compressing(struct intel_fbc *fbc) { - return intel_de_read(fbc->i915, ILK_DPFC_STATUS2) & DPFC_COMP_SEG_MASK_IVB; + return intel_de_read(fbc->i915, ILK_DPFC_STATUS2(fbc->id)) & DPFC_COMP_SEG_MASK_IVB; } static void ivb_fbc_set_false_color(struct intel_fbc *fbc, bool enable) { - intel_de_rmw(fbc->i915, ILK_DPFC_CONTROL, + intel_de_rmw(fbc->i915, ILK_DPFC_CONTROL(fbc->id), DPFC_CTL_FALSE_COLOR, enable ? DPFC_CTL_FALSE_COLOR : 0); } @@ -810,16 +820,16 @@ static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc) void intel_fbc_cleanup(struct drm_i915_private *i915) { - struct intel_fbc *fbc = i915->fbc; - - if (!fbc) - return; + struct intel_fbc *fbc; + enum intel_fbc_id fbc_id; - mutex_lock(&fbc->lock); - __intel_fbc_cleanup_cfb(fbc); - mutex_unlock(&fbc->lock); + for_each_intel_fbc(i915, fbc, fbc_id) { + mutex_lock(&fbc->lock); + __intel_fbc_cleanup_cfb(fbc); + mutex_unlock(&fbc->lock); - kfree(fbc); + kfree(fbc); + } } static bool stride_is_valid(const struct intel_plane_state *plane_state) @@ -1306,15 +1316,10 @@ static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc) return fbc->possible_framebuffer_bits; } -void intel_fbc_invalidate(struct drm_i915_private *i915, - unsigned int frontbuffer_bits, - enum fb_op_origin origin) +static void __intel_fbc_invalidate(struct intel_fbc *fbc, + unsigned int frontbuffer_bits, + enum fb_op_origin origin) { - struct intel_fbc *fbc = i915->fbc; - - if (!fbc) - return; - if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE) return; @@ -1328,14 +1333,22 @@ void intel_fbc_invalidate(struct drm_i915_private *i915, mutex_unlock(&fbc->lock); } -void intel_fbc_flush(struct drm_i915_private *i915, - unsigned int frontbuffer_bits, enum fb_op_origin origin) +void intel_fbc_invalidate(struct drm_i915_private *i915, + unsigned int frontbuffer_bits, + enum fb_op_origin origin) { - struct intel_fbc *fbc = i915->fbc; + struct intel_fbc *fbc; + enum intel_fbc_id fbc_id; - if (!fbc) - return; + for_each_intel_fbc(i915, fbc, fbc_id) + __intel_fbc_invalidate(fbc, frontbuffer_bits, origin); + +} +static void __intel_fbc_flush(struct intel_fbc *fbc, + unsigned int frontbuffer_bits, + enum fb_op_origin origin) +{ mutex_lock(&fbc->lock); fbc->busy_bits &= ~frontbuffer_bits; @@ -1355,6 +1368,17 @@ out: mutex_unlock(&fbc->lock); } +void intel_fbc_flush(struct drm_i915_private *i915, + unsigned int frontbuffer_bits, + enum fb_op_origin origin) +{ + struct intel_fbc *fbc; + enum intel_fbc_id fbc_id; + + for_each_intel_fbc(i915, fbc, fbc_id) + __intel_fbc_flush(fbc, frontbuffer_bits, origin); +} + int intel_fbc_atomic_check(struct intel_atomic_state *state) { struct intel_plane_state *plane_state; @@ -1484,15 +1508,15 @@ void intel_fbc_update(struct intel_atomic_state *state, */ void intel_fbc_global_disable(struct drm_i915_private *i915) { - struct intel_fbc *fbc = i915->fbc; - - if (!fbc) - return; + struct intel_fbc *fbc; + enum intel_fbc_id fbc_id; - mutex_lock(&fbc->lock); - if (fbc->state.plane) - __intel_fbc_disable(fbc); - mutex_unlock(&fbc->lock); + for_each_intel_fbc(i915, fbc, fbc_id) { + mutex_lock(&fbc->lock); + if (fbc->state.plane) + __intel_fbc_disable(fbc); + mutex_unlock(&fbc->lock); + } } static void intel_fbc_underrun_work_fn(struct work_struct *work) @@ -1517,19 +1541,9 @@ out: mutex_unlock(&fbc->lock); } -/* - * intel_fbc_reset_underrun - reset FBC fifo underrun status. - * @i915: the i915 device - * - * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we - * want to re-enable FBC after an underrun to increase test coverage. - */ -void intel_fbc_reset_underrun(struct drm_i915_private *i915) +static void __intel_fbc_reset_underrun(struct intel_fbc *fbc) { - struct intel_fbc *fbc = i915->fbc; - - if (!fbc) - return; + struct drm_i915_private *i915 = fbc->i915; cancel_work_sync(&fbc->underrun_work); @@ -1545,6 +1559,38 @@ void intel_fbc_reset_underrun(struct drm_i915_private *i915) mutex_unlock(&fbc->lock); } +/* + * intel_fbc_reset_underrun - reset FBC fifo underrun status. + * @i915: the i915 device + * + * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we + * want to re-enable FBC after an underrun to increase test coverage. + */ +void intel_fbc_reset_underrun(struct drm_i915_private *i915) +{ + struct intel_fbc *fbc; + enum intel_fbc_id fbc_id; + + for_each_intel_fbc(i915, fbc, fbc_id) + __intel_fbc_reset_underrun(fbc); +} + +static void __intel_fbc_handle_fifo_underrun_irq(struct intel_fbc *fbc) +{ + /* + * There's no guarantee that underrun_detected won't be set to true + * right after this check and before the work is scheduled, but that's + * not a problem since we'll check it again under the work function + * while FBC is locked. This check here is just to prevent us from + * unnecessarily scheduling the work, and it relies on the fact that we + * never switch underrun_detect back to false after it's true. + */ + if (READ_ONCE(fbc->underrun_detected)) + return; + + schedule_work(&fbc->underrun_work); +} + /** * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun * @i915: i915 device @@ -1561,21 +1607,11 @@ void intel_fbc_reset_underrun(struct drm_i915_private *i915) */ void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *i915) { - struct intel_fbc *fbc = i915->fbc; - - if (!fbc) - return; - - /* There's no guarantee that underrun_detected won't be set to true - * right after this check and before the work is scheduled, but that's - * not a problem since we'll check it again under the work function - * while FBC is locked. This check here is just to prevent us from - * unnecessarily scheduling the work, and it relies on the fact that we - * never switch underrun_detect back to false after it's true. */ - if (READ_ONCE(fbc->underrun_detected)) - return; + struct intel_fbc *fbc; + enum intel_fbc_id fbc_id; - schedule_work(&fbc->underrun_work); + for_each_intel_fbc(i915, fbc, fbc_id) + __intel_fbc_handle_fifo_underrun_irq(fbc); } /* @@ -1623,7 +1659,8 @@ void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane) fbc->possible_framebuffer_bits |= plane->frontbuffer_bit; } -static struct intel_fbc *intel_fbc_create(struct drm_i915_private *i915) +static struct intel_fbc *intel_fbc_create(struct drm_i915_private *i915, + enum intel_fbc_id fbc_id) { struct intel_fbc *fbc; @@ -1631,6 +1668,7 @@ static struct intel_fbc *intel_fbc_create(struct drm_i915_private *i915) if (!fbc) return NULL; + fbc->id = fbc_id; fbc->i915 = i915; INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn); mutex_init(&fbc->lock); @@ -1659,32 +1697,35 @@ static struct intel_fbc *intel_fbc_create(struct drm_i915_private *i915) */ void intel_fbc_init(struct drm_i915_private *i915) { - struct intel_fbc *fbc; + enum intel_fbc_id fbc_id; if (!drm_mm_initialized(&i915->mm.stolen)) - mkwrite_device_info(i915)->display.has_fbc = false; + mkwrite_device_info(i915)->display.fbc_mask = 0; if (need_fbc_vtd_wa(i915)) - mkwrite_device_info(i915)->display.has_fbc = false; + mkwrite_device_info(i915)->display.fbc_mask = 0; i915->params.enable_fbc = intel_sanitize_fbc_option(i915); drm_dbg_kms(&i915->drm, "Sanitized enable_fbc value: %d\n", i915->params.enable_fbc); - if (!HAS_FBC(i915)) - return; + for_each_fbc_id(i915, fbc_id) { + struct intel_fbc *fbc; - fbc = intel_fbc_create(i915); - if (!fbc) - return; + fbc = intel_fbc_create(i915, fbc_id); + if (!fbc) + continue; - /* We still don't have any sort of hardware state readout for FBC, so - * deactivate it in case the BIOS activated it to make sure software - * matches the hardware state. */ - if (intel_fbc_hw_is_active(fbc)) - intel_fbc_hw_deactivate(fbc); + /* + * We still don't have any sort of hardware state readout + * for FBC, so deactivate it in case the BIOS activated it + * to make sure software matches the hardware state. + */ + if (intel_fbc_hw_is_active(fbc)) + intel_fbc_hw_deactivate(fbc); - i915->fbc = fbc; + i915->fbc[fbc->id] = fbc; + } } static int intel_fbc_debugfs_status_show(struct seq_file *m, void *unused) @@ -1760,25 +1801,32 @@ DEFINE_SIMPLE_ATTRIBUTE(intel_fbc_debugfs_false_color_fops, intel_fbc_debugfs_false_color_set, "%llu\n"); -static void intel_fbc_debugfs_add(struct intel_fbc *fbc) +static void intel_fbc_debugfs_add(struct intel_fbc *fbc, + struct dentry *parent) { - struct drm_i915_private *i915 = fbc->i915; - struct drm_minor *minor = i915->drm.primary; - - debugfs_create_file("i915_fbc_status", 0444, - minor->debugfs_root, fbc, - &intel_fbc_debugfs_status_fops); + debugfs_create_file("i915_fbc_status", 0444, parent, + fbc, &intel_fbc_debugfs_status_fops); if (fbc->funcs->set_false_color) - debugfs_create_file("i915_fbc_false_color", 0644, - minor->debugfs_root, fbc, - &intel_fbc_debugfs_false_color_fops); + debugfs_create_file("i915_fbc_false_color", 0644, parent, + fbc, &intel_fbc_debugfs_false_color_fops); } +void intel_fbc_crtc_debugfs_add(struct intel_crtc *crtc) +{ + struct intel_plane *plane = to_intel_plane(crtc->base.primary); + + if (plane->fbc) + intel_fbc_debugfs_add(plane->fbc, crtc->base.debugfs_entry); +} + +/* FIXME: remove this once igt is on board with per-crtc stuff */ void intel_fbc_debugfs_register(struct drm_i915_private *i915) { - struct intel_fbc *fbc = i915->fbc; + struct drm_minor *minor = i915->drm.primary; + struct intel_fbc *fbc; + fbc = i915->fbc[INTEL_FBC_A]; if (fbc) - intel_fbc_debugfs_add(fbc); + intel_fbc_debugfs_add(fbc, minor->debugfs_root); } diff --git a/drivers/gpu/drm/i915/display/intel_fbc.h b/drivers/gpu/drm/i915/display/intel_fbc.h index 07ad0411fcc3..8c5a7339a27f 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.h +++ b/drivers/gpu/drm/i915/display/intel_fbc.h @@ -17,6 +17,12 @@ struct intel_fbc; struct intel_plane; struct intel_plane_state; +enum intel_fbc_id { + INTEL_FBC_A, + + I915_MAX_FBCS, +}; + int intel_fbc_atomic_check(struct intel_atomic_state *state); bool intel_fbc_pre_update(struct intel_atomic_state *state, struct intel_crtc *crtc); @@ -36,6 +42,7 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv, void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane); void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *i915); void intel_fbc_reset_underrun(struct drm_i915_private *i915); +void intel_fbc_crtc_debugfs_add(struct intel_crtc *crtc); void intel_fbc_debugfs_register(struct drm_i915_private *i915); #endif /* __INTEL_FBC_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index adc3a81be9f7..2cd62a187df3 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -50,6 +50,23 @@ #include "intel_fbdev.h" #include "intel_frontbuffer.h" +struct intel_fbdev { + struct drm_fb_helper helper; + struct intel_framebuffer *fb; + struct i915_vma *vma; + unsigned long vma_flags; + async_cookie_t cookie; + int preferred_bpp; + + /* Whether or not fbdev hpd processing is temporarily suspended */ + bool hpd_suspended: 1; + /* Set when a hotplug was received while HPD processing was suspended */ + bool hpd_waiting: 1; + + /* Protects hpd_suspended */ + struct mutex hpd_lock; +}; + static struct intel_frontbuffer *to_frontbuffer(struct intel_fbdev *ifbdev) { return ifbdev->fb->frontbuffer; @@ -180,7 +197,7 @@ static int intelfb_create(struct drm_fb_helper *helper, struct drm_device *dev = helper->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; + struct i915_ggtt *ggtt = to_gt(dev_priv)->ggtt; const struct i915_ggtt_view view = { .type = I915_GGTT_VIEW_NORMAL, }; @@ -248,7 +265,7 @@ static int intelfb_create(struct drm_fb_helper *helper, struct intel_memory_region *mem = obj->mm.region; info->apertures->ranges[0].base = mem->io_start; - info->apertures->ranges[0].size = mem->total; + info->apertures->ranges[0].size = mem->io_size; /* Use fbdev's framebuffer from lmem for discrete */ info->fix.smem_start = @@ -680,3 +697,11 @@ void intel_fbdev_restore_mode(struct drm_device *dev) if (drm_fb_helper_restore_fbdev_mode_unlocked(&ifbdev->helper) == 0) intel_fbdev_invalidate(ifbdev); } + +struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev) +{ + if (!fbdev || !fbdev->helper.fb) + return NULL; + + return to_intel_framebuffer(fbdev->helper.fb); +} diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.h b/drivers/gpu/drm/i915/display/intel_fbdev.h index de7c84250eb5..0e95e9472fa3 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.h +++ b/drivers/gpu/drm/i915/display/intel_fbdev.h @@ -10,6 +10,8 @@ struct drm_device; struct drm_i915_private; +struct intel_fbdev; +struct intel_framebuffer; #ifdef CONFIG_DRM_FBDEV_EMULATION int intel_fbdev_init(struct drm_device *dev); @@ -19,6 +21,7 @@ void intel_fbdev_fini(struct drm_i915_private *dev_priv); void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous); void intel_fbdev_output_poll_changed(struct drm_device *dev); void intel_fbdev_restore_mode(struct drm_device *dev); +struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev); #else static inline int intel_fbdev_init(struct drm_device *dev) { @@ -48,6 +51,10 @@ static inline void intel_fbdev_output_poll_changed(struct drm_device *dev) static inline void intel_fbdev_restore_mode(struct drm_device *dev) { } +static inline struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev) +{ + return NULL; +} #endif #endif /* __INTEL_FBDEV_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c index 3d6e22923601..4e4b43669b14 100644 --- a/drivers/gpu/drm/i915/display/intel_fdi.c +++ b/drivers/gpu/drm/i915/display/intel_fdi.c @@ -10,6 +10,11 @@ #include "intel_display_types.h" #include "intel_fdi.h" +struct intel_fdi_funcs { + void (*fdi_link_train)(struct intel_crtc *crtc, + const struct intel_crtc_state *crtc_state); +}; + static void assert_fdi_tx(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c index 3b8b84177085..2fad03250661 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.c +++ b/drivers/gpu/drm/i915/display/intel_gmbus.c @@ -98,11 +98,21 @@ static const struct gmbus_pin gmbus_pins_dg1[] = { [GMBUS_PIN_4_CNP] = { "dpd", GPIOE }, }; +static const struct gmbus_pin gmbus_pins_dg2[] = { + [GMBUS_PIN_1_BXT] = { "dpa", GPIOB }, + [GMBUS_PIN_2_BXT] = { "dpb", GPIOC }, + [GMBUS_PIN_3_BXT] = { "dpc", GPIOD }, + [GMBUS_PIN_4_CNP] = { "dpd", GPIOE }, + [GMBUS_PIN_9_TC1_ICP] = { "tc1", GPIOJ }, +}; + /* pin is expected to be valid */ static const struct gmbus_pin *get_gmbus_pin(struct drm_i915_private *dev_priv, unsigned int pin) { - if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1) + if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG2) + return &gmbus_pins_dg2[pin]; + else if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1) return &gmbus_pins_dg1[pin]; else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) return &gmbus_pins_icp[pin]; @@ -123,7 +133,9 @@ bool intel_gmbus_is_valid_pin(struct drm_i915_private *dev_priv, { unsigned int size; - if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1) + if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG2) + size = ARRAY_SIZE(gmbus_pins_dg2); + else if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1) size = ARRAY_SIZE(gmbus_pins_dg1); else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) size = ARRAY_SIZE(gmbus_pins_icp); @@ -931,13 +943,6 @@ struct i2c_adapter *intel_gmbus_get_adapter(struct drm_i915_private *dev_priv, return &dev_priv->gmbus[pin].adapter; } -void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed) -{ - struct intel_gmbus *bus = to_intel_gmbus(adapter); - - bus->reg0 = (bus->reg0 & ~(0x3 << 8)) | speed; -} - void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit) { struct intel_gmbus *bus = to_intel_gmbus(adapter); diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.h b/drivers/gpu/drm/i915/display/intel_gmbus.h index b96212b85425..8edc2e99cf53 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.h +++ b/drivers/gpu/drm/i915/display/intel_gmbus.h @@ -41,7 +41,6 @@ int intel_gmbus_output_aksv(struct i2c_adapter *adapter); struct i2c_adapter * intel_gmbus_get_adapter(struct drm_i915_private *dev_priv, unsigned int pin); -void intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed); void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit); bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter); void intel_gmbus_reset(struct drm_i915_private *dev_priv); diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c index 4509fe7438e8..e1ecf38db0ef 100644 --- a/drivers/gpu/drm/i915/display/intel_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_hdcp.c @@ -297,8 +297,7 @@ static int intel_hdcp_load_keys(struct drm_i915_private *dev_priv) * Mailbox interface. */ if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv)) { - ret = sandybridge_pcode_write(dev_priv, - SKL_PCODE_LOAD_HDCP_KEYS, 1); + ret = snb_pcode_write(dev_priv, SKL_PCODE_LOAD_HDCP_KEYS, 1); if (ret) { drm_err(&dev_priv->drm, "Failed to initiate HDCP key load (%d)\n", diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index 3b5b9e7b05b7..1aa5bdc7b0dc 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -730,7 +730,7 @@ intel_hdmi_compute_avi_infoframe(struct intel_encoder *encoder, else frame->colorspace = HDMI_COLORSPACE_RGB; - drm_hdmi_avi_infoframe_colorspace(frame, conn_state); + drm_hdmi_avi_infoframe_colorimetry(frame, conn_state); /* nonsense combination */ drm_WARN_ON(encoder->base.dev, crtc_state->limited_color_range && @@ -1869,7 +1869,7 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi, return MODE_OK; } -static int intel_hdmi_tmds_clock(int clock, int bpc, bool ycbcr420_output) +int intel_hdmi_tmds_clock(int clock, int bpc, bool ycbcr420_output) { /* YCBCR420 TMDS rate requirement is half the pixel clock */ if (ycbcr420_output) @@ -1912,7 +1912,7 @@ static bool intel_hdmi_sink_bpc_possible(struct drm_connector *connector, if (ycbcr420_output) return hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_36; else - return info->edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_36; + return info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_36; case 10: if (!has_hdmi_sink) return false; @@ -1920,7 +1920,7 @@ static bool intel_hdmi_sink_bpc_possible(struct drm_connector *connector, if (ycbcr420_output) return hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_30; else - return info->edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_30; + return info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30; case 8: return true; default: @@ -1935,25 +1935,30 @@ intel_hdmi_mode_clock_valid(struct drm_connector *connector, int clock, { struct drm_i915_private *i915 = to_i915(connector->dev); struct intel_hdmi *hdmi = intel_attached_hdmi(to_intel_connector(connector)); - enum drm_mode_status status; + enum drm_mode_status status = MODE_OK; + int bpc; + + /* + * Try all color depths since valid port clock range + * can have holes. Any mode that can be used with at + * least one color depth is accepted. + */ + for (bpc = 12; bpc >= 8; bpc -= 2) { + int tmds_clock = intel_hdmi_tmds_clock(clock, bpc, ycbcr420_output); - /* check if we can do 8bpc */ - status = hdmi_port_clock_valid(hdmi, intel_hdmi_tmds_clock(clock, 8, ycbcr420_output), - true, has_hdmi_sink); + if (!intel_hdmi_source_bpc_possible(i915, bpc)) + continue; + + if (!intel_hdmi_sink_bpc_possible(connector, bpc, has_hdmi_sink, ycbcr420_output)) + continue; - /* if we can't do 8bpc we may still be able to do 12bpc */ - if (status != MODE_OK && - intel_hdmi_source_bpc_possible(i915, 12) && - intel_hdmi_sink_bpc_possible(connector, 12, has_hdmi_sink, ycbcr420_output)) - status = hdmi_port_clock_valid(hdmi, intel_hdmi_tmds_clock(clock, 12, ycbcr420_output), - true, has_hdmi_sink); + status = hdmi_port_clock_valid(hdmi, tmds_clock, true, has_hdmi_sink); + if (status == MODE_OK) + return MODE_OK; + } - /* if we can't do 8,12bpc we may still be able to do 10bpc */ - if (status != MODE_OK && - intel_hdmi_source_bpc_possible(i915, 10) && - intel_hdmi_sink_bpc_possible(connector, 10, has_hdmi_sink, ycbcr420_output)) - status = hdmi_port_clock_valid(hdmi, intel_hdmi_tmds_clock(clock, 10, ycbcr420_output), - true, has_hdmi_sink); + /* can never happen */ + drm_WARN_ON(&i915->drm, status == MODE_OK); return status; } @@ -2002,17 +2007,14 @@ intel_hdmi_mode_valid(struct drm_connector *connector, return intel_mode_valid_max_plane_size(dev_priv, mode, false); } -bool intel_hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, - int bpc, bool has_hdmi_sink, bool ycbcr420_output) +bool intel_hdmi_bpc_possible(const struct intel_crtc_state *crtc_state, + int bpc, bool has_hdmi_sink, bool ycbcr420_output) { struct drm_atomic_state *state = crtc_state->uapi.state; struct drm_connector_state *connector_state; struct drm_connector *connector; int i; - if (crtc_state->pipe_bpp < bpc * 3) - return false; - for_each_new_connector_in_state(state, connector, connector_state, i) { if (connector_state->crtc != crtc_state->uapi.crtc) continue; @@ -2024,8 +2026,7 @@ bool intel_hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, return true; } -static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, - int bpc) +static bool hdmi_bpc_possible(const struct intel_crtc_state *crtc_state, int bpc) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); @@ -2039,7 +2040,7 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, * HDMI deep color affects the clocks, so it's only possible * when not cloning with other encoder types. */ - if (crtc_state->output_types != BIT(INTEL_OUTPUT_HDMI)) + if (bpc > 8 && crtc_state->output_types != BIT(INTEL_OUTPUT_HDMI)) return false; /* Display Wa_1405510057:icl,ehl */ @@ -2049,35 +2050,50 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, adjusted_mode->crtc_hblank_start) % 8 == 2) return false; - return intel_hdmi_deep_color_possible(crtc_state, bpc, - crtc_state->has_hdmi_sink, - intel_hdmi_is_ycbcr420(crtc_state)); + return intel_hdmi_bpc_possible(crtc_state, bpc, crtc_state->has_hdmi_sink, + intel_hdmi_is_ycbcr420(crtc_state)); } static int intel_hdmi_compute_bpc(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state, - int clock) + int clock, bool respect_downstream_limits) { struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); bool ycbcr420_output = intel_hdmi_is_ycbcr420(crtc_state); int bpc; - for (bpc = 12; bpc >= 10; bpc -= 2) { - if (hdmi_deep_color_possible(crtc_state, bpc) && - hdmi_port_clock_valid(intel_hdmi, - intel_hdmi_tmds_clock(clock, bpc, ycbcr420_output), - true, crtc_state->has_hdmi_sink) == MODE_OK) + /* + * pipe_bpp could already be below 8bpc due to FDI + * bandwidth constraints. HDMI minimum is 8bpc however. + */ + bpc = max(crtc_state->pipe_bpp / 3, 8); + + /* + * We will never exceed downstream TMDS clock limits while + * attempting deep color. If the user insists on forcing an + * out of spec mode they will have to be satisfied with 8bpc. + */ + if (!respect_downstream_limits) + bpc = 8; + + for (; bpc >= 8; bpc -= 2) { + int tmds_clock = intel_hdmi_tmds_clock(clock, bpc, ycbcr420_output); + + if (hdmi_bpc_possible(crtc_state, bpc) && + hdmi_port_clock_valid(intel_hdmi, tmds_clock, + respect_downstream_limits, + crtc_state->has_hdmi_sink) == MODE_OK) return bpc; } - return 8; + return -EINVAL; } static int intel_hdmi_compute_clock(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state) + struct intel_crtc_state *crtc_state, + bool respect_downstream_limits) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); - struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; int bpc, clock = adjusted_mode->crtc_clock; @@ -2085,31 +2101,25 @@ static int intel_hdmi_compute_clock(struct intel_encoder *encoder, if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) clock *= 2; - bpc = intel_hdmi_compute_bpc(encoder, crtc_state, clock); + bpc = intel_hdmi_compute_bpc(encoder, crtc_state, clock, + respect_downstream_limits); + if (bpc < 0) + return bpc; - crtc_state->port_clock = intel_hdmi_tmds_clock(clock, bpc, - intel_hdmi_is_ycbcr420(crtc_state)); + crtc_state->port_clock = + intel_hdmi_tmds_clock(clock, bpc, intel_hdmi_is_ycbcr420(crtc_state)); /* * pipe_bpp could already be below 8bpc due to * FDI bandwidth constraints. We shouldn't bump it - * back up to 8bpc in that case. + * back up to the HDMI minimum 8bpc in that case. */ - if (crtc_state->pipe_bpp > bpc * 3) - crtc_state->pipe_bpp = bpc * 3; + crtc_state->pipe_bpp = min(crtc_state->pipe_bpp, bpc * 3); drm_dbg_kms(&i915->drm, "picking %d bpc for HDMI output (pipe bpp: %d)\n", bpc, crtc_state->pipe_bpp); - if (hdmi_port_clock_valid(intel_hdmi, crtc_state->port_clock, - false, crtc_state->has_hdmi_sink) != MODE_OK) { - drm_dbg_kms(&i915->drm, - "unsupported HDMI clock (%d kHz), rejecting mode\n", - crtc_state->port_clock); - return -EINVAL; - } - return 0; } @@ -2170,7 +2180,8 @@ intel_hdmi_output_format(struct intel_connector *connector, static int intel_hdmi_compute_output_format(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) + const struct drm_connector_state *conn_state, + bool respect_downstream_limits) { struct intel_connector *connector = to_intel_connector(conn_state->connector); const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; @@ -2187,7 +2198,7 @@ static int intel_hdmi_compute_output_format(struct intel_encoder *encoder, crtc_state->output_format = INTEL_OUTPUT_FORMAT_RGB; } - ret = intel_hdmi_compute_clock(encoder, crtc_state); + ret = intel_hdmi_compute_clock(encoder, crtc_state, respect_downstream_limits); if (ret) { if (intel_hdmi_is_ycbcr420(crtc_state) || !connector->base.ycbcr_420_allowed || @@ -2195,7 +2206,7 @@ static int intel_hdmi_compute_output_format(struct intel_encoder *encoder, return ret; crtc_state->output_format = intel_hdmi_output_format(connector, true); - ret = intel_hdmi_compute_clock(encoder, crtc_state); + ret = intel_hdmi_compute_clock(encoder, crtc_state, respect_downstream_limits); } return ret; @@ -2231,9 +2242,19 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder, pipe_config->has_audio = intel_hdmi_has_audio(encoder, pipe_config, conn_state); - ret = intel_hdmi_compute_output_format(encoder, pipe_config, conn_state); + /* + * Try to respect downstream TMDS clock limits first, if + * that fails assume the user might know something we don't. + */ + ret = intel_hdmi_compute_output_format(encoder, pipe_config, conn_state, true); if (ret) + ret = intel_hdmi_compute_output_format(encoder, pipe_config, conn_state, false); + if (ret) { + drm_dbg_kms(&dev_priv->drm, + "unsupported HDMI clock (%d kHz), rejecting mode\n", + pipe_config->hw.adjusted_mode.crtc_clock); return ret; + } if (intel_hdmi_is_ycbcr420(pipe_config)) { ret = intel_panel_fitting(pipe_config, conn_state); @@ -2359,6 +2380,14 @@ intel_hdmi_dp_dual_mode_detect(struct drm_connector *connector, bool has_edid) "DP dual mode adaptor (%s) detected (max TMDS clock: %d kHz)\n", drm_dp_get_dual_mode_type_name(type), hdmi->dp_dual_mode.max_tmds_clock); + + /* Older VBTs are often buggy and can't be trusted :( Play it safe. */ + if ((DISPLAY_VER(dev_priv) >= 8 || IS_HASWELL(dev_priv)) && + !intel_bios_is_port_dp_dual_mode(dev_priv, port)) { + drm_dbg_kms(&dev_priv->drm, + "Ignoring DP dual mode adaptor max TMDS clock for native HDMI port\n"); + hdmi->dp_dual_mode.max_tmds_clock = 0; + } } static bool diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.h b/drivers/gpu/drm/i915/display/intel_hdmi.h index 2bf440eb400a..93f65a917c36 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.h +++ b/drivers/gpu/drm/i915/display/intel_hdmi.h @@ -9,8 +9,6 @@ #include <linux/hdmi.h> #include <linux/types.h> -#include "i915_reg.h" - struct drm_connector; struct drm_encoder; struct drm_i915_private; @@ -46,8 +44,9 @@ void intel_read_infoframe(struct intel_encoder *encoder, union hdmi_infoframe *frame); bool intel_hdmi_limited_color_range(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); -bool intel_hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, int bpc, - bool has_hdmi_sink, bool ycbcr420_output); +bool intel_hdmi_bpc_possible(const struct intel_crtc_state *crtc_state, + int bpc, bool has_hdmi_sink, bool ycbcr420_output); +int intel_hdmi_tmds_clock(int clock, int bpc, bool ycbcr420_output); int intel_hdmi_dsc_get_bpp(int src_fractional_bpp, int slice_width, int num_slices, int output_format, bool hdmi_all_bpp, int hdmi_max_chunk_bytes); diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.c b/drivers/gpu/drm/i915/display/intel_hotplug.c index 955f6d07b0e1..8204126d17f9 100644 --- a/drivers/gpu/drm/i915/display/intel_hotplug.c +++ b/drivers/gpu/drm/i915/display/intel_hotplug.c @@ -24,6 +24,7 @@ #include <linux/kernel.h> #include "i915_drv.h" +#include "i915_irq.h" #include "intel_display_types.h" #include "intel_hotplug.h" @@ -213,12 +214,6 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv) } } -static void intel_hpd_irq_setup(struct drm_i915_private *i915) -{ - if (i915->display_irqs_enabled && i915->hotplug_funcs) - i915->hotplug_funcs->hpd_irq_setup(i915); -} - static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) { struct drm_i915_private *dev_priv = @@ -281,13 +276,13 @@ intel_encoder_hotplug(struct intel_encoder *encoder, ret = true; if (ret) { - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s (epoch counter %llu->%llu)\n", - connector->base.base.id, - connector->base.name, - drm_get_connector_status_name(old_status), - drm_get_connector_status_name(connector->base.status), - old_epoch_counter, - connector->base.epoch_counter); + drm_dbg_kms(dev, "[CONNECTOR:%d:%s] status updated from %s to %s (epoch counter %llu->%llu)\n", + connector->base.base.id, + connector->base.name, + drm_get_connector_status_name(old_status), + drm_get_connector_status_name(connector->base.status), + old_epoch_counter, + connector->base.epoch_counter); return INTEL_HOTPLUG_CHANGED; } return INTEL_HOTPLUG_UNCHANGED; diff --git a/drivers/gpu/drm/i915/display/intel_lspcon.c b/drivers/gpu/drm/i915/display/intel_lspcon.c index 05d2d750fa53..76357c9b76e4 100644 --- a/drivers/gpu/drm/i915/display/intel_lspcon.c +++ b/drivers/gpu/drm/i915/display/intel_lspcon.c @@ -24,7 +24,7 @@ */ #include <drm/drm_atomic_helper.h> -#include <drm/drm_dp_dual_mode_helper.h> +#include <drm/dp/drm_dp_dual_mode_helper.h> #include <drm/drm_edid.h> #include "intel_de.h" @@ -78,11 +78,12 @@ static const char *lspcon_mode_name(enum drm_lspcon_mode mode) static bool lspcon_detect_vendor(struct intel_lspcon *lspcon) { struct intel_dp *dp = lspcon_to_intel_dp(lspcon); + struct drm_i915_private *i915 = dp_to_i915(dp); struct drm_dp_dpcd_ident *ident; u32 vendor_oui; if (drm_dp_read_desc(&dp->aux, &dp->desc, drm_dp_is_branch(dp->dpcd))) { - DRM_ERROR("Can't read description\n"); + drm_err(&i915->drm, "Can't read description\n"); return false; } @@ -93,16 +94,16 @@ static bool lspcon_detect_vendor(struct intel_lspcon *lspcon) switch (vendor_oui) { case LSPCON_VENDOR_MCA_OUI: lspcon->vendor = LSPCON_VENDOR_MCA; - DRM_DEBUG_KMS("Vendor: Mega Chips\n"); + drm_dbg_kms(&i915->drm, "Vendor: Mega Chips\n"); break; case LSPCON_VENDOR_PARADE_OUI: lspcon->vendor = LSPCON_VENDOR_PARADE; - DRM_DEBUG_KMS("Vendor: Parade Tech\n"); + drm_dbg_kms(&i915->drm, "Vendor: Parade Tech\n"); break; default: - DRM_ERROR("Invalid/Unknown vendor OUI\n"); + drm_err(&i915->drm, "Invalid/Unknown vendor OUI\n"); return false; } @@ -119,21 +120,19 @@ static u32 get_hdr_status_reg(struct intel_lspcon *lspcon) void lspcon_detect_hdr_capability(struct intel_lspcon *lspcon) { - struct intel_digital_port *dig_port = - container_of(lspcon, struct intel_digital_port, lspcon); - struct drm_device *dev = dig_port->base.base.dev; - struct intel_dp *dp = lspcon_to_intel_dp(lspcon); + struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 hdr_caps; int ret; - ret = drm_dp_dpcd_read(&dp->aux, get_hdr_status_reg(lspcon), + ret = drm_dp_dpcd_read(&intel_dp->aux, get_hdr_status_reg(lspcon), &hdr_caps, 1); if (ret < 0) { - drm_dbg_kms(dev, "HDR capability detection failed\n"); + drm_dbg_kms(&i915->drm, "HDR capability detection failed\n"); lspcon->hdr_supported = false; } else if (hdr_caps & 0x1) { - drm_dbg_kms(dev, "LSPCON capable of HDR\n"); + drm_dbg_kms(&i915->drm, "LSPCON capable of HDR\n"); lspcon->hdr_supported = true; } } @@ -141,11 +140,12 @@ void lspcon_detect_hdr_capability(struct intel_lspcon *lspcon) static enum drm_lspcon_mode lspcon_get_current_mode(struct intel_lspcon *lspcon) { struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); enum drm_lspcon_mode current_mode; struct i2c_adapter *adapter = &intel_dp->aux.ddc; if (drm_lspcon_get_mode(intel_dp->aux.drm_dev, adapter, ¤t_mode)) { - DRM_DEBUG_KMS("Error reading LSPCON mode\n"); + drm_dbg_kms(&i915->drm, "Error reading LSPCON mode\n"); return DRM_LSPCON_MODE_INVALID; } return current_mode; @@ -154,22 +154,24 @@ static enum drm_lspcon_mode lspcon_get_current_mode(struct intel_lspcon *lspcon) static enum drm_lspcon_mode lspcon_wait_mode(struct intel_lspcon *lspcon, enum drm_lspcon_mode mode) { + struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); enum drm_lspcon_mode current_mode; current_mode = lspcon_get_current_mode(lspcon); if (current_mode == mode) goto out; - DRM_DEBUG_KMS("Waiting for LSPCON mode %s to settle\n", - lspcon_mode_name(mode)); + drm_dbg_kms(&i915->drm, "Waiting for LSPCON mode %s to settle\n", + lspcon_mode_name(mode)); wait_for((current_mode = lspcon_get_current_mode(lspcon)) == mode, 400); if (current_mode != mode) - DRM_ERROR("LSPCON mode hasn't settled\n"); + drm_err(&i915->drm, "LSPCON mode hasn't settled\n"); out: - DRM_DEBUG_KMS("Current LSPCON mode %s\n", - lspcon_mode_name(current_mode)); + drm_dbg_kms(&i915->drm, "Current LSPCON mode %s\n", + lspcon_mode_name(current_mode)); return current_mode; } @@ -178,44 +180,47 @@ static int lspcon_change_mode(struct intel_lspcon *lspcon, enum drm_lspcon_mode mode) { struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); int err; enum drm_lspcon_mode current_mode; struct i2c_adapter *adapter = &intel_dp->aux.ddc; err = drm_lspcon_get_mode(intel_dp->aux.drm_dev, adapter, ¤t_mode); if (err) { - DRM_ERROR("Error reading LSPCON mode\n"); + drm_err(&i915->drm, "Error reading LSPCON mode\n"); return err; } if (current_mode == mode) { - DRM_DEBUG_KMS("Current mode = desired LSPCON mode\n"); + drm_dbg_kms(&i915->drm, "Current mode = desired LSPCON mode\n"); return 0; } err = drm_lspcon_set_mode(intel_dp->aux.drm_dev, adapter, mode); if (err < 0) { - DRM_ERROR("LSPCON mode change failed\n"); + drm_err(&i915->drm, "LSPCON mode change failed\n"); return err; } lspcon->mode = mode; - DRM_DEBUG_KMS("LSPCON mode changed done\n"); + drm_dbg_kms(&i915->drm, "LSPCON mode changed done\n"); return 0; } static bool lspcon_wake_native_aux_ch(struct intel_lspcon *lspcon) { + struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 rev; if (drm_dp_dpcd_readb(&lspcon_to_intel_dp(lspcon)->aux, DP_DPCD_REV, &rev) != 1) { - DRM_DEBUG_KMS("Native AUX CH down\n"); + drm_dbg_kms(&i915->drm, "Native AUX CH down\n"); return false; } - DRM_DEBUG_KMS("Native AUX CH up, DPCD version: %d.%d\n", - rev >> 4, rev & 0xf); + drm_dbg_kms(&i915->drm, "Native AUX CH up, DPCD version: %d.%d\n", + rev >> 4, rev & 0xf); return true; } @@ -225,6 +230,7 @@ static bool lspcon_probe(struct intel_lspcon *lspcon) int retry; enum drm_dp_dual_mode_type adaptor_type; struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct i2c_adapter *adapter = &intel_dp->aux.ddc; enum drm_lspcon_mode expected_mode; @@ -242,13 +248,13 @@ static bool lspcon_probe(struct intel_lspcon *lspcon) } if (adaptor_type != DRM_DP_DUAL_MODE_LSPCON) { - DRM_DEBUG_KMS("No LSPCON detected, found %s\n", - drm_dp_get_dual_mode_type_name(adaptor_type)); + drm_dbg_kms(&i915->drm, "No LSPCON detected, found %s\n", + drm_dp_get_dual_mode_type_name(adaptor_type)); return false; } /* Yay ... got a LSPCON device */ - DRM_DEBUG_KMS("LSPCON detected\n"); + drm_dbg_kms(&i915->drm, "LSPCON detected\n"); lspcon->mode = lspcon_wait_mode(lspcon, expected_mode); /* @@ -258,7 +264,7 @@ static bool lspcon_probe(struct intel_lspcon *lspcon) */ if (lspcon->mode != DRM_LSPCON_MODE_PCON) { if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON) < 0) { - DRM_ERROR("LSPCON mode change to PCON failed\n"); + drm_err(&i915->drm, "LSPCON mode change to PCON failed\n"); return false; } } @@ -268,13 +274,14 @@ static bool lspcon_probe(struct intel_lspcon *lspcon) static void lspcon_resume_in_pcon_wa(struct intel_lspcon *lspcon) { struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); unsigned long start = jiffies; while (1) { if (intel_digital_port_connected(&dig_port->base)) { - DRM_DEBUG_KMS("LSPCON recovering in PCON mode after %u ms\n", - jiffies_to_msecs(jiffies - start)); + drm_dbg_kms(&i915->drm, "LSPCON recovering in PCON mode after %u ms\n", + jiffies_to_msecs(jiffies - start)); return; } @@ -284,7 +291,7 @@ static void lspcon_resume_in_pcon_wa(struct intel_lspcon *lspcon) usleep_range(10000, 15000); } - DRM_DEBUG_KMS("LSPCON DP descriptor mismatch after resume\n"); + drm_dbg_kms(&i915->drm, "LSPCON DP descriptor mismatch after resume\n"); } static bool lspcon_parade_fw_ready(struct drm_dp_aux *aux) @@ -301,7 +308,7 @@ static bool lspcon_parade_fw_ready(struct drm_dp_aux *aux) ret = drm_dp_dpcd_read(aux, LSPCON_PARADE_AVI_IF_CTRL, &avi_if_ctrl, 1); if (ret < 0) { - DRM_ERROR("Failed to read AVI IF control\n"); + drm_err(aux->drm_dev, "Failed to read AVI IF control\n"); return false; } @@ -309,7 +316,7 @@ static bool lspcon_parade_fw_ready(struct drm_dp_aux *aux) return true; } - DRM_ERROR("Parade FW not ready to accept AVI IF\n"); + drm_err(aux->drm_dev, "Parade FW not ready to accept AVI IF\n"); return false; } @@ -324,8 +331,8 @@ static bool _lspcon_parade_write_infoframe_blocks(struct drm_dp_aux *aux, while (block_count < 4) { if (!lspcon_parade_fw_ready(aux)) { - DRM_DEBUG_KMS("LSPCON FW not ready, block %d\n", - block_count); + drm_dbg_kms(aux->drm_dev, "LSPCON FW not ready, block %d\n", + block_count); return false; } @@ -333,8 +340,8 @@ static bool _lspcon_parade_write_infoframe_blocks(struct drm_dp_aux *aux, data = avi_buf + block_count * 8; ret = drm_dp_dpcd_write(aux, reg, data, 8); if (ret < 0) { - DRM_ERROR("Failed to write AVI IF block %d\n", - block_count); + drm_err(aux->drm_dev, "Failed to write AVI IF block %d\n", + block_count); return false; } @@ -348,15 +355,15 @@ static bool _lspcon_parade_write_infoframe_blocks(struct drm_dp_aux *aux, avi_if_ctrl = LSPCON_PARADE_AVI_IF_KICKOFF | block_count; ret = drm_dp_dpcd_write(aux, reg, &avi_if_ctrl, 1); if (ret < 0) { - DRM_ERROR("Failed to update (0x%x), block %d\n", - reg, block_count); + drm_err(aux->drm_dev, "Failed to update (0x%x), block %d\n", + reg, block_count); return false; } block_count++; } - DRM_DEBUG_KMS("Wrote AVI IF blocks successfully\n"); + drm_dbg_kms(aux->drm_dev, "Wrote AVI IF blocks successfully\n"); return true; } @@ -378,14 +385,14 @@ static bool _lspcon_write_avi_infoframe_parade(struct drm_dp_aux *aux, */ if (len > LSPCON_PARADE_AVI_IF_DATA_SIZE - 1) { - DRM_ERROR("Invalid length of infoframes\n"); + drm_err(aux->drm_dev, "Invalid length of infoframes\n"); return false; } memcpy(&avi_if[1], frame, len); if (!_lspcon_parade_write_infoframe_blocks(aux, avi_if)) { - DRM_DEBUG_KMS("Failed to write infoframe blocks\n"); + drm_dbg_kms(aux->drm_dev, "Failed to write infoframe blocks\n"); return false; } @@ -412,7 +419,7 @@ static bool _lspcon_write_avi_infoframe_mca(struct drm_dp_aux *aux, mdelay(50); continue; } else { - DRM_ERROR("DPCD write failed at:0x%x\n", reg); + drm_err(aux->drm_dev, "DPCD write failed at:0x%x\n", reg); return false; } } @@ -423,7 +430,7 @@ static bool _lspcon_write_avi_infoframe_mca(struct drm_dp_aux *aux, reg = LSPCON_MCA_AVI_IF_CTRL; ret = drm_dp_dpcd_read(aux, reg, &val, 1); if (ret < 0) { - DRM_ERROR("DPCD read failed, address 0x%x\n", reg); + drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg); return false; } @@ -433,19 +440,19 @@ static bool _lspcon_write_avi_infoframe_mca(struct drm_dp_aux *aux, ret = drm_dp_dpcd_write(aux, reg, &val, 1); if (ret < 0) { - DRM_ERROR("DPCD read failed, address 0x%x\n", reg); + drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg); return false; } val = 0; ret = drm_dp_dpcd_read(aux, reg, &val, 1); if (ret < 0) { - DRM_ERROR("DPCD read failed, address 0x%x\n", reg); + drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg); return false; } if (val == LSPCON_MCA_AVI_IF_HANDLED) - DRM_DEBUG_KMS("AVI IF handled by FW\n"); + drm_dbg_kms(aux->drm_dev, "AVI IF handled by FW\n"); return true; } @@ -457,6 +464,7 @@ void lspcon_write_infoframe(struct intel_encoder *encoder, { bool ret = true; struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct intel_lspcon *lspcon = enc_to_intel_lspcon(encoder); switch (type) { @@ -469,7 +477,7 @@ void lspcon_write_infoframe(struct intel_encoder *encoder, frame, len); break; case HDMI_PACKET_TYPE_GAMUT_METADATA: - drm_dbg_kms(encoder->base.dev, "Update HDR metadata for lspcon\n"); + drm_dbg_kms(&i915->drm, "Update HDR metadata for lspcon\n"); /* It uses the legacy hsw implementation for the same */ hsw_write_infoframe(encoder, crtc_state, type, frame, len); break; @@ -478,7 +486,7 @@ void lspcon_write_infoframe(struct intel_encoder *encoder, } if (!ret) { - DRM_ERROR("Failed to write infoframes\n"); + drm_err(&i915->drm, "Failed to write infoframes\n"); return; } } @@ -504,11 +512,12 @@ void lspcon_set_infoframes(struct intel_encoder *encoder, u8 buf[VIDEO_DIP_DATA_SIZE]; struct intel_digital_port *dig_port = enc_to_dig_port(encoder); struct intel_lspcon *lspcon = &dig_port->lspcon; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; if (!lspcon->active) { - DRM_ERROR("Writing infoframes while LSPCON disabled ?\n"); + drm_err(&i915->drm, "Writing infoframes while LSPCON disabled ?\n"); return; } @@ -518,7 +527,7 @@ void lspcon_set_infoframes(struct intel_encoder *encoder, conn_state->connector, adjusted_mode); if (ret < 0) { - DRM_ERROR("couldn't fill AVI infoframe\n"); + drm_err(&i915->drm, "couldn't fill AVI infoframe\n"); return; } @@ -537,7 +546,7 @@ void lspcon_set_infoframes(struct intel_encoder *encoder, frame.avi.colorspace = HDMI_COLORSPACE_RGB; /* Set the Colorspace as per the HDMI spec */ - drm_hdmi_avi_infoframe_colorspace(&frame.avi, conn_state); + drm_hdmi_avi_infoframe_colorimetry(&frame.avi, conn_state); /* nonsense combination */ drm_WARN_ON(encoder->base.dev, crtc_state->limited_color_range && @@ -559,7 +568,7 @@ void lspcon_set_infoframes(struct intel_encoder *encoder, ret = hdmi_infoframe_pack(&frame, buf, sizeof(buf)); if (ret < 0) { - DRM_ERROR("Failed to pack AVI IF\n"); + drm_err(&i915->drm, "Failed to pack AVI IF\n"); return; } @@ -575,7 +584,7 @@ static bool _lspcon_read_avi_infoframe_enabled_mca(struct drm_dp_aux *aux) ret = drm_dp_dpcd_read(aux, reg, &val, 1); if (ret < 0) { - DRM_ERROR("DPCD read failed, address 0x%x\n", reg); + drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg); return false; } @@ -590,7 +599,7 @@ static bool _lspcon_read_avi_infoframe_enabled_parade(struct drm_dp_aux *aux) ret = drm_dp_dpcd_read(aux, reg, &val, 1); if (ret < 0) { - DRM_ERROR("DPCD read failed, address 0x%x\n", reg); + drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg); return false; } @@ -634,31 +643,32 @@ void lspcon_wait_pcon_mode(struct intel_lspcon *lspcon) bool lspcon_init(struct intel_digital_port *dig_port) { - struct intel_dp *dp = &dig_port->dp; + struct intel_dp *intel_dp = &dig_port->dp; struct intel_lspcon *lspcon = &dig_port->lspcon; - struct drm_connector *connector = &dp->attached_connector->base; + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct drm_connector *connector = &intel_dp->attached_connector->base; lspcon->active = false; lspcon->mode = DRM_LSPCON_MODE_INVALID; if (!lspcon_probe(lspcon)) { - DRM_ERROR("Failed to probe lspcon\n"); + drm_err(&i915->drm, "Failed to probe lspcon\n"); return false; } - if (drm_dp_read_dpcd_caps(&dp->aux, dp->dpcd) != 0) { - DRM_ERROR("LSPCON DPCD read failed\n"); + if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd) != 0) { + drm_err(&i915->drm, "LSPCON DPCD read failed\n"); return false; } if (!lspcon_detect_vendor(lspcon)) { - DRM_ERROR("LSPCON vendor detection failed\n"); + drm_err(&i915->drm, "LSPCON vendor detection failed\n"); return false; } connector->ycbcr_420_allowed = true; lspcon->active = true; - DRM_DEBUG_KMS("Success: LSPCON init\n"); + drm_dbg_kms(&i915->drm, "Success: LSPCON init\n"); return true; } @@ -674,16 +684,16 @@ void lspcon_resume(struct intel_digital_port *dig_port) { struct intel_lspcon *lspcon = &dig_port->lspcon; struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *i915 = to_i915(dev); enum drm_lspcon_mode expected_mode; - if (!intel_bios_is_lspcon_present(dev_priv, dig_port->base.port)) + if (!intel_bios_is_lspcon_present(i915, dig_port->base.port)) return; if (!lspcon->active) { if (!lspcon_init(dig_port)) { - DRM_ERROR("LSPCON init failed on port %c\n", - port_name(dig_port->base.port)); + drm_err(&i915->drm, "LSPCON init failed on port %c\n", + port_name(dig_port->base.port)); return; } } @@ -699,7 +709,7 @@ void lspcon_resume(struct intel_digital_port *dig_port) return; if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON)) - DRM_ERROR("LSPCON resume failed\n"); + drm_err(&i915->drm, "LSPCON resume failed\n"); else - DRM_DEBUG_KMS("LSPCON resume success\n"); + drm_dbg_kms(&i915->drm, "LSPCON resume success\n"); } diff --git a/drivers/gpu/drm/i915/display/intel_lvds.h b/drivers/gpu/drm/i915/display/intel_lvds.h index bc9c8b84ba2f..9d3372dc503f 100644 --- a/drivers/gpu/drm/i915/display/intel_lvds.h +++ b/drivers/gpu/drm/i915/display/intel_lvds.h @@ -8,7 +8,7 @@ #include <linux/types.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" enum pipe; struct drm_i915_private; diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c index 4a2662838cd8..f31e8c3f8ce0 100644 --- a/drivers/gpu/drm/i915/display/intel_opregion.c +++ b/drivers/gpu/drm/i915/display/intel_opregion.c @@ -35,6 +35,7 @@ #include "intel_backlight.h" #include "intel_display_types.h" #include "intel_opregion.h" +#include "intel_pci_config.h" #define OPREGION_HEADER_OFFSET 0 #define OPREGION_ACPI_OFFSET 0x100 @@ -46,10 +47,11 @@ #define OPREGION_ASLE_EXT_OFFSET 0x1C00 #define OPREGION_SIGNATURE "IntelGraphicsMem" -#define MBOX_ACPI (1<<0) -#define MBOX_SWSCI (1<<1) -#define MBOX_ASLE (1<<2) -#define MBOX_ASLE_EXT (1<<4) +#define MBOX_ACPI BIT(0) /* Mailbox #1 */ +#define MBOX_SWSCI BIT(1) /* Mailbox #2 (obsolete from v2.x) */ +#define MBOX_ASLE BIT(2) /* Mailbox #3 */ +#define MBOX_ASLE_EXT BIT(4) /* Mailbox #5 */ +#define MBOX_BACKLIGHT BIT(5) /* Mailbox #2 (valid from v3.x) */ struct opregion_header { u8 signature[16]; @@ -195,6 +197,8 @@ struct opregion_asle_ext { #define ASLE_IUER_WINDOWS_BTN (1 << 1) #define ASLE_IUER_POWER_BTN (1 << 0) +#define ASLE_PHED_EDID_VALID_MASK 0x3 + /* Software System Control Interrupt (SWSCI) */ #define SWSCI_SCIC_INDICATOR (1 << 0) #define SWSCI_SCIC_MAIN_FUNCTION_SHIFT 1 @@ -242,14 +246,10 @@ struct opregion_asle_ext { #define MAX_DSLP 1500 -static int swsci(struct drm_i915_private *dev_priv, - u32 function, u32 parm, u32 *parm_out) +static int check_swsci_function(struct drm_i915_private *i915, u32 function) { - struct opregion_swsci *swsci = dev_priv->opregion.swsci; - struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); - u32 main_function, sub_function, scic; - u16 swsci_val; - u32 dslp; + struct opregion_swsci *swsci = i915->opregion.swsci; + u32 main_function, sub_function; if (!swsci) return -ENODEV; @@ -261,15 +261,31 @@ static int swsci(struct drm_i915_private *dev_priv, /* Check if we can call the function. See swsci_setup for details. */ if (main_function == SWSCI_SBCB) { - if ((dev_priv->opregion.swsci_sbcb_sub_functions & + if ((i915->opregion.swsci_sbcb_sub_functions & (1 << sub_function)) == 0) return -EINVAL; } else if (main_function == SWSCI_GBDA) { - if ((dev_priv->opregion.swsci_gbda_sub_functions & + if ((i915->opregion.swsci_gbda_sub_functions & (1 << sub_function)) == 0) return -EINVAL; } + return 0; +} + +static int swsci(struct drm_i915_private *dev_priv, + u32 function, u32 parm, u32 *parm_out) +{ + struct opregion_swsci *swsci = dev_priv->opregion.swsci; + struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); + u32 scic, dslp; + u16 swsci_val; + int ret; + + ret = check_swsci_function(dev_priv, function); + if (ret) + return ret; + /* Driver sleep timeout in ms. */ dslp = swsci->dslp; if (!dslp) { @@ -343,11 +359,17 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, u32 parm = 0; u32 type = 0; u32 port; + int ret; /* don't care about old stuff for now */ if (!HAS_DDI(dev_priv)) return 0; + /* Avoid port out of bounds checks if SWSCI isn't there. */ + ret = check_swsci_function(dev_priv, SWSCI_SBCB_DISPLAY_POWER_STATE); + if (ret) + return ret; + if (intel_encoder->type == INTEL_OUTPUT_DSI) port = 0; else @@ -911,9 +933,17 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv) } if (mboxes & MBOX_SWSCI) { - drm_dbg(&dev_priv->drm, "SWSCI supported\n"); - opregion->swsci = base + OPREGION_SWSCI_OFFSET; - swsci_setup(dev_priv); + u8 major = opregion->header->over.major; + + if (major >= 3) { + drm_err(&dev_priv->drm, "SWSCI Mailbox #2 present for opregion v3.x, ignoring\n"); + } else { + if (major >= 2) + drm_dbg(&dev_priv->drm, "SWSCI Mailbox #2 present for opregion v2.x\n"); + drm_dbg(&dev_priv->drm, "SWSCI supported\n"); + opregion->swsci = base + OPREGION_SWSCI_OFFSET; + swsci_setup(dev_priv); + } } if (mboxes & MBOX_ASLE) { @@ -923,8 +953,14 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv) opregion->asle->ardy = ASLE_ARDY_NOT_READY; } - if (mboxes & MBOX_ASLE_EXT) + if (mboxes & MBOX_ASLE_EXT) { drm_dbg(&dev_priv->drm, "ASLE extension supported\n"); + opregion->asle_ext = base + OPREGION_ASLE_EXT_OFFSET; + } + + if (mboxes & MBOX_BACKLIGHT) { + drm_dbg(&dev_priv->drm, "Mailbox #2 for backlight present\n"); + } if (intel_load_vbt_firmware(dev_priv) == 0) goto out; @@ -1051,6 +1087,54 @@ intel_opregion_get_panel_type(struct drm_i915_private *dev_priv) return ret - 1; } +/** + * intel_opregion_get_edid - Fetch EDID from ACPI OpRegion mailbox #5 + * @intel_connector: eDP connector + * + * This reads the ACPI Opregion mailbox #5 to extract the EDID that is passed + * to it. + * + * Returns: + * The EDID in the OpRegion, or NULL if there is none or it's invalid. + * + */ +struct edid *intel_opregion_get_edid(struct intel_connector *intel_connector) +{ + struct drm_connector *connector = &intel_connector->base; + struct drm_i915_private *i915 = to_i915(connector->dev); + struct intel_opregion *opregion = &i915->opregion; + const void *in_edid; + const struct edid *edid; + struct edid *new_edid; + int len; + + if (!opregion->asle_ext) + return NULL; + + in_edid = opregion->asle_ext->bddc; + + /* Validity corresponds to number of 128-byte blocks */ + len = (opregion->asle_ext->phed & ASLE_PHED_EDID_VALID_MASK) * 128; + if (!len || !memchr_inv(in_edid, 0, len)) + return NULL; + + edid = in_edid; + + if (len < EDID_LENGTH * (1 + edid->extensions)) { + drm_dbg_kms(&i915->drm, "Invalid EDID in ACPI OpRegion (Mailbox #5): too short\n"); + return NULL; + } + new_edid = drm_edid_duplicate(edid); + if (!new_edid) + return NULL; + if (!drm_edid_is_valid(new_edid)) { + kfree(new_edid); + drm_dbg_kms(&i915->drm, "Invalid EDID in ACPI OpRegion (Mailbox #5)\n"); + return NULL; + } + return new_edid; +} + void intel_opregion_register(struct drm_i915_private *i915) { struct intel_opregion *opregion = &i915->opregion; @@ -1144,6 +1228,7 @@ void intel_opregion_unregister(struct drm_i915_private *i915) opregion->acpi = NULL; opregion->swsci = NULL; opregion->asle = NULL; + opregion->asle_ext = NULL; opregion->vbt = NULL; opregion->lid_state = NULL; } diff --git a/drivers/gpu/drm/i915/display/intel_opregion.h b/drivers/gpu/drm/i915/display/intel_opregion.h index 4aa68ffbd30e..82cc0ba34af7 100644 --- a/drivers/gpu/drm/i915/display/intel_opregion.h +++ b/drivers/gpu/drm/i915/display/intel_opregion.h @@ -29,12 +29,14 @@ #include <linux/pci.h> struct drm_i915_private; +struct intel_connector; struct intel_encoder; struct opregion_header; struct opregion_acpi; struct opregion_swsci; struct opregion_asle; +struct opregion_asle_ext; struct intel_opregion { struct opregion_header *header; @@ -43,6 +45,7 @@ struct intel_opregion { u32 swsci_gbda_sub_functions; u32 swsci_sbcb_sub_functions; struct opregion_asle *asle; + struct opregion_asle_ext *asle_ext; void *rvda; void *vbt_firmware; const void *vbt; @@ -71,6 +74,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv, pci_power_t state); int intel_opregion_get_panel_type(struct drm_i915_private *dev_priv); +struct edid *intel_opregion_get_edid(struct intel_connector *connector); #else /* CONFIG_ACPI*/ @@ -117,6 +121,12 @@ static inline int intel_opregion_get_panel_type(struct drm_i915_private *dev) return -ENODEV; } +static inline struct edid * +intel_opregion_get_edid(struct intel_connector *connector) +{ + return NULL; +} + #endif /* CONFIG_ACPI */ #endif diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c index d610e48cab94..76845d34ad0c 100644 --- a/drivers/gpu/drm/i915/display/intel_overlay.c +++ b/drivers/gpu/drm/i915/display/intel_overlay.c @@ -28,6 +28,7 @@ #include <drm/drm_fourcc.h> +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gt/intel_gpu_commands.h" #include "gt/intel_ring.h" @@ -38,6 +39,7 @@ #include "intel_display_types.h" #include "intel_frontbuffer.h" #include "intel_overlay.h" +#include "intel_pci_config.h" /* Limits for overlay size. According to intel doc, the real limits are: * Y width: 4095, UV width (planar): 2047, Y height: 2047, diff --git a/drivers/gpu/drm/i915/display/intel_pch_display.c b/drivers/gpu/drm/i915/display/intel_pch_display.c index a55c4bfacd0d..9192769e3337 100644 --- a/drivers/gpu/drm/i915/display/intel_pch_display.c +++ b/drivers/gpu/drm/i915/display/intel_pch_display.c @@ -88,6 +88,50 @@ static void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv, pipe_name(pipe)); } +static void intel_pch_transcoder_set_m1_n1(struct intel_crtc *crtc, + const struct intel_link_m_n *m_n) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + + intel_set_m_n(dev_priv, m_n, + PCH_TRANS_DATA_M1(pipe), PCH_TRANS_DATA_N1(pipe), + PCH_TRANS_LINK_M1(pipe), PCH_TRANS_LINK_N1(pipe)); +} + +static void intel_pch_transcoder_set_m2_n2(struct intel_crtc *crtc, + const struct intel_link_m_n *m_n) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + + intel_set_m_n(dev_priv, m_n, + PCH_TRANS_DATA_M2(pipe), PCH_TRANS_DATA_N2(pipe), + PCH_TRANS_LINK_M2(pipe), PCH_TRANS_LINK_N2(pipe)); +} + +void intel_pch_transcoder_get_m1_n1(struct intel_crtc *crtc, + struct intel_link_m_n *m_n) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + + intel_get_m_n(dev_priv, m_n, + PCH_TRANS_DATA_M1(pipe), PCH_TRANS_DATA_N1(pipe), + PCH_TRANS_LINK_M1(pipe), PCH_TRANS_LINK_N1(pipe)); +} + +void intel_pch_transcoder_get_m2_n2(struct intel_crtc *crtc, + struct intel_link_m_n *m_n) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + + intel_get_m_n(dev_priv, m_n, + PCH_TRANS_DATA_M2(pipe), PCH_TRANS_DATA_N2(pipe), + PCH_TRANS_LINK_M2(pipe), PCH_TRANS_LINK_N2(pipe)); +} + static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state, enum pipe pch_transcoder) { @@ -157,20 +201,20 @@ static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state) */ val &= ~PIPECONF_BPC_MASK; if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - val |= PIPECONF_8BPC; + val |= PIPECONF_BPC_8; else val |= pipeconf_val & PIPECONF_BPC_MASK; } val &= ~TRANS_INTERLACE_MASK; - if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) { + if ((pipeconf_val & PIPECONF_INTERLACE_MASK_ILK) == PIPECONF_INTERLACE_IF_ID_ILK) { if (HAS_PCH_IBX(dev_priv) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) - val |= TRANS_LEGACY_INTERLACED_ILK; + val |= TRANS_INTERLACE_LEGACY_VSYNC_IBX; else - val |= TRANS_INTERLACED; + val |= TRANS_INTERLACE_INTERLACED; } else { - val |= TRANS_PROGRESSIVE; + val |= TRANS_INTERLACE_PROGRESSIVE; } intel_de_write(dev_priv, reg, val | TRANS_ENABLE); @@ -211,6 +255,20 @@ static void ilk_disable_pch_transcoder(struct intel_crtc *crtc) } } +void ilk_pch_pre_enable(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + const struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + /* + * Note: FDI PLL enabling _must_ be done before we enable the + * cpu pipes, hence this is separate from all the other fdi/pch + * enabling. + */ + ilk_fdi_pll_enable(crtc_state); +} + /* * Enable PCH resources required for PCH ports: * - PCH PLLs @@ -264,6 +322,10 @@ void ilk_pch_enable(struct intel_atomic_state *state, /* set transcoder timing, panel must allow it */ assert_pps_unlocked(dev_priv, pipe); + if (intel_crtc_has_dp_encoder(crtc_state)) { + intel_pch_transcoder_set_m1_n1(crtc, &crtc_state->dp_m_n); + intel_pch_transcoder_set_m2_n2(crtc, &crtc_state->dp_m2_n2); + } ilk_pch_transcoder_set_timings(crtc_state, pipe); intel_fdi_normal_train(crtc); @@ -279,7 +341,8 @@ void ilk_pch_enable(struct intel_atomic_state *state, temp = intel_de_read(dev_priv, reg); temp &= ~(TRANS_DP_PORT_SEL_MASK | - TRANS_DP_SYNC_MASK | + TRANS_DP_VSYNC_ACTIVE_HIGH | + TRANS_DP_HSYNC_ACTIVE_HIGH | TRANS_DP_BPC_MASK); temp |= TRANS_DP_OUTPUT_ENABLE; temp |= bpc << 9; /* same format but at 11:9 */ @@ -371,7 +434,8 @@ void ilk_pch_get_config(struct intel_crtc_state *crtc_state) crtc_state->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >> FDI_DP_PORT_WIDTH_SHIFT) + 1; - ilk_get_fdi_m_n_config(crtc, crtc_state); + intel_cpu_transcoder_get_m1_n1(crtc, crtc_state->cpu_transcoder, + &crtc_state->fdi_m_n); if (HAS_PCH_IBX(dev_priv)) { /* @@ -422,11 +486,10 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv, val = TRANS_ENABLE; pipeconf_val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder)); - if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) == - PIPECONF_INTERLACED_ILK) - val |= TRANS_INTERLACED; + if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) == PIPECONF_INTERLACE_IF_ID_ILK) + val |= TRANS_INTERLACE_INTERLACED; else - val |= TRANS_PROGRESSIVE; + val |= TRANS_INTERLACE_PROGRESSIVE; intel_de_write(dev_priv, LPT_TRANSCONF, val); if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF, @@ -495,7 +558,8 @@ void lpt_pch_get_config(struct intel_crtc_state *crtc_state) crtc_state->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >> FDI_DP_PORT_WIDTH_SHIFT) + 1; - ilk_get_fdi_m_n_config(crtc, crtc_state); + intel_cpu_transcoder_get_m1_n1(crtc, crtc_state->cpu_transcoder, + &crtc_state->fdi_m_n); crtc_state->hw.adjusted_mode.crtc_clock = lpt_get_iclkip(dev_priv); } diff --git a/drivers/gpu/drm/i915/display/intel_pch_display.h b/drivers/gpu/drm/i915/display/intel_pch_display.h index 2c387fe3a467..749473d99320 100644 --- a/drivers/gpu/drm/i915/display/intel_pch_display.h +++ b/drivers/gpu/drm/i915/display/intel_pch_display.h @@ -9,7 +9,10 @@ struct intel_atomic_state; struct intel_crtc; struct intel_crtc_state; +struct intel_link_m_n; +void ilk_pch_pre_enable(struct intel_atomic_state *state, + struct intel_crtc *crtc); void ilk_pch_enable(struct intel_atomic_state *state, struct intel_crtc *crtc); void ilk_pch_disable(struct intel_atomic_state *state, @@ -24,4 +27,9 @@ void lpt_pch_disable(struct intel_atomic_state *state, struct intel_crtc *crtc); void lpt_pch_get_config(struct intel_crtc_state *crtc_state); +void intel_pch_transcoder_get_m1_n1(struct intel_crtc *crtc, + struct intel_link_m_n *m_n); +void intel_pch_transcoder_get_m2_n2(struct intel_crtc *crtc, + struct intel_link_m_n *m_n); + #endif diff --git a/drivers/gpu/drm/i915/display/intel_plane_initial.c b/drivers/gpu/drm/i915/display/intel_plane_initial.c index 01ce1d72297f..d7b1de4cc205 100644 --- a/drivers/gpu/drm/i915/display/intel_plane_initial.c +++ b/drivers/gpu/drm/i915/display/intel_plane_initial.c @@ -46,17 +46,18 @@ static struct i915_vma * initial_plane_vma(struct drm_i915_private *i915, struct intel_initial_plane_config *plane_config) { + struct intel_memory_region *mem = i915->mm.stolen_region; struct drm_i915_gem_object *obj; struct i915_vma *vma; u32 base, size; - if (plane_config->size == 0) + if (!mem || plane_config->size == 0) return NULL; base = round_down(plane_config->base, I915_GTT_MIN_ALIGNMENT); size = round_up(plane_config->base + plane_config->size, - I915_GTT_MIN_ALIGNMENT); + mem->min_page_size); size -= base; /* @@ -94,7 +95,7 @@ initial_plane_vma(struct drm_i915_private *i915, goto err_obj; } - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &to_gt(i915)->ggtt->vm, NULL); if (IS_ERR(vma)) goto err_obj; @@ -165,8 +166,6 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc, { struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); struct intel_plane *plane = to_intel_plane(crtc->base.primary); struct intel_plane_state *plane_state = @@ -203,11 +202,6 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc, * pretend the BIOS never had it enabled. */ intel_plane_disable_noatomic(crtc, plane); - if (crtc_state->bigjoiner) { - struct intel_crtc *slave = - crtc_state->bigjoiner_linked_crtc; - intel_plane_disable_noatomic(slave, to_intel_plane(slave->base.primary)); - } return; diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c index e9c679bb1b2e..64bd4ca0edd4 100644 --- a/drivers/gpu/drm/i915/display/intel_pps.c +++ b/drivers/gpu/drm/i915/display/intel_pps.c @@ -1075,14 +1075,14 @@ static void intel_pps_vdd_sanitize(struct intel_dp *intel_dp) edp_panel_vdd_schedule_off(intel_dp); } -bool intel_pps_have_power(struct intel_dp *intel_dp) +bool intel_pps_have_panel_power_or_vdd(struct intel_dp *intel_dp) { intel_wakeref_t wakeref; bool have_power = false; with_intel_pps_lock(intel_dp, wakeref) { - have_power = edp_have_panel_power(intel_dp) && - edp_have_panel_vdd(intel_dp); + have_power = edp_have_panel_power(intel_dp) || + edp_have_panel_vdd(intel_dp); } return have_power; @@ -1131,16 +1131,20 @@ intel_pps_readout_hw_state(struct intel_dp *intel_dp, struct edp_power_seq *seq) } static void -intel_pps_dump_state(const char *state_name, const struct edp_power_seq *seq) +intel_pps_dump_state(struct intel_dp *intel_dp, const char *state_name, + const struct edp_power_seq *seq) { - DRM_DEBUG_KMS("%s t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", - state_name, - seq->t1_t3, seq->t8, seq->t9, seq->t10, seq->t11_t12); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + + drm_dbg_kms(&i915->drm, "%s t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", + state_name, + seq->t1_t3, seq->t8, seq->t9, seq->t10, seq->t11_t12); } static void intel_pps_verify_state(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct edp_power_seq hw; struct edp_power_seq *sw = &intel_dp->pps.pps_delays; @@ -1148,9 +1152,9 @@ intel_pps_verify_state(struct intel_dp *intel_dp) if (hw.t1_t3 != sw->t1_t3 || hw.t8 != sw->t8 || hw.t9 != sw->t9 || hw.t10 != sw->t10 || hw.t11_t12 != sw->t11_t12) { - DRM_ERROR("PPS state mismatch\n"); - intel_pps_dump_state("sw", sw); - intel_pps_dump_state("hw", &hw); + drm_err(&i915->drm, "PPS state mismatch\n"); + intel_pps_dump_state(intel_dp, "sw", sw); + intel_pps_dump_state(intel_dp, "hw", &hw); } } @@ -1168,7 +1172,7 @@ static void pps_init_delays(struct intel_dp *intel_dp) intel_pps_readout_hw_state(intel_dp, &cur); - intel_pps_dump_state("cur", &cur); + intel_pps_dump_state(intel_dp, "cur", &cur); vbt = dev_priv->vbt.edp.pps; /* On Toshiba Satellite P50-C-18C system the VBT T12 delay @@ -1200,7 +1204,7 @@ static void pps_init_delays(struct intel_dp *intel_dp) * too. */ spec.t11_t12 = (510 + 100) * 10; - intel_pps_dump_state("vbt", &vbt); + intel_pps_dump_state(intel_dp, "vbt", &vbt); /* Use the max of the register settings and vbt. If both are * unset, fall back to the spec limits. */ diff --git a/drivers/gpu/drm/i915/display/intel_pps.h b/drivers/gpu/drm/i915/display/intel_pps.h index fbb47f6f453e..e64144659d31 100644 --- a/drivers/gpu/drm/i915/display/intel_pps.h +++ b/drivers/gpu/drm/i915/display/intel_pps.h @@ -37,7 +37,7 @@ void intel_pps_vdd_on(struct intel_dp *intel_dp); void intel_pps_on(struct intel_dp *intel_dp); void intel_pps_off(struct intel_dp *intel_dp); void intel_pps_vdd_off_sync(struct intel_dp *intel_dp); -bool intel_pps_have_power(struct intel_dp *intel_dp); +bool intel_pps_have_panel_power_or_vdd(struct intel_dp *intel_dp); void intel_pps_wait_power_cycle(struct intel_dp *intel_dp); void intel_pps_init(struct intel_dp *intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 00279e8c2775..bff8c2d73cdf 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -1063,31 +1063,28 @@ static void intel_psr_activate(struct intel_dp *intel_dp) intel_dp->psr.active = true; } -static void intel_psr_enable_source(struct intel_dp *intel_dp) +static u32 wa_16013835468_bit_get(struct intel_dp *intel_dp) +{ + switch (intel_dp->psr.pipe) { + case PIPE_A: + return LATENCY_REPORTING_REMOVED_PIPE_A; + case PIPE_B: + return LATENCY_REPORTING_REMOVED_PIPE_B; + case PIPE_C: + return LATENCY_REPORTING_REMOVED_PIPE_C; + default: + MISSING_CASE(intel_dp->psr.pipe); + return 0; + } +} + +static void intel_psr_enable_source(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); enum transcoder cpu_transcoder = intel_dp->psr.transcoder; u32 mask; - if (intel_dp->psr.psr2_enabled && DISPLAY_VER(dev_priv) == 9) { - i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder); - u32 chicken = intel_de_read(dev_priv, reg); - - chicken |= PSR2_VSC_ENABLE_PROG_HEADER | - PSR2_ADD_VERTICAL_LINE_COUNT; - intel_de_write(dev_priv, reg, chicken); - } - - /* - * Wa_16014451276:adlp - * All supported adlp panels have 1-based X granularity, this may - * cause issues if non-supported panels are used. - */ - if (IS_ALDERLAKE_P(dev_priv) && - intel_dp->psr.psr2_enabled) - intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), 0, - ADLP_1_BASED_X_GRANULARITY); - /* * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD also * mask LPSP to avoid dependency on other drivers that might block @@ -1126,18 +1123,47 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp) intel_dp->psr.psr2_sel_fetch_enabled ? IGNORE_PSR2_HW_TRACKING : 0); - /* Wa_16011168373:adl-p */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) && - intel_dp->psr.psr2_enabled) - intel_de_rmw(dev_priv, - TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder), - TRANS_SET_CONTEXT_LATENCY_MASK, - TRANS_SET_CONTEXT_LATENCY_VALUE(1)); + if (intel_dp->psr.psr2_enabled) { + if (DISPLAY_VER(dev_priv) == 9) + intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), 0, + PSR2_VSC_ENABLE_PROG_HEADER | + PSR2_ADD_VERTICAL_LINE_COUNT); - /* Wa_16012604467:adlp */ - if (IS_ALDERLAKE_P(dev_priv) && intel_dp->psr.psr2_enabled) - intel_de_rmw(dev_priv, CLKGATE_DIS_MISC, 0, - CLKGATE_DIS_MISC_DMASC_GATING_DIS); + /* + * Wa_16014451276:adlp + * All supported adlp panels have 1-based X granularity, this may + * cause issues if non-supported panels are used. + */ + if (IS_ALDERLAKE_P(dev_priv)) + intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), 0, + ADLP_1_BASED_X_GRANULARITY); + + /* Wa_16011168373:adl-p */ + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) + intel_de_rmw(dev_priv, + TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder), + TRANS_SET_CONTEXT_LATENCY_MASK, + TRANS_SET_CONTEXT_LATENCY_VALUE(1)); + + /* Wa_16012604467:adlp */ + if (IS_ALDERLAKE_P(dev_priv)) + intel_de_rmw(dev_priv, CLKGATE_DIS_MISC, 0, + CLKGATE_DIS_MISC_DMASC_GATING_DIS); + + /* Wa_16013835468:tgl[b0+], dg1 */ + if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_B0, STEP_FOREVER) || + IS_DG1(dev_priv)) { + u16 vtotal, vblank; + + vtotal = crtc_state->uapi.adjusted_mode.crtc_vtotal - + crtc_state->uapi.adjusted_mode.crtc_vdisplay; + vblank = crtc_state->uapi.adjusted_mode.crtc_vblank_end - + crtc_state->uapi.adjusted_mode.crtc_vblank_start; + if (vblank > vtotal) + intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, 0, + wa_16013835468_bit_get(intel_dp)); + } + } } static bool psr_interrupt_error_check(struct intel_dp *intel_dp) @@ -1202,7 +1228,7 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp, intel_write_dp_vsc_sdp(encoder, crtc_state, &crtc_state->psr_vsc); intel_snps_phy_update_psr_power_state(dev_priv, phy, true); intel_psr_enable_sink(intel_dp); - intel_psr_enable_source(intel_dp); + intel_psr_enable_source(intel_dp, crtc_state); intel_dp->psr.enabled = true; intel_dp->psr.paused = false; @@ -1290,17 +1316,24 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp) intel_de_rmw(dev_priv, CHICKEN_PAR1_1, DIS_RAM_BYPASS_PSR2_MAN_TRACK, 0); - /* Wa_16011168373:adl-p */ - if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) && - intel_dp->psr.psr2_enabled) - intel_de_rmw(dev_priv, - TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder), - TRANS_SET_CONTEXT_LATENCY_MASK, 0); + if (intel_dp->psr.psr2_enabled) { + /* Wa_16011168373:adl-p */ + if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) + intel_de_rmw(dev_priv, + TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder), + TRANS_SET_CONTEXT_LATENCY_MASK, 0); + + /* Wa_16012604467:adlp */ + if (IS_ALDERLAKE_P(dev_priv)) + intel_de_rmw(dev_priv, CLKGATE_DIS_MISC, + CLKGATE_DIS_MISC_DMASC_GATING_DIS, 0); - /* Wa_16012604467:adlp */ - if (IS_ALDERLAKE_P(dev_priv) && intel_dp->psr.psr2_enabled) - intel_de_rmw(dev_priv, CLKGATE_DIS_MISC, - CLKGATE_DIS_MISC_DMASC_GATING_DIS, 0); + /* Wa_16013835468:tgl[b0+], dg1 */ + if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_B0, STEP_FOREVER) || + IS_DG1(dev_priv)) + intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, + wa_16013835468_bit_get(intel_dp), 0); + } intel_snps_phy_update_psr_power_state(dev_priv, phy, false); @@ -1816,6 +1849,9 @@ static void _intel_psr_post_plane_update(const struct intel_atomic_state *state, mutex_lock(&psr->lock); + if (psr->sink_not_reliable) + goto exit; + drm_WARN_ON(&dev_priv->drm, psr->enabled && !crtc_state->active_planes); /* Only enable if there is active planes */ @@ -1826,6 +1862,7 @@ static void _intel_psr_post_plane_update(const struct intel_atomic_state *state, if (crtc_state->crc_enabled && psr->enabled) psr_force_hw_tracking_exit(intel_dp); +exit: mutex_unlock(&psr->lock); } } diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.h b/drivers/gpu/drm/i915/display/intel_sdvo.h index 72065e4360d5..2868852c85f2 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.h +++ b/drivers/gpu/drm/i915/display/intel_sdvo.h @@ -8,7 +8,7 @@ #include <linux/types.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" struct drm_i915_private; enum pipe; diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c index 92ff654f54f5..7e6245b97fed 100644 --- a/drivers/gpu/drm/i915/display/intel_snps_phy.c +++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c @@ -10,6 +10,7 @@ #include "intel_de.h" #include "intel_display_types.h" #include "intel_snps_phy.h" +#include "intel_snps_phy_regs.h" /** * DOC: Synopsis PHY support @@ -23,18 +24,18 @@ * since it is not handled by the shared DPLL framework as on other platforms. */ -void intel_snps_phy_wait_for_calibration(struct drm_i915_private *dev_priv) +void intel_snps_phy_wait_for_calibration(struct drm_i915_private *i915) { enum phy phy; for_each_phy_masked(phy, ~0) { - if (!intel_phy_is_snps(dev_priv, phy)) + if (!intel_phy_is_snps(i915, phy)) continue; - if (intel_de_wait_for_clear(dev_priv, ICL_PHY_MISC(phy), + if (intel_de_wait_for_clear(i915, DG2_PHY_MISC(phy), DG2_PHY_DP_TX_ACK_MASK, 25)) - DRM_ERROR("SNPS PHY %c failed to calibrate after 25ms.\n", - phy_name(phy)); + drm_err(&i915->drm, "SNPS PHY %c failed to calibrate after 25ms.\n", + phy_name(phy)); } } @@ -250,197 +251,6 @@ static const struct intel_mpllb_state * const dg2_dp_100_tables[] = { }; /* - * Basic DP link rates with 38.4 MHz reference clock. - */ - -static const struct intel_mpllb_state dg2_dp_rbr_38_4 = { - .clock = 162000, - .ref_control = - REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), - .mpllb_cp = - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 25) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), - .mpllb_div = - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), - .mpllb_div2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 304), - .mpllb_fracn1 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), - .mpllb_fracn2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 49152), -}; - -static const struct intel_mpllb_state dg2_dp_hbr1_38_4 = { - .clock = 270000, - .ref_control = - REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), - .mpllb_cp = - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 25) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), - .mpllb_div = - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), - .mpllb_div2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 248), - .mpllb_fracn1 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), - .mpllb_fracn2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 40960), -}; - -static const struct intel_mpllb_state dg2_dp_hbr2_38_4 = { - .clock = 540000, - .ref_control = - REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), - .mpllb_cp = - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 25) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), - .mpllb_div = - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), - .mpllb_div2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 248), - .mpllb_fracn1 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), - .mpllb_fracn2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 40960), -}; - -static const struct intel_mpllb_state dg2_dp_hbr3_38_4 = { - .clock = 810000, - .ref_control = - REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), - .mpllb_cp = - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 26) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), - .mpllb_div = - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), - .mpllb_div2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 388), - .mpllb_fracn1 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), - .mpllb_fracn2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 61440), -}; - -static const struct intel_mpllb_state dg2_dp_uhbr10_38_4 = { - .clock = 1000000, - .ref_control = - REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), - .mpllb_cp = - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 5) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 26) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), - .mpllb_div = - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_CLK_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_MULTIPLIER, 8) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_WORD_DIV2_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_DP2_MODE, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_SHIM_DIV32_CLK_SEL, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2), - .mpllb_div2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 488), - .mpllb_fracn1 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 3), - .mpllb_fracn2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 27306), - - /* - * SSC will be enabled, DP UHBR has a minimum SSC requirement. - */ - .mpllb_sscen = - REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 76800), - .mpllb_sscstep = - REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 129024), -}; - -static const struct intel_mpllb_state dg2_dp_uhbr13_38_4 = { - .clock = 1350000, - .ref_control = - REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 1), - .mpllb_cp = - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 56) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 65) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 127), - .mpllb_div = - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_CLK_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV_MULTIPLIER, 8) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_WORD_DIV2_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_DP2_MODE, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 3), - .mpllb_div2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 670), - .mpllb_fracn1 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 1), - .mpllb_fracn2 = - REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 36864), - - /* - * SSC will be enabled, DP UHBR has a minimum SSC requirement. - */ - .mpllb_sscen = - REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_EN, 1) | - REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_PEAK, 103680), - .mpllb_sscstep = - REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_STEPSIZE, 174182), -}; - -static const struct intel_mpllb_state * const dg2_dp_38_4_tables[] = { - &dg2_dp_rbr_38_4, - &dg2_dp_hbr1_38_4, - &dg2_dp_hbr2_38_4, - &dg2_dp_hbr3_38_4, - &dg2_dp_uhbr10_38_4, - &dg2_dp_uhbr13_38_4, - NULL, -}; - -/* * eDP link rates with 100 MHz reference clock. */ @@ -748,22 +558,7 @@ intel_mpllb_tables_get(struct intel_crtc_state *crtc_state, if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) { return dg2_edp_tables; } else if (intel_crtc_has_dp_encoder(crtc_state)) { - /* - * FIXME: Initially we're just enabling the "combo" outputs on - * port A-D. The MPLLB for those ports takes an input from the - * "Display Filter PLL" which always has an output frequency - * of 100 MHz, hence the use of the _100 tables below. - * - * Once we enable port TC1 it will either use the same 100 MHz - * "Display Filter PLL" (when strapped to support a native - * display connection) or different 38.4 MHz "Filter PLL" when - * strapped to support a USB connection, so we'll need to check - * that to determine which table to use. - */ - if (0) - return dg2_dp_38_4_tables; - else - return dg2_dp_100_tables; + return dg2_dp_100_tables; } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { return dg2_hdmi_tables; } @@ -775,6 +570,7 @@ intel_mpllb_tables_get(struct intel_crtc_state *crtc_state, int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state, struct intel_encoder *encoder) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); const struct intel_mpllb_state * const *tables; int i; @@ -786,8 +582,8 @@ int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state, * until we have a proper algorithm under a valid * license. */ - DRM_DEBUG_KMS("Can't support HDMI link rate %d\n", - crtc_state->port_clock); + drm_dbg_kms(&i915->drm, "Can't support HDMI link rate %d\n", + crtc_state->port_clock); return -EINVAL; } } @@ -854,7 +650,7 @@ void intel_mpllb_enable(struct intel_encoder *encoder, * dp_mpllb_state interface signal. */ if (intel_de_wait_for_set(dev_priv, enable_reg, PLL_LOCK, 5)) - DRM_ERROR("Port %c PLL not locked\n", phy_name(phy)); + drm_dbg_kms(&dev_priv->drm, "Port %c PLL not locked\n", phy_name(phy)); /* * 11. If the frequency will result in a change to the voltage @@ -867,8 +663,8 @@ void intel_mpllb_enable(struct intel_encoder *encoder, void intel_mpllb_disable(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + enum phy phy = intel_port_to_phy(i915, encoder->port); i915_reg_t enable_reg = (phy <= PHY_D ? DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0)); @@ -881,21 +677,21 @@ void intel_mpllb_disable(struct intel_encoder *encoder) */ /* 2. Software programs DPLL_ENABLE [PLL Enable] to "0" */ - intel_uncore_rmw(&dev_priv->uncore, enable_reg, PLL_ENABLE, 0); + intel_uncore_rmw(&i915->uncore, enable_reg, PLL_ENABLE, 0); /* * 4. Software programs SNPS_PHY_MPLLB_DIV dp_mpllb_force_en to "0". * This will allow the PLL to stop running. */ - intel_uncore_rmw(&dev_priv->uncore, SNPS_PHY_MPLLB_DIV(phy), + intel_uncore_rmw(&i915->uncore, SNPS_PHY_MPLLB_DIV(phy), SNPS_PHY_MPLLB_FORCE_EN, 0); /* * 5. Software polls DPLL_ENABLE [PLL Lock] for PHY acknowledgment * (dp_txX_ack) that the new transmitter setting request is completed. */ - if (intel_de_wait_for_clear(dev_priv, enable_reg, PLL_LOCK, 5)) - DRM_ERROR("Port %c PLL not locked\n", phy_name(phy)); + if (intel_de_wait_for_clear(i915, enable_reg, PLL_LOCK, 5)) + drm_err(&i915->drm, "Port %c PLL not locked\n", phy_name(phy)); /* * 6. If the frequency will result in a change to the voltage diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy_regs.h b/drivers/gpu/drm/i915/display/intel_snps_phy_regs.h new file mode 100644 index 000000000000..0543465aaf14 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_snps_phy_regs.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_SNPS_PHY_REGS__ +#define __INTEL_SNPS_PHY_REGS__ + +#include "i915_reg_defs.h" + +#define _SNPS_PHY_A_BASE 0x168000 +#define _SNPS_PHY_B_BASE 0x169000 +#define _SNPS_PHY(phy) _PHY(phy, \ + _SNPS_PHY_A_BASE, \ + _SNPS_PHY_B_BASE) +#define _SNPS2(phy, reg) (_SNPS_PHY(phy) - \ + _SNPS_PHY_A_BASE + (reg)) +#define _MMIO_SNPS(phy, reg) _MMIO(_SNPS2(phy, reg)) +#define _MMIO_SNPS_LN(ln, phy, reg) _MMIO(_SNPS2(phy, \ + (reg) + (ln) * 0x10)) + +#define SNPS_PHY_MPLLB_CP(phy) _MMIO_SNPS(phy, 0x168000) +#define SNPS_PHY_MPLLB_CP_INT REG_GENMASK(31, 25) +#define SNPS_PHY_MPLLB_CP_INT_GS REG_GENMASK(23, 17) +#define SNPS_PHY_MPLLB_CP_PROP REG_GENMASK(15, 9) +#define SNPS_PHY_MPLLB_CP_PROP_GS REG_GENMASK(7, 1) + +#define SNPS_PHY_MPLLB_DIV(phy) _MMIO_SNPS(phy, 0x168004) +#define SNPS_PHY_MPLLB_FORCE_EN REG_BIT(31) +#define SNPS_PHY_MPLLB_DIV_CLK_EN REG_BIT(30) +#define SNPS_PHY_MPLLB_DIV5_CLK_EN REG_BIT(29) +#define SNPS_PHY_MPLLB_V2I REG_GENMASK(27, 26) +#define SNPS_PHY_MPLLB_FREQ_VCO REG_GENMASK(25, 24) +#define SNPS_PHY_MPLLB_DIV_MULTIPLIER REG_GENMASK(23, 16) +#define SNPS_PHY_MPLLB_PMIX_EN REG_BIT(10) +#define SNPS_PHY_MPLLB_DP2_MODE REG_BIT(9) +#define SNPS_PHY_MPLLB_WORD_DIV2_EN REG_BIT(8) +#define SNPS_PHY_MPLLB_TX_CLK_DIV REG_GENMASK(7, 5) +#define SNPS_PHY_MPLLB_SHIM_DIV32_CLK_SEL REG_BIT(0) + +#define SNPS_PHY_MPLLB_FRACN1(phy) _MMIO_SNPS(phy, 0x168008) +#define SNPS_PHY_MPLLB_FRACN_EN REG_BIT(31) +#define SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN REG_BIT(30) +#define SNPS_PHY_MPLLB_FRACN_DEN REG_GENMASK(15, 0) + +#define SNPS_PHY_MPLLB_FRACN2(phy) _MMIO_SNPS(phy, 0x16800C) +#define SNPS_PHY_MPLLB_FRACN_REM REG_GENMASK(31, 16) +#define SNPS_PHY_MPLLB_FRACN_QUOT REG_GENMASK(15, 0) + +#define SNPS_PHY_MPLLB_SSCEN(phy) _MMIO_SNPS(phy, 0x168014) +#define SNPS_PHY_MPLLB_SSC_EN REG_BIT(31) +#define SNPS_PHY_MPLLB_SSC_UP_SPREAD REG_BIT(30) +#define SNPS_PHY_MPLLB_SSC_PEAK REG_GENMASK(29, 10) + +#define SNPS_PHY_MPLLB_SSCSTEP(phy) _MMIO_SNPS(phy, 0x168018) +#define SNPS_PHY_MPLLB_SSC_STEPSIZE REG_GENMASK(31, 11) + +#define SNPS_PHY_MPLLB_DIV2(phy) _MMIO_SNPS(phy, 0x16801C) +#define SNPS_PHY_MPLLB_HDMI_PIXEL_CLK_DIV REG_GENMASK(19, 18) +#define SNPS_PHY_MPLLB_HDMI_DIV REG_GENMASK(17, 15) +#define SNPS_PHY_MPLLB_REF_CLK_DIV REG_GENMASK(14, 12) +#define SNPS_PHY_MPLLB_MULTIPLIER REG_GENMASK(11, 0) + +#define SNPS_PHY_REF_CONTROL(phy) _MMIO_SNPS(phy, 0x168188) +#define SNPS_PHY_REF_CONTROL_REF_RANGE REG_GENMASK(31, 27) + +#define SNPS_PHY_TX_REQ(phy) _MMIO_SNPS(phy, 0x168200) +#define SNPS_PHY_TX_REQ_LN_DIS_PWR_STATE_PSR REG_GENMASK(31, 30) + +#define SNPS_PHY_TX_EQ(ln, phy) _MMIO_SNPS_LN(ln, phy, 0x168300) +#define SNPS_PHY_TX_EQ_MAIN REG_GENMASK(23, 18) +#define SNPS_PHY_TX_EQ_POST REG_GENMASK(15, 10) +#define SNPS_PHY_TX_EQ_PRE REG_GENMASK(7, 2) + +#endif /* __INTEL_SNPS_PHY_REGS__ */ diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c index 2357a1301f48..2d71294aaceb 100644 --- a/drivers/gpu/drm/i915/display/intel_sprite.c +++ b/drivers/gpu/drm/i915/display/intel_sprite.c @@ -53,6 +53,7 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state) { + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); const struct drm_framebuffer *fb = plane_state->hw.fb; struct drm_rect *src = &plane_state->uapi.src; u32 src_x, src_y, src_w, src_h, hsub, vsub; @@ -94,14 +95,14 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state) hsub = vsub = max(hsub, vsub); if (src_x % hsub || src_w % hsub) { - DRM_DEBUG_KMS("src x/w (%u, %u) must be a multiple of %u (rotated: %s)\n", - src_x, src_w, hsub, yesno(rotated)); + drm_dbg_kms(&i915->drm, "src x/w (%u, %u) must be a multiple of %u (rotated: %s)\n", + src_x, src_w, hsub, yesno(rotated)); return -EINVAL; } if (src_y % vsub || src_h % vsub) { - DRM_DEBUG_KMS("src y/h (%u, %u) must be a multiple of %u (rotated: %s)\n", - src_y, src_h, vsub, yesno(rotated)); + drm_dbg_kms(&i915->drm, "src y/h (%u, %u) must be a multiple of %u (rotated: %s)\n", + src_y, src_h, vsub, yesno(rotated)); return -EINVAL; } @@ -313,7 +314,7 @@ static u32 vlv_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state) u32 sprctl = 0; if (crtc_state->gamma_enable) - sprctl |= SP_GAMMA_ENABLE; + sprctl |= SP_PIPE_GAMMA_ENABLE; return sprctl; } @@ -436,9 +437,9 @@ vlv_sprite_update_noarm(struct intel_plane *plane, intel_de_write_fw(dev_priv, SPSTRIDE(pipe, plane_id), plane_state->view.color_plane[0].mapping_stride); intel_de_write_fw(dev_priv, SPPOS(pipe, plane_id), - (crtc_y << 16) | crtc_x); + SP_POS_Y(crtc_y) | SP_POS_X(crtc_x)); intel_de_write_fw(dev_priv, SPSIZE(pipe, plane_id), - ((crtc_h - 1) << 16) | (crtc_w - 1)); + SP_HEIGHT(crtc_h - 1) | SP_WIDTH(crtc_w - 1)); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } @@ -479,7 +480,8 @@ vlv_sprite_update_arm(struct intel_plane *plane, intel_de_write_fw(dev_priv, SPCONSTALPHA(pipe, plane_id), 0); intel_de_write_fw(dev_priv, SPLINOFF(pipe, plane_id), linear_offset); - intel_de_write_fw(dev_priv, SPTILEOFF(pipe, plane_id), (y << 16) | x); + intel_de_write_fw(dev_priv, SPTILEOFF(pipe, plane_id), + SP_OFFSET_Y(y) | SP_OFFSET_X(x)); /* * The control register self-arms if the plane was previously @@ -700,7 +702,7 @@ static u32 ivb_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state) u32 sprctl = 0; if (crtc_state->gamma_enable) - sprctl |= SPRITE_GAMMA_ENABLE; + sprctl |= SPRITE_PIPE_GAMMA_ENABLE; if (crtc_state->csc_enable) sprctl |= SPRITE_PIPE_CSC_ENABLE; @@ -770,7 +772,7 @@ static u32 ivb_sprite_ctl(const struct intel_crtc_state *crtc_state, } if (!ivb_need_sprite_gamma(plane_state)) - sprctl |= SPRITE_INT_GAMMA_DISABLE; + sprctl |= SPRITE_PLANE_GAMMA_DISABLE; if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709) sprctl |= SPRITE_YUV_TO_RGB_CSC_FORMAT_BT709; @@ -863,14 +865,18 @@ ivb_sprite_update_noarm(struct intel_plane *plane, unsigned long irqflags; if (crtc_w != src_w || crtc_h != src_h) - sprscale = SPRITE_SCALE_ENABLE | ((src_w - 1) << 16) | (src_h - 1); + sprscale = SPRITE_SCALE_ENABLE | + SPRITE_SRC_WIDTH(src_w - 1) | + SPRITE_SRC_HEIGHT(src_h - 1); spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); intel_de_write_fw(dev_priv, SPRSTRIDE(pipe), plane_state->view.color_plane[0].mapping_stride); - intel_de_write_fw(dev_priv, SPRPOS(pipe), (crtc_y << 16) | crtc_x); - intel_de_write_fw(dev_priv, SPRSIZE(pipe), ((crtc_h - 1) << 16) | (crtc_w - 1)); + intel_de_write_fw(dev_priv, SPRPOS(pipe), + SPRITE_POS_Y(crtc_y) | SPRITE_POS_X(crtc_x)); + intel_de_write_fw(dev_priv, SPRSIZE(pipe), + SPRITE_HEIGHT(crtc_h - 1) | SPRITE_WIDTH(crtc_w - 1)); if (IS_IVYBRIDGE(dev_priv)) intel_de_write_fw(dev_priv, SPRSCALE(pipe), sprscale); @@ -907,10 +913,12 @@ ivb_sprite_update_arm(struct intel_plane *plane, /* HSW consolidates SPRTILEOFF and SPRLINOFF into a single SPROFFSET * register */ if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - intel_de_write_fw(dev_priv, SPROFFSET(pipe), (y << 16) | x); + intel_de_write_fw(dev_priv, SPROFFSET(pipe), + SPRITE_OFFSET_Y(y) | SPRITE_OFFSET_X(x)); } else { intel_de_write_fw(dev_priv, SPRLINOFF(pipe), linear_offset); - intel_de_write_fw(dev_priv, SPRTILEOFF(pipe), (y << 16) | x); + intel_de_write_fw(dev_priv, SPRTILEOFF(pipe), + SPRITE_OFFSET_Y(y) | SPRITE_OFFSET_X(x)); } /* @@ -1047,7 +1055,7 @@ static u32 g4x_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state) u32 dvscntr = 0; if (crtc_state->gamma_enable) - dvscntr |= DVS_GAMMA_ENABLE; + dvscntr |= DVS_PIPE_GAMMA_ENABLE; if (crtc_state->csc_enable) dvscntr |= DVS_PIPE_CSC_ENABLE; @@ -1199,14 +1207,18 @@ g4x_sprite_update_noarm(struct intel_plane *plane, unsigned long irqflags; if (crtc_w != src_w || crtc_h != src_h) - dvsscale = DVS_SCALE_ENABLE | ((src_w - 1) << 16) | (src_h - 1); + dvsscale = DVS_SCALE_ENABLE | + DVS_SRC_WIDTH(src_w - 1) | + DVS_SRC_HEIGHT(src_h - 1); spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); intel_de_write_fw(dev_priv, DVSSTRIDE(pipe), plane_state->view.color_plane[0].mapping_stride); - intel_de_write_fw(dev_priv, DVSPOS(pipe), (crtc_y << 16) | crtc_x); - intel_de_write_fw(dev_priv, DVSSIZE(pipe), ((crtc_h - 1) << 16) | (crtc_w - 1)); + intel_de_write_fw(dev_priv, DVSPOS(pipe), + DVS_POS_Y(crtc_y) | DVS_POS_X(crtc_x)); + intel_de_write_fw(dev_priv, DVSSIZE(pipe), + DVS_HEIGHT(crtc_h - 1) | DVS_WIDTH(crtc_w - 1)); intel_de_write_fw(dev_priv, DVSSCALE(pipe), dvsscale); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); @@ -1321,6 +1333,7 @@ static int g4x_sprite_check_scaling(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state) { + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); const struct drm_framebuffer *fb = plane_state->hw.fb; const struct drm_rect *src = &plane_state->uapi.src; const struct drm_rect *dst = &plane_state->uapi.dst; @@ -1346,7 +1359,7 @@ g4x_sprite_check_scaling(struct intel_crtc_state *crtc_state, if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { if (src_h & 1) { - DRM_DEBUG_KMS("Source height must be even with interlaced modes\n"); + drm_dbg_kms(&i915->drm, "Source height must be even with interlaced modes\n"); return -EINVAL; } min_height = 6; @@ -1358,20 +1371,20 @@ g4x_sprite_check_scaling(struct intel_crtc_state *crtc_state, if (src_w < min_width || src_h < min_height || src_w > 2048 || src_h > 2048) { - DRM_DEBUG_KMS("Source dimensions (%dx%d) exceed hardware limits (%dx%d - %dx%d)\n", - src_w, src_h, min_width, min_height, 2048, 2048); + drm_dbg_kms(&i915->drm, "Source dimensions (%dx%d) exceed hardware limits (%dx%d - %dx%d)\n", + src_w, src_h, min_width, min_height, 2048, 2048); return -EINVAL; } if (width_bytes > 4096) { - DRM_DEBUG_KMS("Fetch width (%d) exceeds hardware max with scaling (%u)\n", - width_bytes, 4096); + drm_dbg_kms(&i915->drm, "Fetch width (%d) exceeds hardware max with scaling (%u)\n", + width_bytes, 4096); return -EINVAL; } if (stride > 4096) { - DRM_DEBUG_KMS("Stride (%u) exceeds hardware max with scaling (%u)\n", - stride, 4096); + drm_dbg_kms(&i915->drm, "Stride (%u) exceeds hardware max with scaling (%u)\n", + stride, 4096); return -EINVAL; } diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c index 7784c30fe893..fc037c027ea5 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.c +++ b/drivers/gpu/drm/i915/display/intel_tc.c @@ -4,10 +4,12 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "intel_display.h" #include "intel_display_types.h" #include "intel_dp_mst.h" #include "intel_tc.h" +#include "intel_tc_phy_regs.h" static const char *tc_port_mode_name(enum tc_port_mode mode) { diff --git a/drivers/gpu/drm/i915/display/intel_tc_phy_regs.h b/drivers/gpu/drm/i915/display/intel_tc_phy_regs.h new file mode 100644 index 000000000000..5a545086f959 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_tc_phy_regs.h @@ -0,0 +1,280 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_TC_PHY_REGS__ +#define __INTEL_TC_PHY_REGS__ + +#include "i915_reg_defs.h" + +#define MG_PHY_PORT_LN(ln, tc_port, ln0p1, ln0p2, ln1p1) \ + _MMIO(_PORT(tc_port, ln0p1, ln0p2) + (ln) * ((ln1p1) - (ln0p1))) + +#define MG_TX_LINK_PARAMS_TX1LN0_PORT1 0x16812C +#define MG_TX_LINK_PARAMS_TX1LN1_PORT1 0x16852C +#define MG_TX_LINK_PARAMS_TX1LN0_PORT2 0x16912C +#define MG_TX_LINK_PARAMS_TX1LN1_PORT2 0x16952C +#define MG_TX1_LINK_PARAMS(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_LINK_PARAMS_TX1LN0_PORT1, \ + MG_TX_LINK_PARAMS_TX1LN0_PORT2, \ + MG_TX_LINK_PARAMS_TX1LN1_PORT1) + +#define MG_TX_LINK_PARAMS_TX2LN0_PORT1 0x1680AC +#define MG_TX_LINK_PARAMS_TX2LN1_PORT1 0x1684AC +#define MG_TX_LINK_PARAMS_TX2LN0_PORT2 0x1690AC +#define MG_TX_LINK_PARAMS_TX2LN1_PORT2 0x1694AC +#define MG_TX2_LINK_PARAMS(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_LINK_PARAMS_TX2LN0_PORT1, \ + MG_TX_LINK_PARAMS_TX2LN0_PORT2, \ + MG_TX_LINK_PARAMS_TX2LN1_PORT1) +#define CRI_USE_FS32 (1 << 5) + +#define MG_TX_PISO_READLOAD_TX1LN0_PORT1 0x16814C +#define MG_TX_PISO_READLOAD_TX1LN1_PORT1 0x16854C +#define MG_TX_PISO_READLOAD_TX1LN0_PORT2 0x16914C +#define MG_TX_PISO_READLOAD_TX1LN1_PORT2 0x16954C +#define MG_TX1_PISO_READLOAD(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_PISO_READLOAD_TX1LN0_PORT1, \ + MG_TX_PISO_READLOAD_TX1LN0_PORT2, \ + MG_TX_PISO_READLOAD_TX1LN1_PORT1) + +#define MG_TX_PISO_READLOAD_TX2LN0_PORT1 0x1680CC +#define MG_TX_PISO_READLOAD_TX2LN1_PORT1 0x1684CC +#define MG_TX_PISO_READLOAD_TX2LN0_PORT2 0x1690CC +#define MG_TX_PISO_READLOAD_TX2LN1_PORT2 0x1694CC +#define MG_TX2_PISO_READLOAD(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_PISO_READLOAD_TX2LN0_PORT1, \ + MG_TX_PISO_READLOAD_TX2LN0_PORT2, \ + MG_TX_PISO_READLOAD_TX2LN1_PORT1) +#define CRI_CALCINIT (1 << 1) + +#define MG_TX_SWINGCTRL_TX1LN0_PORT1 0x168148 +#define MG_TX_SWINGCTRL_TX1LN1_PORT1 0x168548 +#define MG_TX_SWINGCTRL_TX1LN0_PORT2 0x169148 +#define MG_TX_SWINGCTRL_TX1LN1_PORT2 0x169548 +#define MG_TX1_SWINGCTRL(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_SWINGCTRL_TX1LN0_PORT1, \ + MG_TX_SWINGCTRL_TX1LN0_PORT2, \ + MG_TX_SWINGCTRL_TX1LN1_PORT1) + +#define MG_TX_SWINGCTRL_TX2LN0_PORT1 0x1680C8 +#define MG_TX_SWINGCTRL_TX2LN1_PORT1 0x1684C8 +#define MG_TX_SWINGCTRL_TX2LN0_PORT2 0x1690C8 +#define MG_TX_SWINGCTRL_TX2LN1_PORT2 0x1694C8 +#define MG_TX2_SWINGCTRL(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_SWINGCTRL_TX2LN0_PORT1, \ + MG_TX_SWINGCTRL_TX2LN0_PORT2, \ + MG_TX_SWINGCTRL_TX2LN1_PORT1) +#define CRI_TXDEEMPH_OVERRIDE_17_12(x) ((x) << 0) +#define CRI_TXDEEMPH_OVERRIDE_17_12_MASK (0x3F << 0) + +#define MG_TX_DRVCTRL_TX1LN0_TXPORT1 0x168144 +#define MG_TX_DRVCTRL_TX1LN1_TXPORT1 0x168544 +#define MG_TX_DRVCTRL_TX1LN0_TXPORT2 0x169144 +#define MG_TX_DRVCTRL_TX1LN1_TXPORT2 0x169544 +#define MG_TX_DRVCTRL_TX1LN0_TXPORT3 0x16A144 +#define MG_TX_DRVCTRL_TX1LN1_TXPORT3 0x16A544 +#define MG_TX_DRVCTRL_TX1LN0_TXPORT4 0x16B144 +#define MG_TX_DRVCTRL_TX1LN1_TXPORT4 0x16B544 +#define MG_TX1_DRVCTRL(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_DRVCTRL_TX1LN0_TXPORT1, \ + MG_TX_DRVCTRL_TX1LN0_TXPORT2, \ + MG_TX_DRVCTRL_TX1LN1_TXPORT1) + +#define MG_TX_DRVCTRL_TX2LN0_PORT1 0x1680C4 +#define MG_TX_DRVCTRL_TX2LN1_PORT1 0x1684C4 +#define MG_TX_DRVCTRL_TX2LN0_PORT2 0x1690C4 +#define MG_TX_DRVCTRL_TX2LN1_PORT2 0x1694C4 +#define MG_TX2_DRVCTRL(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_DRVCTRL_TX2LN0_PORT1, \ + MG_TX_DRVCTRL_TX2LN0_PORT2, \ + MG_TX_DRVCTRL_TX2LN1_PORT1) +#define CRI_TXDEEMPH_OVERRIDE_11_6(x) ((x) << 24) +#define CRI_TXDEEMPH_OVERRIDE_11_6_MASK (0x3F << 24) +#define CRI_TXDEEMPH_OVERRIDE_EN (1 << 22) +#define CRI_TXDEEMPH_OVERRIDE_5_0(x) ((x) << 16) +#define CRI_TXDEEMPH_OVERRIDE_5_0_MASK (0x3F << 16) +#define CRI_LOADGEN_SEL(x) ((x) << 12) +#define CRI_LOADGEN_SEL_MASK (0x3 << 12) + +#define MG_CLKHUB_LN0_PORT1 0x16839C +#define MG_CLKHUB_LN1_PORT1 0x16879C +#define MG_CLKHUB_LN0_PORT2 0x16939C +#define MG_CLKHUB_LN1_PORT2 0x16979C +#define MG_CLKHUB(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_CLKHUB_LN0_PORT1, \ + MG_CLKHUB_LN0_PORT2, \ + MG_CLKHUB_LN1_PORT1) +#define CFG_LOW_RATE_LKREN_EN (1 << 11) + +#define MG_TX_DCC_TX1LN0_PORT1 0x168110 +#define MG_TX_DCC_TX1LN1_PORT1 0x168510 +#define MG_TX_DCC_TX1LN0_PORT2 0x169110 +#define MG_TX_DCC_TX1LN1_PORT2 0x169510 +#define MG_TX1_DCC(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_DCC_TX1LN0_PORT1, \ + MG_TX_DCC_TX1LN0_PORT2, \ + MG_TX_DCC_TX1LN1_PORT1) +#define MG_TX_DCC_TX2LN0_PORT1 0x168090 +#define MG_TX_DCC_TX2LN1_PORT1 0x168490 +#define MG_TX_DCC_TX2LN0_PORT2 0x169090 +#define MG_TX_DCC_TX2LN1_PORT2 0x169490 +#define MG_TX2_DCC(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_TX_DCC_TX2LN0_PORT1, \ + MG_TX_DCC_TX2LN0_PORT2, \ + MG_TX_DCC_TX2LN1_PORT1) +#define CFG_AMI_CK_DIV_OVERRIDE_VAL(x) ((x) << 25) +#define CFG_AMI_CK_DIV_OVERRIDE_VAL_MASK (0x3 << 25) +#define CFG_AMI_CK_DIV_OVERRIDE_EN (1 << 24) + +#define MG_DP_MODE_LN0_ACU_PORT1 0x1683A0 +#define MG_DP_MODE_LN1_ACU_PORT1 0x1687A0 +#define MG_DP_MODE_LN0_ACU_PORT2 0x1693A0 +#define MG_DP_MODE_LN1_ACU_PORT2 0x1697A0 +#define MG_DP_MODE(ln, tc_port) \ + MG_PHY_PORT_LN(ln, tc_port, MG_DP_MODE_LN0_ACU_PORT1, \ + MG_DP_MODE_LN0_ACU_PORT2, \ + MG_DP_MODE_LN1_ACU_PORT1) +#define MG_DP_MODE_CFG_DP_X2_MODE (1 << 7) +#define MG_DP_MODE_CFG_DP_X1_MODE (1 << 6) + +#define FIA1_BASE 0x163000 +#define FIA2_BASE 0x16E000 +#define FIA3_BASE 0x16F000 +#define _FIA(fia) _PICK((fia), FIA1_BASE, FIA2_BASE, FIA3_BASE) +#define _MMIO_FIA(fia, off) _MMIO(_FIA(fia) + (off)) + +/* ICL PHY DFLEX registers */ +#define PORT_TX_DFLEXDPMLE1(fia) _MMIO_FIA((fia), 0x008C0) +#define DFLEXDPMLE1_DPMLETC_MASK(idx) (0xf << (4 * (idx))) +#define DFLEXDPMLE1_DPMLETC_ML0(idx) (1 << (4 * (idx))) +#define DFLEXDPMLE1_DPMLETC_ML1_0(idx) (3 << (4 * (idx))) +#define DFLEXDPMLE1_DPMLETC_ML3(idx) (8 << (4 * (idx))) +#define DFLEXDPMLE1_DPMLETC_ML3_2(idx) (12 << (4 * (idx))) +#define DFLEXDPMLE1_DPMLETC_ML3_0(idx) (15 << (4 * (idx))) + +#define _MG_REFCLKIN_CTL_PORT1 0x16892C +#define _MG_REFCLKIN_CTL_PORT2 0x16992C +#define MG_REFCLKIN_CTL_OD_2_MUX(x) ((x) << 8) +#define MG_REFCLKIN_CTL_OD_2_MUX_MASK (0x7 << 8) +#define MG_REFCLKIN_CTL(tc_port) _MMIO_PORT((tc_port), \ + _MG_REFCLKIN_CTL_PORT1, \ + _MG_REFCLKIN_CTL_PORT2) + +#define _MG_CLKTOP2_CORECLKCTL1_PORT1 0x1688D8 +#define _MG_CLKTOP2_CORECLKCTL1_PORT2 0x1698D8 +#define MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO(x) ((x) << 16) +#define MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO_MASK (0xff << 16) +#define MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(x) ((x) << 8) +#define MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK (0xff << 8) +#define MG_CLKTOP2_CORECLKCTL1(tc_port) _MMIO_PORT((tc_port), \ + _MG_CLKTOP2_CORECLKCTL1_PORT1, \ + _MG_CLKTOP2_CORECLKCTL1_PORT2) + +#define _MG_CLKTOP2_HSCLKCTL_PORT1 0x1688D4 +#define _MG_CLKTOP2_HSCLKCTL_PORT2 0x1698D4 +#define MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(x) ((x) << 16) +#define MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK (0x1 << 16) +#define MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(x) ((x) << 14) +#define MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK (0x3 << 14) +#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK (0x3 << 12) +#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2 (0 << 12) +#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_3 (1 << 12) +#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_5 (2 << 12) +#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_7 (3 << 12) +#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(x) ((x) << 8) +#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT 8 +#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK (0xf << 8) +#define MG_CLKTOP2_HSCLKCTL(tc_port) _MMIO_PORT((tc_port), \ + _MG_CLKTOP2_HSCLKCTL_PORT1, \ + _MG_CLKTOP2_HSCLKCTL_PORT2) + +#define _MG_PLL_DIV0_PORT1 0x168A00 +#define _MG_PLL_DIV0_PORT2 0x169A00 +#define MG_PLL_DIV0_FRACNEN_H (1 << 30) +#define MG_PLL_DIV0_FBDIV_FRAC_MASK (0x3fffff << 8) +#define MG_PLL_DIV0_FBDIV_FRAC_SHIFT 8 +#define MG_PLL_DIV0_FBDIV_FRAC(x) ((x) << 8) +#define MG_PLL_DIV0_FBDIV_INT_MASK (0xff << 0) +#define MG_PLL_DIV0_FBDIV_INT(x) ((x) << 0) +#define MG_PLL_DIV0(tc_port) _MMIO_PORT((tc_port), _MG_PLL_DIV0_PORT1, \ + _MG_PLL_DIV0_PORT2) + +#define _MG_PLL_DIV1_PORT1 0x168A04 +#define _MG_PLL_DIV1_PORT2 0x169A04 +#define MG_PLL_DIV1_IREF_NDIVRATIO(x) ((x) << 16) +#define MG_PLL_DIV1_DITHER_DIV_1 (0 << 12) +#define MG_PLL_DIV1_DITHER_DIV_2 (1 << 12) +#define MG_PLL_DIV1_DITHER_DIV_4 (2 << 12) +#define MG_PLL_DIV1_DITHER_DIV_8 (3 << 12) +#define MG_PLL_DIV1_NDIVRATIO(x) ((x) << 4) +#define MG_PLL_DIV1_FBPREDIV_MASK (0xf << 0) +#define MG_PLL_DIV1_FBPREDIV(x) ((x) << 0) +#define MG_PLL_DIV1(tc_port) _MMIO_PORT((tc_port), _MG_PLL_DIV1_PORT1, \ + _MG_PLL_DIV1_PORT2) + +#define _MG_PLL_LF_PORT1 0x168A08 +#define _MG_PLL_LF_PORT2 0x169A08 +#define MG_PLL_LF_TDCTARGETCNT(x) ((x) << 24) +#define MG_PLL_LF_AFCCNTSEL_256 (0 << 20) +#define MG_PLL_LF_AFCCNTSEL_512 (1 << 20) +#define MG_PLL_LF_GAINCTRL(x) ((x) << 16) +#define MG_PLL_LF_INT_COEFF(x) ((x) << 8) +#define MG_PLL_LF_PROP_COEFF(x) ((x) << 0) +#define MG_PLL_LF(tc_port) _MMIO_PORT((tc_port), _MG_PLL_LF_PORT1, \ + _MG_PLL_LF_PORT2) + +#define _MG_PLL_FRAC_LOCK_PORT1 0x168A0C +#define _MG_PLL_FRAC_LOCK_PORT2 0x169A0C +#define MG_PLL_FRAC_LOCK_TRUELOCK_CRIT_32 (1 << 18) +#define MG_PLL_FRAC_LOCK_EARLYLOCK_CRIT_32 (1 << 16) +#define MG_PLL_FRAC_LOCK_LOCKTHRESH(x) ((x) << 11) +#define MG_PLL_FRAC_LOCK_DCODITHEREN (1 << 10) +#define MG_PLL_FRAC_LOCK_FEEDFWRDCAL_EN (1 << 8) +#define MG_PLL_FRAC_LOCK_FEEDFWRDGAIN(x) ((x) << 0) +#define MG_PLL_FRAC_LOCK(tc_port) _MMIO_PORT((tc_port), \ + _MG_PLL_FRAC_LOCK_PORT1, \ + _MG_PLL_FRAC_LOCK_PORT2) + +#define _MG_PLL_SSC_PORT1 0x168A10 +#define _MG_PLL_SSC_PORT2 0x169A10 +#define MG_PLL_SSC_EN (1 << 28) +#define MG_PLL_SSC_TYPE(x) ((x) << 26) +#define MG_PLL_SSC_STEPLENGTH(x) ((x) << 16) +#define MG_PLL_SSC_STEPNUM(x) ((x) << 10) +#define MG_PLL_SSC_FLLEN (1 << 9) +#define MG_PLL_SSC_STEPSIZE(x) ((x) << 0) +#define MG_PLL_SSC(tc_port) _MMIO_PORT((tc_port), _MG_PLL_SSC_PORT1, \ + _MG_PLL_SSC_PORT2) + +#define _MG_PLL_BIAS_PORT1 0x168A14 +#define _MG_PLL_BIAS_PORT2 0x169A14 +#define MG_PLL_BIAS_BIAS_GB_SEL(x) ((x) << 30) +#define MG_PLL_BIAS_BIAS_GB_SEL_MASK (0x3 << 30) +#define MG_PLL_BIAS_INIT_DCOAMP(x) ((x) << 24) +#define MG_PLL_BIAS_INIT_DCOAMP_MASK (0x3f << 24) +#define MG_PLL_BIAS_BIAS_BONUS(x) ((x) << 16) +#define MG_PLL_BIAS_BIAS_BONUS_MASK (0xff << 16) +#define MG_PLL_BIAS_BIASCAL_EN (1 << 15) +#define MG_PLL_BIAS_CTRIM(x) ((x) << 8) +#define MG_PLL_BIAS_CTRIM_MASK (0x1f << 8) +#define MG_PLL_BIAS_VREF_RDAC(x) ((x) << 5) +#define MG_PLL_BIAS_VREF_RDAC_MASK (0x7 << 5) +#define MG_PLL_BIAS_IREFTRIM(x) ((x) << 0) +#define MG_PLL_BIAS_IREFTRIM_MASK (0x1f << 0) +#define MG_PLL_BIAS(tc_port) _MMIO_PORT((tc_port), _MG_PLL_BIAS_PORT1, \ + _MG_PLL_BIAS_PORT2) + +#define _MG_PLL_TDC_COLDST_BIAS_PORT1 0x168A18 +#define _MG_PLL_TDC_COLDST_BIAS_PORT2 0x169A18 +#define MG_PLL_TDC_COLDST_IREFINT_EN (1 << 27) +#define MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(x) ((x) << 17) +#define MG_PLL_TDC_COLDST_COLDSTART (1 << 16) +#define MG_PLL_TDC_TDCOVCCORR_EN (1 << 2) +#define MG_PLL_TDC_TDCSEL(x) ((x) << 0) +#define MG_PLL_TDC_COLDST_BIAS(tc_port) _MMIO_PORT((tc_port), \ + _MG_PLL_TDC_COLDST_BIAS_PORT1, \ + _MG_PLL_TDC_COLDST_BIAS_PORT2) + +#endif /* __INTEL_TC_PHY_REGS__ */ diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h index f043d85ba64d..b9397d9363c5 100644 --- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h +++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h @@ -162,6 +162,14 @@ struct bdb_general_features { u8 dp_ssc_freq:1; /* SSC freq for PCH attached eDP */ u8 dp_ssc_dongle_supported:1; u8 rsvd11:2; /* finish byte */ + + /* bits 6 */ + u8 tc_hpd_retry_timeout:7; /* 242 */ + u8 rsvd12:1; + + /* bits 7 */ + u8 afc_startup_config:2;/* 249 */ + u8 rsvd13:6; } __packed; /* @@ -226,32 +234,6 @@ struct bdb_general_features { #define DEVICE_TYPE_DIGITAL_OUTPUT (1 << 1) #define DEVICE_TYPE_ANALOG_OUTPUT (1 << 0) -/* - * Bits we care about when checking for DEVICE_TYPE_eDP. Depending on the - * system, the other bits may or may not be set for eDP outputs. - */ -#define DEVICE_TYPE_eDP_BITS \ - (DEVICE_TYPE_INTERNAL_CONNECTOR | \ - DEVICE_TYPE_MIPI_OUTPUT | \ - DEVICE_TYPE_COMPOSITE_OUTPUT | \ - DEVICE_TYPE_DUAL_CHANNEL | \ - DEVICE_TYPE_LVDS_SIGNALING | \ - DEVICE_TYPE_TMDS_DVI_SIGNALING | \ - DEVICE_TYPE_VIDEO_SIGNALING | \ - DEVICE_TYPE_DISPLAYPORT_OUTPUT | \ - DEVICE_TYPE_ANALOG_OUTPUT) - -#define DEVICE_TYPE_DP_DUAL_MODE_BITS \ - (DEVICE_TYPE_INTERNAL_CONNECTOR | \ - DEVICE_TYPE_MIPI_OUTPUT | \ - DEVICE_TYPE_COMPOSITE_OUTPUT | \ - DEVICE_TYPE_LVDS_SIGNALING | \ - DEVICE_TYPE_TMDS_DVI_SIGNALING | \ - DEVICE_TYPE_VIDEO_SIGNALING | \ - DEVICE_TYPE_DISPLAYPORT_OUTPUT | \ - DEVICE_TYPE_DIGITAL_OUTPUT | \ - DEVICE_TYPE_ANALOG_OUTPUT) - #define DEVICE_CFG_NONE 0x00 #define DEVICE_CFG_12BIT_DVOB 0x01 #define DEVICE_CFG_12BIT_DVOC 0x02 diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index 9b05f93ed8bc..545eff5bf158 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -341,19 +341,14 @@ bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state) const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - enum pipe pipe = crtc->pipe; if (!INTEL_INFO(i915)->display.has_dsc) return false; - /* On TGL, DSC is supported on all Pipes */ if (DISPLAY_VER(i915) >= 12) return true; - if (DISPLAY_VER(i915) >= 11 && - (pipe != PIPE_A || cpu_transcoder == TRANSCODER_EDP || - cpu_transcoder == TRANSCODER_DSI_0 || - cpu_transcoder == TRANSCODER_DSI_1)) + if (DISPLAY_VER(i915) >= 11 && cpu_transcoder != TRANSCODER_A) return true; return false; @@ -1112,18 +1107,6 @@ static i915_reg_t dss_ctl2_reg(struct intel_crtc *crtc, enum transcoder cpu_tran ICL_PIPE_DSS_CTL2(crtc->pipe) : DSS_CTL2; } -struct intel_crtc * -intel_dsc_get_bigjoiner_secondary(const struct intel_crtc *primary_crtc) -{ - return intel_crtc_for_pipe(to_i915(primary_crtc->base.dev), primary_crtc->pipe + 1); -} - -static struct intel_crtc * -intel_dsc_get_bigjoiner_primary(const struct intel_crtc *secondary_crtc) -{ - return intel_crtc_for_pipe(to_i915(secondary_crtc->base.dev), secondary_crtc->pipe - 1); -} - void intel_uncompressed_joiner_enable(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); @@ -1131,7 +1114,7 @@ void intel_uncompressed_joiner_enable(const struct intel_crtc_state *crtc_state) u32 dss_ctl1_val = 0; if (crtc_state->bigjoiner && !crtc_state->dsc.compression_enable) { - if (crtc_state->bigjoiner_slave) + if (intel_crtc_is_bigjoiner_slave(crtc_state)) dss_ctl1_val |= UNCOMPRESSED_JOINER_SLAVE; else dss_ctl1_val |= UNCOMPRESSED_JOINER_MASTER; @@ -1159,7 +1142,7 @@ void intel_dsc_enable(const struct intel_crtc_state *crtc_state) } if (crtc_state->bigjoiner) { dss_ctl1_val |= BIG_JOINER_ENABLE; - if (!crtc_state->bigjoiner_slave) + if (!intel_crtc_is_bigjoiner_slave(crtc_state)) dss_ctl1_val |= MASTER_BIG_JOINER_ENABLE; } intel_de_write(dev_priv, dss_ctl1_reg(crtc, crtc_state->cpu_transcoder), dss_ctl1_val); @@ -1179,25 +1162,6 @@ void intel_dsc_disable(const struct intel_crtc_state *old_crtc_state) } } -void intel_uncompressed_joiner_get_config(struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 dss_ctl1; - - dss_ctl1 = intel_de_read(dev_priv, dss_ctl1_reg(crtc, crtc_state->cpu_transcoder)); - if (dss_ctl1 & UNCOMPRESSED_JOINER_MASTER) { - crtc_state->bigjoiner = true; - crtc_state->bigjoiner_linked_crtc = intel_dsc_get_bigjoiner_secondary(crtc); - drm_WARN_ON(&dev_priv->drm, !crtc_state->bigjoiner_linked_crtc); - } else if (dss_ctl1 & UNCOMPRESSED_JOINER_SLAVE) { - crtc_state->bigjoiner = true; - crtc_state->bigjoiner_slave = true; - crtc_state->bigjoiner_linked_crtc = intel_dsc_get_bigjoiner_primary(crtc); - drm_WARN_ON(&dev_priv->drm, !crtc_state->bigjoiner_linked_crtc); - } -} - void intel_dsc_get_config(struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); @@ -1228,18 +1192,6 @@ void intel_dsc_get_config(struct intel_crtc_state *crtc_state) crtc_state->dsc.dsc_split = (dss_ctl2 & RIGHT_BRANCH_VDSC_ENABLE) && (dss_ctl1 & JOINER_ENABLE); - if (dss_ctl1 & BIG_JOINER_ENABLE) { - crtc_state->bigjoiner = true; - - if (!(dss_ctl1 & MASTER_BIG_JOINER_ENABLE)) { - crtc_state->bigjoiner_slave = true; - crtc_state->bigjoiner_linked_crtc = intel_dsc_get_bigjoiner_primary(crtc); - } else { - crtc_state->bigjoiner_linked_crtc = intel_dsc_get_bigjoiner_secondary(crtc); - } - drm_WARN_ON(&dev_priv->drm, !crtc_state->bigjoiner_linked_crtc); - } - /* FIXME: add more state readout as needed */ /* PPS1 */ diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.h b/drivers/gpu/drm/i915/display/intel_vdsc.h index 4ec75f715986..8763f00fa7e2 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.h +++ b/drivers/gpu/drm/i915/display/intel_vdsc.h @@ -18,7 +18,6 @@ void intel_uncompressed_joiner_enable(const struct intel_crtc_state *crtc_state) void intel_dsc_enable(const struct intel_crtc_state *crtc_state); void intel_dsc_disable(const struct intel_crtc_state *crtc_state); int intel_dsc_compute_params(struct intel_crtc_state *pipe_config); -void intel_uncompressed_joiner_get_config(struct intel_crtc_state *crtc_state); void intel_dsc_get_config(struct intel_crtc_state *crtc_state); enum intel_display_power_domain intel_dsc_power_domain(struct intel_crtc *crtc, enum transcoder cpu_transcoder); diff --git a/drivers/gpu/drm/i915/display/intel_vga.c b/drivers/gpu/drm/i915/display/intel_vga.c index fa779f7ea415..b5d058404c14 100644 --- a/drivers/gpu/drm/i915/display/intel_vga.c +++ b/drivers/gpu/drm/i915/display/intel_vga.c @@ -7,6 +7,7 @@ #include <linux/vgaarb.h> #include <drm/i915_drm.h> +#include <video/vga.h> #include "i915_drv.h" #include "intel_de.h" @@ -34,9 +35,9 @@ void intel_vga_disable(struct drm_i915_private *dev_priv) /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */ vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO); - outb(SR01, VGA_SR_INDEX); - sr1 = inb(VGA_SR_DATA); - outb(sr1 | 1 << 5, VGA_SR_DATA); + outb(0x01, VGA_SEQ_I); + sr1 = inb(VGA_SEQ_D); + outb(sr1 | VGA_SR01_SCREEN_OFF, VGA_SEQ_D); vga_put(pdev, VGA_RSRC_LEGACY_IO); udelay(300); @@ -92,7 +93,7 @@ void intel_vga_reset_io_mem(struct drm_i915_private *i915) * and error messages. */ vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO); - outb(inb(VGA_MSR_READ), VGA_MSR_WRITE); + outb(inb(VGA_MIS_R), VGA_MIS_W); vga_put(pdev, VGA_RSRC_LEGACY_IO); } diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 93a385396512..1223075595ff 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -961,6 +961,7 @@ static u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state, static u32 skl_surf_address(const struct intel_plane_state *plane_state, int color_plane) { + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); const struct drm_framebuffer *fb = plane_state->hw.fb; u32 offset = plane_state->view.color_plane[color_plane].offset; @@ -969,11 +970,11 @@ static u32 skl_surf_address(const struct intel_plane_state *plane_state, * The DPT object contains only one vma, so the VMA's offset * within the DPT is always 0. */ - WARN_ON(plane_state->dpt_vma->node.start); - WARN_ON(offset & 0x1fffff); + drm_WARN_ON(&i915->drm, plane_state->dpt_vma->node.start); + drm_WARN_ON(&i915->drm, offset & 0x1fffff); return offset >> 9; } else { - WARN_ON(offset & 0xfff); + drm_WARN_ON(&i915->drm, offset & 0xfff); return offset; } } @@ -992,6 +993,54 @@ static u32 skl_plane_surf(const struct intel_plane_state *plane_state, return plane_surf; } +static u32 skl_plane_aux_dist(const struct intel_plane_state *plane_state, + int color_plane) +{ + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); + const struct drm_framebuffer *fb = plane_state->hw.fb; + int aux_plane = skl_main_to_aux_plane(fb, color_plane); + u32 aux_dist; + + if (!aux_plane) + return 0; + + aux_dist = skl_surf_address(plane_state, aux_plane) - + skl_surf_address(plane_state, color_plane); + + if (DISPLAY_VER(i915) < 12) + aux_dist |= PLANE_AUX_STRIDE(skl_plane_stride(plane_state, aux_plane)); + + return aux_dist; +} + +static u32 skl_plane_keyval(const struct intel_plane_state *plane_state) +{ + const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; + + return key->min_value; +} + +static u32 skl_plane_keymax(const struct intel_plane_state *plane_state) +{ + const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; + u8 alpha = plane_state->hw.alpha >> 8; + + return (key->max_value & 0xffffff) | PLANE_KEYMAX_ALPHA(alpha); +} + +static u32 skl_plane_keymsk(const struct intel_plane_state *plane_state) +{ + const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; + u8 alpha = plane_state->hw.alpha >> 8; + u32 keymsk; + + keymsk = key->channel_mask & 0x7ffffff; + if (alpha < 0xff) + keymsk |= PLANE_KEYMSK_ALPHA_ENABLE; + + return keymsk; +} + static void icl_plane_csc_load_black(struct intel_plane *plane) { struct drm_i915_private *i915 = to_i915(plane->base.dev); @@ -1016,15 +1065,24 @@ static void icl_plane_csc_load_black(struct intel_plane *plane) intel_de_write_fw(i915, PLANE_CSC_POSTOFF(pipe, plane_id, 2), 0); } +static int skl_plane_color_plane(const struct intel_plane_state *plane_state) +{ + /* Program the UV plane on planar master */ + if (plane_state->planar_linked_plane && !plane_state->planar_slave) + return 1; + else + return 0; +} + static void -skl_program_plane_noarm(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state, - int color_plane) +skl_plane_update_noarm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum plane_id plane_id = plane->id; enum pipe pipe = plane->pipe; + int color_plane = skl_plane_color_plane(plane_state); u32 stride = skl_plane_stride(plane_state, color_plane); const struct drm_framebuffer *fb = plane_state->hw.fb; int crtc_x = plane_state->uapi.dst.x1; @@ -1048,11 +1106,12 @@ skl_program_plane_noarm(struct intel_plane *plane, if (plane_state->force_black) icl_plane_csc_load_black(plane); - intel_de_write_fw(dev_priv, PLANE_STRIDE(pipe, plane_id), stride); + intel_de_write_fw(dev_priv, PLANE_STRIDE(pipe, plane_id), + PLANE_STRIDE_(stride)); intel_de_write_fw(dev_priv, PLANE_POS(pipe, plane_id), - (crtc_y << 16) | crtc_x); + PLANE_POS_Y(crtc_y) | PLANE_POS_X(crtc_x)); intel_de_write_fw(dev_priv, PLANE_SIZE(pipe, plane_id), - ((src_h - 1) << 16) | (src_w - 1)); + PLANE_HEIGHT(src_h - 1) | PLANE_WIDTH(src_w - 1)); if (intel_fb_is_rc_ccs_cc_modifier(fb->modifier)) { intel_de_write_fw(dev_priv, PLANE_CC_VAL(pipe, plane_id, 0), @@ -1076,21 +1135,17 @@ skl_program_plane_noarm(struct intel_plane *plane, } static void -skl_program_plane_arm(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state, - int color_plane) +skl_plane_update_arm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) { struct drm_i915_private *dev_priv = to_i915(plane->base.dev); enum plane_id plane_id = plane->id; enum pipe pipe = plane->pipe; - const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; - const struct drm_framebuffer *fb = plane_state->hw.fb; - int aux_plane = skl_main_to_aux_plane(fb, color_plane); + int color_plane = skl_plane_color_plane(plane_state); u32 x = plane_state->view.color_plane[color_plane].x; u32 y = plane_state->view.color_plane[color_plane].y; - u32 keymsk, keymax, aux_dist = 0, plane_color_ctl = 0; - u8 alpha = plane_state->hw.alpha >> 8; + u32 plane_color_ctl = 0; u32 plane_ctl = plane_state->ctl; unsigned long irqflags; @@ -1100,36 +1155,22 @@ skl_program_plane_arm(struct intel_plane *plane, plane_color_ctl = plane_state->color_ctl | glk_plane_color_ctl_crtc(crtc_state); - keymax = (key->max_value & 0xffffff) | PLANE_KEYMAX_ALPHA(alpha); - - keymsk = key->channel_mask & 0x7ffffff; - if (alpha < 0xff) - keymsk |= PLANE_KEYMSK_ALPHA_ENABLE; - - if (aux_plane) { - aux_dist = skl_surf_address(plane_state, aux_plane) - - skl_surf_address(plane_state, color_plane); - - if (DISPLAY_VER(dev_priv) < 12) - aux_dist |= skl_plane_stride(plane_state, aux_plane); - } - spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - intel_de_write_fw(dev_priv, PLANE_KEYVAL(pipe, plane_id), - key->min_value); - intel_de_write_fw(dev_priv, PLANE_KEYMSK(pipe, plane_id), keymsk); - intel_de_write_fw(dev_priv, PLANE_KEYMAX(pipe, plane_id), keymax); + intel_de_write_fw(dev_priv, PLANE_KEYVAL(pipe, plane_id), skl_plane_keyval(plane_state)); + intel_de_write_fw(dev_priv, PLANE_KEYMSK(pipe, plane_id), skl_plane_keymsk(plane_state)); + intel_de_write_fw(dev_priv, PLANE_KEYMAX(pipe, plane_id), skl_plane_keymax(plane_state)); intel_de_write_fw(dev_priv, PLANE_OFFSET(pipe, plane_id), - (y << 16) | x); + PLANE_OFFSET_Y(y) | PLANE_OFFSET_X(x)); - intel_de_write_fw(dev_priv, PLANE_AUX_DIST(pipe, plane_id), aux_dist); + intel_de_write_fw(dev_priv, PLANE_AUX_DIST(pipe, plane_id), + skl_plane_aux_dist(plane_state, color_plane)); if (DISPLAY_VER(dev_priv) < 11) intel_de_write_fw(dev_priv, PLANE_AUX_OFFSET(pipe, plane_id), - (plane_state->view.color_plane[1].y << 16) | - plane_state->view.color_plane[1].x); + PLANE_OFFSET_Y(plane_state->view.color_plane[1].y) | + PLANE_OFFSET_X(plane_state->view.color_plane[1].x)); if (DISPLAY_VER(dev_priv) >= 10) intel_de_write_fw(dev_priv, PLANE_COLOR_CTL(pipe, plane_id), plane_color_ctl); @@ -1182,34 +1223,6 @@ skl_plane_async_flip(struct intel_plane *plane, spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } -static void -skl_plane_update_noarm(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) -{ - int color_plane = 0; - - if (plane_state->planar_linked_plane && !plane_state->planar_slave) - /* Program the UV plane on planar master */ - color_plane = 1; - - skl_program_plane_noarm(plane, crtc_state, plane_state, color_plane); -} - -static void -skl_plane_update_arm(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) -{ - int color_plane = 0; - - if (plane_state->planar_linked_plane && !plane_state->planar_slave) - /* Program the UV plane on planar master */ - color_plane = 1; - - skl_program_plane_arm(plane, crtc_state, plane_state, color_plane); -} - static bool intel_format_is_p01x(u32 format) { switch (format) { @@ -1338,6 +1351,7 @@ static int skl_plane_check_dst_coordinates(const struct intel_crtc_state *crtc_s static int skl_plane_check_nv12_rotation(const struct intel_plane_state *plane_state) { + struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); const struct drm_framebuffer *fb = plane_state->hw.fb; unsigned int rotation = plane_state->hw.rotation; int src_w = drm_rect_width(&plane_state->uapi.src) >> 16; @@ -1347,7 +1361,7 @@ static int skl_plane_check_nv12_rotation(const struct intel_plane_state *plane_s src_w & 3 && (rotation == DRM_MODE_ROTATE_270 || rotation == (DRM_MODE_REFLECT_X | DRM_MODE_ROTATE_90))) { - DRM_DEBUG_KMS("src width must be multiple of 4 for rotated planar YUV\n"); + drm_dbg_kms(&i915->drm, "src width must be multiple of 4 for rotated planar YUV\n"); return -EINVAL; } @@ -1816,20 +1830,27 @@ static int skl_plane_check(struct intel_crtc_state *crtc_state, return 0; } +static enum intel_fbc_id skl_fbc_id_for_pipe(enum pipe pipe) +{ + return pipe - PIPE_A + INTEL_FBC_A; +} + static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv, - enum pipe pipe, enum plane_id plane_id) + enum intel_fbc_id fbc_id, enum plane_id plane_id) { - if (!HAS_FBC(dev_priv)) + if ((INTEL_INFO(dev_priv)->display.fbc_mask & BIT(fbc_id)) == 0) return false; - return pipe == PIPE_A && plane_id == PLANE_PRIMARY; + return plane_id == PLANE_PRIMARY; } static struct intel_fbc *skl_plane_fbc(struct drm_i915_private *dev_priv, enum pipe pipe, enum plane_id plane_id) { - if (skl_plane_has_fbc(dev_priv, pipe, plane_id)) - return dev_priv->fbc; + enum intel_fbc_id fbc_id = skl_fbc_id_for_pipe(pipe); + + if (skl_plane_has_fbc(dev_priv, fbc_id, plane_id)) + return dev_priv->fbc[fbc_id]; else return NULL; } @@ -2282,16 +2303,17 @@ skl_get_initial_plane_config(struct intel_crtc *crtc, val = intel_de_read(dev_priv, PLANE_CTL(pipe, plane_id)); if (DISPLAY_VER(dev_priv) >= 11) - pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK; + pixel_format = val & PLANE_CTL_FORMAT_MASK_ICL; else - pixel_format = val & PLANE_CTL_FORMAT_MASK; + pixel_format = val & PLANE_CTL_FORMAT_MASK_SKL; if (DISPLAY_VER(dev_priv) >= 10) { - alpha = intel_de_read(dev_priv, - PLANE_COLOR_CTL(pipe, plane_id)); - alpha &= PLANE_COLOR_ALPHA_MASK; + u32 color_ctl; + + color_ctl = intel_de_read(dev_priv, PLANE_COLOR_CTL(pipe, plane_id)); + alpha = REG_FIELD_GET(PLANE_COLOR_ALPHA_MASK, color_ctl); } else { - alpha = val & PLANE_CTL_ALPHA_MASK; + alpha = REG_FIELD_GET(PLANE_CTL_ALPHA_MASK, val); } fourcc = skl_format_to_fourcc(pixel_format, @@ -2355,22 +2377,19 @@ skl_get_initial_plane_config(struct intel_crtc *crtc, if (drm_rotation_90_or_270(plane_config->rotation)) goto error; - base = intel_de_read(dev_priv, PLANE_SURF(pipe, plane_id)) & 0xfffff000; + base = intel_de_read(dev_priv, PLANE_SURF(pipe, plane_id)) & PLANE_SURF_ADDR_MASK; plane_config->base = base; offset = intel_de_read(dev_priv, PLANE_OFFSET(pipe, plane_id)); val = intel_de_read(dev_priv, PLANE_SIZE(pipe, plane_id)); - fb->height = ((val >> 16) & 0xffff) + 1; - fb->width = ((val >> 0) & 0xffff) + 1; + fb->height = REG_FIELD_GET(PLANE_HEIGHT_MASK, val) + 1; + fb->width = REG_FIELD_GET(PLANE_WIDTH_MASK, val) + 1; val = intel_de_read(dev_priv, PLANE_STRIDE(pipe, plane_id)); stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0); - if (DISPLAY_VER(dev_priv) >= 13) - fb->pitches[0] = (val & PLANE_STRIDE_MASK_XELPD) * stride_mult; - else - fb->pitches[0] = (val & PLANE_STRIDE_MASK) * stride_mult; + fb->pitches[0] = REG_FIELD_GET(PLANE_STRIDE__MASK, val) * stride_mult; aligned_height = intel_fb_align_height(fb, 0, fb->height); diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c index 20141f33ed64..0d936f658b3f 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi.c +++ b/drivers/gpu/drm/i915/display/vlv_dsi.c @@ -44,6 +44,7 @@ #include "skl_scaler.h" #include "vlv_dsi.h" #include "vlv_dsi_pll.h" +#include "vlv_dsi_regs.h" #include "vlv_sideband.h" /* return pixels in terms of txbyteclkhs */ @@ -1492,7 +1493,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder, */ if (is_vid_mode(intel_dsi) && - intel_dsi->video_mode_format == VIDEO_MODE_BURST) { + intel_dsi->video_mode == BURST_MODE) { intel_de_write(dev_priv, MIPI_HS_TX_TIMEOUT(port), txbyteclkhs(adjusted_mode->crtc_htotal, bpp, intel_dsi->lane_count, intel_dsi->burst_mode_ratio) + 1); } else { @@ -1568,12 +1569,33 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder, intel_de_write(dev_priv, MIPI_CLK_LANE_SWITCH_TIME_CNT(port), intel_dsi->clk_lp_to_hs_count << LP_HS_SSW_CNT_SHIFT | intel_dsi->clk_hs_to_lp_count << HS_LP_PWR_SW_CNT_SHIFT); - if (is_vid_mode(intel_dsi)) - /* Some panels might have resolution which is not a + if (is_vid_mode(intel_dsi)) { + u32 fmt = intel_dsi->video_frmt_cfg_bits | IP_TG_CONFIG; + + /* + * Some panels might have resolution which is not a * multiple of 64 like 1366 x 768. Enable RANDOM - * resolution support for such panels by default */ - intel_de_write(dev_priv, MIPI_VIDEO_MODE_FORMAT(port), - intel_dsi->video_frmt_cfg_bits | intel_dsi->video_mode_format | IP_TG_CONFIG | RANDOM_DPI_DISPLAY_RESOLUTION); + * resolution support for such panels by default. + */ + fmt |= RANDOM_DPI_DISPLAY_RESOLUTION; + + switch (intel_dsi->video_mode) { + default: + MISSING_CASE(intel_dsi->video_mode); + fallthrough; + case NON_BURST_SYNC_EVENTS: + fmt |= VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS; + break; + case NON_BURST_SYNC_PULSE: + fmt |= VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE; + break; + case BURST_MODE: + fmt |= VIDEO_MODE_BURST; + break; + } + + intel_de_write(dev_priv, MIPI_VIDEO_MODE_FORMAT(port), fmt); + } } } diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c index 1b81797dd02e..df880f44700a 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c +++ b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c @@ -32,6 +32,7 @@ #include "intel_display_types.h" #include "intel_dsi.h" #include "vlv_dsi_pll.h" +#include "vlv_dsi_pll_regs.h" #include "vlv_sideband.h" static const u16 lfsr_converts[] = { diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_pll_regs.h b/drivers/gpu/drm/i915/display/vlv_dsi_pll_regs.h new file mode 100644 index 000000000000..45590e14e54b --- /dev/null +++ b/drivers/gpu/drm/i915/display/vlv_dsi_pll_regs.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __VLV_DSI_PLL_REGS_H__ +#define __VLV_DSI_PLL_REGS_H__ + +#include "vlv_dsi_regs.h" + +#define MIPIO_TXESC_CLK_DIV1 _MMIO(0x160004) +#define GLK_TX_ESC_CLK_DIV1_MASK 0x3FF +#define MIPIO_TXESC_CLK_DIV2 _MMIO(0x160008) +#define GLK_TX_ESC_CLK_DIV2_MASK 0x3FF + +#define BXT_MAX_VAR_OUTPUT_KHZ 39500 + +#define BXT_MIPI_CLOCK_CTL _MMIO(0x46090) +#define BXT_MIPI1_DIV_SHIFT 26 +#define BXT_MIPI2_DIV_SHIFT 10 +#define BXT_MIPI_DIV_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_DIV_SHIFT, \ + BXT_MIPI2_DIV_SHIFT) + +/* TX control divider to select actual TX clock output from (8x/var) */ +#define BXT_MIPI1_TX_ESCLK_SHIFT 26 +#define BXT_MIPI2_TX_ESCLK_SHIFT 10 +#define BXT_MIPI_TX_ESCLK_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_SHIFT, \ + BXT_MIPI2_TX_ESCLK_SHIFT) +#define BXT_MIPI1_TX_ESCLK_FIXDIV_MASK (0x3F << 26) +#define BXT_MIPI2_TX_ESCLK_FIXDIV_MASK (0x3F << 10) +#define BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port) \ + _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_FIXDIV_MASK, \ + BXT_MIPI2_TX_ESCLK_FIXDIV_MASK) +#define BXT_MIPI_TX_ESCLK_DIVIDER(port, val) \ + (((val) & 0x3F) << BXT_MIPI_TX_ESCLK_SHIFT(port)) +/* RX upper control divider to select actual RX clock output from 8x */ +#define BXT_MIPI1_RX_ESCLK_UPPER_SHIFT 21 +#define BXT_MIPI2_RX_ESCLK_UPPER_SHIFT 5 +#define BXT_MIPI_RX_ESCLK_UPPER_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_UPPER_SHIFT, \ + BXT_MIPI2_RX_ESCLK_UPPER_SHIFT) +#define BXT_MIPI1_RX_ESCLK_UPPER_FIXDIV_MASK (3 << 21) +#define BXT_MIPI2_RX_ESCLK_UPPER_FIXDIV_MASK (3 << 5) +#define BXT_MIPI_RX_ESCLK_UPPER_FIXDIV_MASK(port) \ + _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_UPPER_FIXDIV_MASK, \ + BXT_MIPI2_RX_ESCLK_UPPER_FIXDIV_MASK) +#define BXT_MIPI_RX_ESCLK_UPPER_DIVIDER(port, val) \ + (((val) & 3) << BXT_MIPI_RX_ESCLK_UPPER_SHIFT(port)) +/* 8/3X divider to select the actual 8/3X clock output from 8x */ +#define BXT_MIPI1_8X_BY3_SHIFT 19 +#define BXT_MIPI2_8X_BY3_SHIFT 3 +#define BXT_MIPI_8X_BY3_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_8X_BY3_SHIFT, \ + BXT_MIPI2_8X_BY3_SHIFT) +#define BXT_MIPI1_8X_BY3_DIVIDER_MASK (3 << 19) +#define BXT_MIPI2_8X_BY3_DIVIDER_MASK (3 << 3) +#define BXT_MIPI_8X_BY3_DIVIDER_MASK(port) \ + _MIPI_PORT(port, BXT_MIPI1_8X_BY3_DIVIDER_MASK, \ + BXT_MIPI2_8X_BY3_DIVIDER_MASK) +#define BXT_MIPI_8X_BY3_DIVIDER(port, val) \ + (((val) & 3) << BXT_MIPI_8X_BY3_SHIFT(port)) +/* RX lower control divider to select actual RX clock output from 8x */ +#define BXT_MIPI1_RX_ESCLK_LOWER_SHIFT 16 +#define BXT_MIPI2_RX_ESCLK_LOWER_SHIFT 0 +#define BXT_MIPI_RX_ESCLK_LOWER_SHIFT(port) \ + _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_LOWER_SHIFT, \ + BXT_MIPI2_RX_ESCLK_LOWER_SHIFT) +#define BXT_MIPI1_RX_ESCLK_LOWER_FIXDIV_MASK (3 << 16) +#define BXT_MIPI2_RX_ESCLK_LOWER_FIXDIV_MASK (3 << 0) +#define BXT_MIPI_RX_ESCLK_LOWER_FIXDIV_MASK(port) \ + _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_LOWER_FIXDIV_MASK, \ + BXT_MIPI2_RX_ESCLK_LOWER_FIXDIV_MASK) +#define BXT_MIPI_RX_ESCLK_LOWER_DIVIDER(port, val) \ + (((val) & 3) << BXT_MIPI_RX_ESCLK_LOWER_SHIFT(port)) + +#define RX_DIVIDER_BIT_1_2 0x3 +#define RX_DIVIDER_BIT_3_4 0xC + +#define BXT_DSI_PLL_CTL _MMIO(0x161000) +#define BXT_DSI_PLL_PVD_RATIO_SHIFT 16 +#define BXT_DSI_PLL_PVD_RATIO_MASK (3 << BXT_DSI_PLL_PVD_RATIO_SHIFT) +#define BXT_DSI_PLL_PVD_RATIO_1 (1 << BXT_DSI_PLL_PVD_RATIO_SHIFT) +#define BXT_DSIC_16X_BY1 (0 << 10) +#define BXT_DSIC_16X_BY2 (1 << 10) +#define BXT_DSIC_16X_BY3 (2 << 10) +#define BXT_DSIC_16X_BY4 (3 << 10) +#define BXT_DSIC_16X_MASK (3 << 10) +#define BXT_DSIA_16X_BY1 (0 << 8) +#define BXT_DSIA_16X_BY2 (1 << 8) +#define BXT_DSIA_16X_BY3 (2 << 8) +#define BXT_DSIA_16X_BY4 (3 << 8) +#define BXT_DSIA_16X_MASK (3 << 8) +#define BXT_DSI_FREQ_SEL_SHIFT 8 +#define BXT_DSI_FREQ_SEL_MASK (0xF << BXT_DSI_FREQ_SEL_SHIFT) + +#define BXT_DSI_PLL_RATIO_MAX 0x7D +#define BXT_DSI_PLL_RATIO_MIN 0x22 +#define GLK_DSI_PLL_RATIO_MAX 0x6F +#define GLK_DSI_PLL_RATIO_MIN 0x22 +#define BXT_DSI_PLL_RATIO_MASK 0xFF +#define BXT_REF_CLOCK_KHZ 19200 + +#define BXT_DSI_PLL_ENABLE _MMIO(0x46080) +#define BXT_DSI_PLL_DO_ENABLE (1 << 31) +#define BXT_DSI_PLL_LOCKED (1 << 30) + +#endif /* __VLV_DSI_PLL_REGS_H__ */ diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h new file mode 100644 index 000000000000..356e51515346 --- /dev/null +++ b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h @@ -0,0 +1,480 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __VLV_DSI_REGS_H__ +#define __VLV_DSI_REGS_H__ + +#include "i915_reg_defs.h" + +#define VLV_MIPI_BASE VLV_DISPLAY_BASE +#define BXT_MIPI_BASE 0x60000 + +#define _MIPI_PORT(port, a, c) (((port) == PORT_A) ? a : c) /* ports A and C only */ +#define _MMIO_MIPI(port, a, c) _MMIO(_MIPI_PORT(port, a, c)) + +/* BXT MIPI mode configure */ +#define _BXT_MIPIA_TRANS_HACTIVE 0x6B0F8 +#define _BXT_MIPIC_TRANS_HACTIVE 0x6B8F8 +#define BXT_MIPI_TRANS_HACTIVE(tc) _MMIO_MIPI(tc, \ + _BXT_MIPIA_TRANS_HACTIVE, _BXT_MIPIC_TRANS_HACTIVE) + +#define _BXT_MIPIA_TRANS_VACTIVE 0x6B0FC +#define _BXT_MIPIC_TRANS_VACTIVE 0x6B8FC +#define BXT_MIPI_TRANS_VACTIVE(tc) _MMIO_MIPI(tc, \ + _BXT_MIPIA_TRANS_VACTIVE, _BXT_MIPIC_TRANS_VACTIVE) + +#define _BXT_MIPIA_TRANS_VTOTAL 0x6B100 +#define _BXT_MIPIC_TRANS_VTOTAL 0x6B900 +#define BXT_MIPI_TRANS_VTOTAL(tc) _MMIO_MIPI(tc, \ + _BXT_MIPIA_TRANS_VTOTAL, _BXT_MIPIC_TRANS_VTOTAL) + +#define BXT_P_DSI_REGULATOR_CFG _MMIO(0x160020) +#define STAP_SELECT (1 << 0) + +#define BXT_P_DSI_REGULATOR_TX_CTRL _MMIO(0x160054) +#define HS_IO_CTRL_SELECT (1 << 0) + +#define _MIPIA_PORT_CTRL (VLV_DISPLAY_BASE + 0x61190) +#define _MIPIC_PORT_CTRL (VLV_DISPLAY_BASE + 0x61700) +#define MIPI_PORT_CTRL(port) _MMIO_MIPI(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL) + + /* BXT port control */ +#define _BXT_MIPIA_PORT_CTRL 0x6B0C0 +#define _BXT_MIPIC_PORT_CTRL 0x6B8C0 +#define BXT_MIPI_PORT_CTRL(tc) _MMIO_MIPI(tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL) + +#define DPI_ENABLE (1 << 31) /* A + C */ +#define MIPIA_MIPI4DPHY_DELAY_COUNT_SHIFT 27 +#define MIPIA_MIPI4DPHY_DELAY_COUNT_MASK (0xf << 27) +#define DUAL_LINK_MODE_SHIFT 26 +#define DUAL_LINK_MODE_MASK (1 << 26) +#define DUAL_LINK_MODE_FRONT_BACK (0 << 26) +#define DUAL_LINK_MODE_PIXEL_ALTERNATIVE (1 << 26) +#define DITHERING_ENABLE (1 << 25) /* A + C */ +#define FLOPPED_HSTX (1 << 23) +#define DE_INVERT (1 << 19) /* XXX */ +#define MIPIA_FLISDSI_DELAY_COUNT_SHIFT 18 +#define MIPIA_FLISDSI_DELAY_COUNT_MASK (0xf << 18) +#define AFE_LATCHOUT (1 << 17) +#define LP_OUTPUT_HOLD (1 << 16) +#define MIPIC_FLISDSI_DELAY_COUNT_HIGH_SHIFT 15 +#define MIPIC_FLISDSI_DELAY_COUNT_HIGH_MASK (1 << 15) +#define MIPIC_MIPI4DPHY_DELAY_COUNT_SHIFT 11 +#define MIPIC_MIPI4DPHY_DELAY_COUNT_MASK (0xf << 11) +#define CSB_SHIFT 9 +#define CSB_MASK (3 << 9) +#define CSB_20MHZ (0 << 9) +#define CSB_10MHZ (1 << 9) +#define CSB_40MHZ (2 << 9) +#define BANDGAP_MASK (1 << 8) +#define BANDGAP_PNW_CIRCUIT (0 << 8) +#define BANDGAP_LNC_CIRCUIT (1 << 8) +#define MIPIC_FLISDSI_DELAY_COUNT_LOW_SHIFT 5 +#define MIPIC_FLISDSI_DELAY_COUNT_LOW_MASK (7 << 5) +#define TEARING_EFFECT_DELAY (1 << 4) /* A + C */ +#define TEARING_EFFECT_SHIFT 2 /* A + C */ +#define TEARING_EFFECT_MASK (3 << 2) +#define TEARING_EFFECT_OFF (0 << 2) +#define TEARING_EFFECT_DSI (1 << 2) +#define TEARING_EFFECT_GPIO (2 << 2) +#define LANE_CONFIGURATION_SHIFT 0 +#define LANE_CONFIGURATION_MASK (3 << 0) +#define LANE_CONFIGURATION_4LANE (0 << 0) +#define LANE_CONFIGURATION_DUAL_LINK_A (1 << 0) +#define LANE_CONFIGURATION_DUAL_LINK_B (2 << 0) + +#define _MIPIA_TEARING_CTRL (VLV_DISPLAY_BASE + 0x61194) +#define _MIPIC_TEARING_CTRL (VLV_DISPLAY_BASE + 0x61704) +#define MIPI_TEARING_CTRL(port) _MMIO_MIPI(port, _MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL) +#define TEARING_EFFECT_DELAY_SHIFT 0 +#define TEARING_EFFECT_DELAY_MASK (0xffff << 0) + +/* XXX: all bits reserved */ +#define _MIPIA_AUTOPWG (VLV_DISPLAY_BASE + 0x611a0) + +/* MIPI DSI Controller and D-PHY registers */ + +#define _MIPIA_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb000) +#define _MIPIC_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb800) +#define MIPI_DEVICE_READY(port) _MMIO_MIPI(port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY) +#define BUS_POSSESSION (1 << 3) /* set to give bus to receiver */ +#define ULPS_STATE_MASK (3 << 1) +#define ULPS_STATE_ENTER (2 << 1) +#define ULPS_STATE_EXIT (1 << 1) +#define ULPS_STATE_NORMAL_OPERATION (0 << 1) +#define DEVICE_READY (1 << 0) + +#define _MIPIA_INTR_STAT (dev_priv->mipi_mmio_base + 0xb004) +#define _MIPIC_INTR_STAT (dev_priv->mipi_mmio_base + 0xb804) +#define MIPI_INTR_STAT(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT) +#define _MIPIA_INTR_EN (dev_priv->mipi_mmio_base + 0xb008) +#define _MIPIC_INTR_EN (dev_priv->mipi_mmio_base + 0xb808) +#define MIPI_INTR_EN(port) _MMIO_MIPI(port, _MIPIA_INTR_EN, _MIPIC_INTR_EN) +#define TEARING_EFFECT (1 << 31) +#define SPL_PKT_SENT_INTERRUPT (1 << 30) +#define GEN_READ_DATA_AVAIL (1 << 29) +#define LP_GENERIC_WR_FIFO_FULL (1 << 28) +#define HS_GENERIC_WR_FIFO_FULL (1 << 27) +#define RX_PROT_VIOLATION (1 << 26) +#define RX_INVALID_TX_LENGTH (1 << 25) +#define ACK_WITH_NO_ERROR (1 << 24) +#define TURN_AROUND_ACK_TIMEOUT (1 << 23) +#define LP_RX_TIMEOUT (1 << 22) +#define HS_TX_TIMEOUT (1 << 21) +#define DPI_FIFO_UNDERRUN (1 << 20) +#define LOW_CONTENTION (1 << 19) +#define HIGH_CONTENTION (1 << 18) +#define TXDSI_VC_ID_INVALID (1 << 17) +#define TXDSI_DATA_TYPE_NOT_RECOGNISED (1 << 16) +#define TXCHECKSUM_ERROR (1 << 15) +#define TXECC_MULTIBIT_ERROR (1 << 14) +#define TXECC_SINGLE_BIT_ERROR (1 << 13) +#define TXFALSE_CONTROL_ERROR (1 << 12) +#define RXDSI_VC_ID_INVALID (1 << 11) +#define RXDSI_DATA_TYPE_NOT_REGOGNISED (1 << 10) +#define RXCHECKSUM_ERROR (1 << 9) +#define RXECC_MULTIBIT_ERROR (1 << 8) +#define RXECC_SINGLE_BIT_ERROR (1 << 7) +#define RXFALSE_CONTROL_ERROR (1 << 6) +#define RXHS_RECEIVE_TIMEOUT_ERROR (1 << 5) +#define RX_LP_TX_SYNC_ERROR (1 << 4) +#define RXEXCAPE_MODE_ENTRY_ERROR (1 << 3) +#define RXEOT_SYNC_ERROR (1 << 2) +#define RXSOT_SYNC_ERROR (1 << 1) +#define RXSOT_ERROR (1 << 0) + +#define _MIPIA_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb00c) +#define _MIPIC_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb80c) +#define MIPI_DSI_FUNC_PRG(port) _MMIO_MIPI(port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG) +#define CMD_MODE_DATA_WIDTH_MASK (7 << 13) +#define CMD_MODE_NOT_SUPPORTED (0 << 13) +#define CMD_MODE_DATA_WIDTH_16_BIT (1 << 13) +#define CMD_MODE_DATA_WIDTH_9_BIT (2 << 13) +#define CMD_MODE_DATA_WIDTH_8_BIT (3 << 13) +#define CMD_MODE_DATA_WIDTH_OPTION1 (4 << 13) +#define CMD_MODE_DATA_WIDTH_OPTION2 (5 << 13) +#define VID_MODE_FORMAT_MASK (0xf << 7) +#define VID_MODE_NOT_SUPPORTED (0 << 7) +#define VID_MODE_FORMAT_RGB565 (1 << 7) +#define VID_MODE_FORMAT_RGB666_PACKED (2 << 7) +#define VID_MODE_FORMAT_RGB666 (3 << 7) +#define VID_MODE_FORMAT_RGB888 (4 << 7) +#define CMD_MODE_CHANNEL_NUMBER_SHIFT 5 +#define CMD_MODE_CHANNEL_NUMBER_MASK (3 << 5) +#define VID_MODE_CHANNEL_NUMBER_SHIFT 3 +#define VID_MODE_CHANNEL_NUMBER_MASK (3 << 3) +#define DATA_LANES_PRG_REG_SHIFT 0 +#define DATA_LANES_PRG_REG_MASK (7 << 0) + +#define _MIPIA_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb010) +#define _MIPIC_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb810) +#define MIPI_HS_TX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT) +#define HIGH_SPEED_TX_TIMEOUT_COUNTER_MASK 0xffffff + +#define _MIPIA_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb014) +#define _MIPIC_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb814) +#define MIPI_LP_RX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT) +#define LOW_POWER_RX_TIMEOUT_COUNTER_MASK 0xffffff + +#define _MIPIA_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb018) +#define _MIPIC_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb818) +#define MIPI_TURN_AROUND_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT) +#define TURN_AROUND_TIMEOUT_MASK 0x3f + +#define _MIPIA_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb01c) +#define _MIPIC_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb81c) +#define MIPI_DEVICE_RESET_TIMER(port) _MMIO_MIPI(port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER) +#define DEVICE_RESET_TIMER_MASK 0xffff + +#define _MIPIA_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb020) +#define _MIPIC_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb820) +#define MIPI_DPI_RESOLUTION(port) _MMIO_MIPI(port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION) +#define VERTICAL_ADDRESS_SHIFT 16 +#define VERTICAL_ADDRESS_MASK (0xffff << 16) +#define HORIZONTAL_ADDRESS_SHIFT 0 +#define HORIZONTAL_ADDRESS_MASK 0xffff + +#define _MIPIA_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb024) +#define _MIPIC_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb824) +#define MIPI_DBI_FIFO_THROTTLE(port) _MMIO_MIPI(port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE) +#define DBI_FIFO_EMPTY_HALF (0 << 0) +#define DBI_FIFO_EMPTY_QUARTER (1 << 0) +#define DBI_FIFO_EMPTY_7_LOCATIONS (2 << 0) + +/* regs below are bits 15:0 */ +#define _MIPIA_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb028) +#define _MIPIC_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb828) +#define MIPI_HSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT) + +#define _MIPIA_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb02c) +#define _MIPIC_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb82c) +#define MIPI_HBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT) + +#define _MIPIA_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb030) +#define _MIPIC_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb830) +#define MIPI_HFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT) + +#define _MIPIA_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb034) +#define _MIPIC_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb834) +#define MIPI_HACTIVE_AREA_COUNT(port) _MMIO_MIPI(port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT) + +#define _MIPIA_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb038) +#define _MIPIC_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb838) +#define MIPI_VSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT) + +#define _MIPIA_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb03c) +#define _MIPIC_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb83c) +#define MIPI_VBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT) + +#define _MIPIA_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb040) +#define _MIPIC_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb840) +#define MIPI_VFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT) + +#define _MIPIA_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb044) +#define _MIPIC_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb844) +#define MIPI_HIGH_LOW_SWITCH_COUNT(port) _MMIO_MIPI(port, _MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT) + +#define _MIPIA_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb048) +#define _MIPIC_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb848) +#define MIPI_DPI_CONTROL(port) _MMIO_MIPI(port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL) +#define DPI_LP_MODE (1 << 6) +#define BACKLIGHT_OFF (1 << 5) +#define BACKLIGHT_ON (1 << 4) +#define COLOR_MODE_OFF (1 << 3) +#define COLOR_MODE_ON (1 << 2) +#define TURN_ON (1 << 1) +#define SHUTDOWN (1 << 0) + +#define _MIPIA_DPI_DATA (dev_priv->mipi_mmio_base + 0xb04c) +#define _MIPIC_DPI_DATA (dev_priv->mipi_mmio_base + 0xb84c) +#define MIPI_DPI_DATA(port) _MMIO_MIPI(port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA) +#define COMMAND_BYTE_SHIFT 0 +#define COMMAND_BYTE_MASK (0x3f << 0) + +#define _MIPIA_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb050) +#define _MIPIC_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb850) +#define MIPI_INIT_COUNT(port) _MMIO_MIPI(port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT) +#define MASTER_INIT_TIMER_SHIFT 0 +#define MASTER_INIT_TIMER_MASK (0xffff << 0) + +#define _MIPIA_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb054) +#define _MIPIC_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb854) +#define MIPI_MAX_RETURN_PKT_SIZE(port) _MMIO_MIPI(port, \ + _MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE) +#define MAX_RETURN_PKT_SIZE_SHIFT 0 +#define MAX_RETURN_PKT_SIZE_MASK (0x3ff << 0) + +#define _MIPIA_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb058) +#define _MIPIC_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb858) +#define MIPI_VIDEO_MODE_FORMAT(port) _MMIO_MIPI(port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT) +#define RANDOM_DPI_DISPLAY_RESOLUTION (1 << 4) +#define DISABLE_VIDEO_BTA (1 << 3) +#define IP_TG_CONFIG (1 << 2) +#define VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE (1 << 0) +#define VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS (2 << 0) +#define VIDEO_MODE_BURST (3 << 0) + +#define _MIPIA_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb05c) +#define _MIPIC_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb85c) +#define MIPI_EOT_DISABLE(port) _MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE) +#define BXT_DEFEATURE_DPI_FIFO_CTR (1 << 9) +#define BXT_DPHY_DEFEATURE_EN (1 << 8) +#define LP_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 7) +#define HS_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 6) +#define LOW_CONTENTION_RECOVERY_DISABLE (1 << 5) +#define HIGH_CONTENTION_RECOVERY_DISABLE (1 << 4) +#define TXDSI_TYPE_NOT_RECOGNISED_ERROR_RECOVERY_DISABLE (1 << 3) +#define TXECC_MULTIBIT_ERROR_RECOVERY_DISABLE (1 << 2) +#define CLOCKSTOP (1 << 1) +#define EOT_DISABLE (1 << 0) + +#define _MIPIA_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb060) +#define _MIPIC_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb860) +#define MIPI_LP_BYTECLK(port) _MMIO_MIPI(port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK) +#define LP_BYTECLK_SHIFT 0 +#define LP_BYTECLK_MASK (0xffff << 0) + +#define _MIPIA_TLPX_TIME_COUNT (dev_priv->mipi_mmio_base + 0xb0a4) +#define _MIPIC_TLPX_TIME_COUNT (dev_priv->mipi_mmio_base + 0xb8a4) +#define MIPI_TLPX_TIME_COUNT(port) _MMIO_MIPI(port, _MIPIA_TLPX_TIME_COUNT, _MIPIC_TLPX_TIME_COUNT) + +#define _MIPIA_CLK_LANE_TIMING (dev_priv->mipi_mmio_base + 0xb098) +#define _MIPIC_CLK_LANE_TIMING (dev_priv->mipi_mmio_base + 0xb898) +#define MIPI_CLK_LANE_TIMING(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_TIMING, _MIPIC_CLK_LANE_TIMING) + +/* bits 31:0 */ +#define _MIPIA_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb064) +#define _MIPIC_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb864) +#define MIPI_LP_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA) + +/* bits 31:0 */ +#define _MIPIA_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb068) +#define _MIPIC_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb868) +#define MIPI_HS_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA) + +#define _MIPIA_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb06c) +#define _MIPIC_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb86c) +#define MIPI_LP_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL) +#define _MIPIA_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb070) +#define _MIPIC_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb870) +#define MIPI_HS_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL) +#define LONG_PACKET_WORD_COUNT_SHIFT 8 +#define LONG_PACKET_WORD_COUNT_MASK (0xffff << 8) +#define SHORT_PACKET_PARAM_SHIFT 8 +#define SHORT_PACKET_PARAM_MASK (0xffff << 8) +#define VIRTUAL_CHANNEL_SHIFT 6 +#define VIRTUAL_CHANNEL_MASK (3 << 6) +#define DATA_TYPE_SHIFT 0 +#define DATA_TYPE_MASK (0x3f << 0) +/* data type values, see include/video/mipi_display.h */ + +#define _MIPIA_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb074) +#define _MIPIC_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb874) +#define MIPI_GEN_FIFO_STAT(port) _MMIO_MIPI(port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT) +#define DPI_FIFO_EMPTY (1 << 28) +#define DBI_FIFO_EMPTY (1 << 27) +#define LP_CTRL_FIFO_EMPTY (1 << 26) +#define LP_CTRL_FIFO_HALF_EMPTY (1 << 25) +#define LP_CTRL_FIFO_FULL (1 << 24) +#define HS_CTRL_FIFO_EMPTY (1 << 18) +#define HS_CTRL_FIFO_HALF_EMPTY (1 << 17) +#define HS_CTRL_FIFO_FULL (1 << 16) +#define LP_DATA_FIFO_EMPTY (1 << 10) +#define LP_DATA_FIFO_HALF_EMPTY (1 << 9) +#define LP_DATA_FIFO_FULL (1 << 8) +#define HS_DATA_FIFO_EMPTY (1 << 2) +#define HS_DATA_FIFO_HALF_EMPTY (1 << 1) +#define HS_DATA_FIFO_FULL (1 << 0) + +#define _MIPIA_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb078) +#define _MIPIC_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb878) +#define MIPI_HS_LP_DBI_ENABLE(port) _MMIO_MIPI(port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE) +#define DBI_HS_LP_MODE_MASK (1 << 0) +#define DBI_LP_MODE (1 << 0) +#define DBI_HS_MODE (0 << 0) + +#define _MIPIA_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb080) +#define _MIPIC_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb880) +#define MIPI_DPHY_PARAM(port) _MMIO_MIPI(port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM) +#define EXIT_ZERO_COUNT_SHIFT 24 +#define EXIT_ZERO_COUNT_MASK (0x3f << 24) +#define TRAIL_COUNT_SHIFT 16 +#define TRAIL_COUNT_MASK (0x1f << 16) +#define CLK_ZERO_COUNT_SHIFT 8 +#define CLK_ZERO_COUNT_MASK (0xff << 8) +#define PREPARE_COUNT_SHIFT 0 +#define PREPARE_COUNT_MASK (0x3f << 0) + +#define _MIPIA_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb084) +#define _MIPIC_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb884) +#define MIPI_DBI_BW_CTRL(port) _MMIO_MIPI(port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL) + +#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb088) +#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb888) +#define MIPI_CLK_LANE_SWITCH_TIME_CNT(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT) +#define LP_HS_SSW_CNT_SHIFT 16 +#define LP_HS_SSW_CNT_MASK (0xffff << 16) +#define HS_LP_PWR_SW_CNT_SHIFT 0 +#define HS_LP_PWR_SW_CNT_MASK (0xffff << 0) + +#define _MIPIA_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb08c) +#define _MIPIC_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb88c) +#define MIPI_STOP_STATE_STALL(port) _MMIO_MIPI(port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL) +#define STOP_STATE_STALL_COUNTER_SHIFT 0 +#define STOP_STATE_STALL_COUNTER_MASK (0xff << 0) + +#define _MIPIA_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb090) +#define _MIPIC_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb890) +#define MIPI_INTR_STAT_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1) +#define _MIPIA_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb094) +#define _MIPIC_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb894) +#define MIPI_INTR_EN_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1) +#define RX_CONTENTION_DETECTED (1 << 0) + +/* XXX: only pipe A ?!? */ +#define MIPIA_DBI_TYPEC_CTRL (dev_priv->mipi_mmio_base + 0xb100) +#define DBI_TYPEC_ENABLE (1 << 31) +#define DBI_TYPEC_WIP (1 << 30) +#define DBI_TYPEC_OPTION_SHIFT 28 +#define DBI_TYPEC_OPTION_MASK (3 << 28) +#define DBI_TYPEC_FREQ_SHIFT 24 +#define DBI_TYPEC_FREQ_MASK (0xf << 24) +#define DBI_TYPEC_OVERRIDE (1 << 8) +#define DBI_TYPEC_OVERRIDE_COUNTER_SHIFT 0 +#define DBI_TYPEC_OVERRIDE_COUNTER_MASK (0xff << 0) + +/* MIPI adapter registers */ + +#define _MIPIA_CTRL (dev_priv->mipi_mmio_base + 0xb104) +#define _MIPIC_CTRL (dev_priv->mipi_mmio_base + 0xb904) +#define MIPI_CTRL(port) _MMIO_MIPI(port, _MIPIA_CTRL, _MIPIC_CTRL) +#define ESCAPE_CLOCK_DIVIDER_SHIFT 5 /* A only */ +#define ESCAPE_CLOCK_DIVIDER_MASK (3 << 5) +#define ESCAPE_CLOCK_DIVIDER_1 (0 << 5) +#define ESCAPE_CLOCK_DIVIDER_2 (1 << 5) +#define ESCAPE_CLOCK_DIVIDER_4 (2 << 5) +#define READ_REQUEST_PRIORITY_SHIFT 3 +#define READ_REQUEST_PRIORITY_MASK (3 << 3) +#define READ_REQUEST_PRIORITY_LOW (0 << 3) +#define READ_REQUEST_PRIORITY_HIGH (3 << 3) +#define RGB_FLIP_TO_BGR (1 << 2) + +#define BXT_PIPE_SELECT_SHIFT 7 +#define BXT_PIPE_SELECT_MASK (7 << 7) +#define BXT_PIPE_SELECT(pipe) ((pipe) << 7) +#define GLK_PHY_STATUS_PORT_READY (1 << 31) /* RO */ +#define GLK_ULPS_NOT_ACTIVE (1 << 30) /* RO */ +#define GLK_MIPIIO_RESET_RELEASED (1 << 28) +#define GLK_CLOCK_LANE_STOP_STATE (1 << 27) /* RO */ +#define GLK_DATA_LANE_STOP_STATE (1 << 26) /* RO */ +#define GLK_LP_WAKE (1 << 22) +#define GLK_LP11_LOW_PWR_MODE (1 << 21) +#define GLK_LP00_LOW_PWR_MODE (1 << 20) +#define GLK_FIREWALL_ENABLE (1 << 16) +#define BXT_PIXEL_OVERLAP_CNT_MASK (0xf << 10) +#define BXT_PIXEL_OVERLAP_CNT_SHIFT 10 +#define BXT_DSC_ENABLE (1 << 3) +#define BXT_RGB_FLIP (1 << 2) +#define GLK_MIPIIO_PORT_POWERED (1 << 1) /* RO */ +#define GLK_MIPIIO_ENABLE (1 << 0) + +#define _MIPIA_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb108) +#define _MIPIC_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb908) +#define MIPI_DATA_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS) +#define DATA_MEM_ADDRESS_SHIFT 5 +#define DATA_MEM_ADDRESS_MASK (0x7ffffff << 5) +#define DATA_VALID (1 << 0) + +#define _MIPIA_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb10c) +#define _MIPIC_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb90c) +#define MIPI_DATA_LENGTH(port) _MMIO_MIPI(port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH) +#define DATA_LENGTH_SHIFT 0 +#define DATA_LENGTH_MASK (0xfffff << 0) + +#define _MIPIA_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb110) +#define _MIPIC_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb910) +#define MIPI_COMMAND_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS) +#define COMMAND_MEM_ADDRESS_SHIFT 5 +#define COMMAND_MEM_ADDRESS_MASK (0x7ffffff << 5) +#define AUTO_PWG_ENABLE (1 << 2) +#define MEMORY_WRITE_DATA_FROM_PIPE_RENDERING (1 << 1) +#define COMMAND_VALID (1 << 0) + +#define _MIPIA_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb114) +#define _MIPIC_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb914) +#define MIPI_COMMAND_LENGTH(port) _MMIO_MIPI(port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH) +#define COMMAND_LENGTH_SHIFT(n) (8 * (n)) /* n: 0...3 */ +#define COMMAND_LENGTH_MASK(n) (0xff << (8 * (n))) + +#define _MIPIA_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb118) +#define _MIPIC_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb918) +#define MIPI_READ_DATA_RETURN(port, n) _MMIO(_MIPI(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) + 4 * (n)) /* n: 0...7 */ + +#define _MIPIA_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb138) +#define _MIPIC_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb938) +#define MIPI_READ_DATA_VALID(port) _MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID) +#define READ_DATA_VALID(n) (1 << (n)) + +#endif /* __VLV_DSI_REGS_H__ */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c index 8a248003dfae..ce91b23385cf 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c @@ -4,6 +4,8 @@ * Copyright © 2016 Intel Corporation */ +#include <drm/drm_cache.h> + #include "display/intel_frontbuffer.h" #include "i915_drv.h" diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index 00327b750fbb..9ae294eb7fb4 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -67,6 +67,7 @@ #include <linux/log2.h> #include <linux/nospec.h> +#include <drm/drm_cache.h> #include <drm/drm_syncobj.h> #include "gt/gen6_ppgtt.h" @@ -79,6 +80,7 @@ #include "pxp/intel_pxp.h" +#include "i915_file_private.h" #include "i915_gem_context.h" #include "i915_trace.h" #include "i915_user_extensions.h" @@ -343,6 +345,20 @@ static int proto_context_register(struct drm_i915_file_private *fpriv, return ret; } +static struct i915_address_space * +i915_gem_vm_lookup(struct drm_i915_file_private *file_priv, u32 id) +{ + struct i915_address_space *vm; + + xa_lock(&file_priv->vm_xa); + vm = xa_load(&file_priv->vm_xa, id); + if (vm) + kref_get(&vm->ref); + xa_unlock(&file_priv->vm_xa); + + return vm; +} + static int set_proto_ctx_vm(struct drm_i915_file_private *fpriv, struct i915_gem_proto_context *pc, const struct drm_i915_gem_context_param *args) @@ -571,10 +587,6 @@ set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, struct intel_engine_cs **siblings = NULL; intel_engine_mask_t prev_mask; - /* FIXME: This is NIY for execlists */ - if (!(intel_uc_uses_guc_submission(&to_gt(i915)->uc))) - return -ENODEV; - if (get_user(slot, &ext->engine_index)) return -EFAULT; @@ -584,6 +596,13 @@ set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, if (get_user(num_siblings, &ext->num_siblings)) return -EFAULT; + if (!intel_uc_uses_guc_submission(&to_gt(i915)->uc) && + num_siblings != 1) { + drm_dbg(&i915->drm, "Only 1 sibling (%d) supported in non-GuC mode\n", + num_siblings); + return -EINVAL; + } + if (slot >= set->num_engines) { drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", slot, set->num_engines); @@ -651,6 +670,16 @@ set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, goto out_err; } + /* + * We don't support breadcrumb handshake on these + * classes + */ + if (siblings[n]->class == RENDER_CLASS || + siblings[n]->class == COMPUTE_CLASS) { + err = -EINVAL; + goto out_err; + } + if (n) { if (prev_engine.engine_class != ci.engine_class) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.h b/drivers/gpu/drm/i915/gem/i915_gem_context.h index babfecb17ad1..e5b0f66ea1fe 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.h @@ -174,7 +174,7 @@ i915_gem_context_get_eb_vm(struct i915_gem_context *ctx) vm = ctx->vm; if (!vm) - vm = &ctx->i915->ggtt.vm; + vm = &to_gt(ctx->i915)->ggtt->vm; vm = i915_vm_get(vm); return vm; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_create.c b/drivers/gpu/drm/i915/gem/i915_gem_create.c index 9402d4bf4ffc..c6eb023d3d86 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_create.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_create.c @@ -3,12 +3,15 @@ * Copyright © 2020 Intel Corporation */ +#include <drm/drm_fourcc.h> + #include "gem/i915_gem_ioctls.h" #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" #include "pxp/intel_pxp.h" #include "i915_drv.h" +#include "i915_gem_create.h" #include "i915_trace.h" #include "i915_user_extensions.h" diff --git a/drivers/gpu/drm/i915/gem/i915_gem_create.h b/drivers/gpu/drm/i915/gem/i915_gem_create.h new file mode 100644 index 000000000000..9536aa906001 --- /dev/null +++ b/drivers/gpu/drm/i915/gem/i915_gem_create.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_GEM_CREATE_H__ +#define __I915_GEM_CREATE_H__ + +struct drm_file; +struct drm_device; +struct drm_mode_create_dumb; + +int i915_gem_dumb_create(struct drm_file *file_priv, + struct drm_device *dev, + struct drm_mode_create_dumb *args); + +#endif /* __I915_GEM_CREATE_H__ */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c index 1b526039a60d..13917231ae81 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c @@ -11,6 +11,7 @@ #include <asm/smp.h> +#include "gem/i915_gem_dmabuf.h" #include "i915_drv.h" #include "i915_gem_object.h" #include "i915_scatterlist.h" @@ -74,7 +75,8 @@ static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment, kfree(sg); } -static int i915_gem_dmabuf_vmap(struct dma_buf *dma_buf, struct dma_buf_map *map) +static int i915_gem_dmabuf_vmap(struct dma_buf *dma_buf, + struct iosys_map *map) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); void *vaddr; @@ -83,12 +85,13 @@ static int i915_gem_dmabuf_vmap(struct dma_buf *dma_buf, struct dma_buf_map *map if (IS_ERR(vaddr)) return PTR_ERR(vaddr); - dma_buf_map_set_vaddr(map, vaddr); + iosys_map_set_vaddr(map, vaddr); return 0; } -static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, struct dma_buf_map *map) +static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, + struct iosys_map *map) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.h b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.h new file mode 100644 index 000000000000..6e0405d47ce1 --- /dev/null +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __I915_GEM_DMABUF_H__ +#define __I915_GEM_DMABUF_H__ + +struct drm_gem_object; +struct drm_device; +struct dma_buf; + +struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf); + +struct dma_buf *i915_gem_prime_export(struct drm_gem_object *gem_obj, int flags); + +#endif /* __I915_GEM_DMABUF_H__ */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c index 26532c07d467..3e5d6057b3ef 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c @@ -9,12 +9,13 @@ #include "i915_drv.h" #include "i915_gem_clflush.h" +#include "i915_gem_domain.h" #include "i915_gem_gtt.h" #include "i915_gem_ioctls.h" -#include "i915_gem_object.h" -#include "i915_vma.h" #include "i915_gem_lmem.h" #include "i915_gem_mman.h" +#include "i915_gem_object.h" +#include "i915_vma.h" static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.h b/drivers/gpu/drm/i915/gem/i915_gem_domain.h new file mode 100644 index 000000000000..9622df962bfc --- /dev/null +++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __I915_GEM_DOMAIN_H__ +#define __I915_GEM_DOMAIN_H__ + +struct drm_i915_gem_object; +enum i915_cache_level; + +int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, + enum i915_cache_level cache_level); + +#endif /* __I915_GEM_DOMAIN_H__ */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 1736efa43339..d42f437149c9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -23,13 +23,15 @@ #include "pxp/intel_pxp.h" +#include "i915_cmd_parser.h" #include "i915_drv.h" +#include "i915_file_private.h" #include "i915_gem_clflush.h" #include "i915_gem_context.h" +#include "i915_gem_evict.h" #include "i915_gem_ioctls.h" #include "i915_trace.h" #include "i915_user_extensions.h" -#include "i915_vma_snapshot.h" struct eb_vma { struct i915_vma *vma; @@ -441,7 +443,7 @@ eb_pin_vma(struct i915_execbuffer *eb, else pin_flags = entry->offset & PIN_OFFSET_MASK; - pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED; + pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED | PIN_VALIDATE; if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT)) pin_flags |= PIN_GLOBAL; @@ -459,17 +461,15 @@ eb_pin_vma(struct i915_execbuffer *eb, entry->pad_to_size, entry->alignment, eb_pin_flags(entry, ev->flags) | - PIN_USER | PIN_NOEVICT); + PIN_USER | PIN_NOEVICT | PIN_VALIDATE); if (unlikely(err)) return err; } if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { err = i915_vma_pin_fence(vma); - if (unlikely(err)) { - i915_vma_unpin(vma); + if (unlikely(err)) return err; - } if (vma->fence) ev->flags |= __EXEC_OBJECT_HAS_FENCE; @@ -485,13 +485,9 @@ eb_pin_vma(struct i915_execbuffer *eb, static inline void eb_unreserve_vma(struct eb_vma *ev) { - if (!(ev->flags & __EXEC_OBJECT_HAS_PIN)) - return; - if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE)) __i915_vma_unpin_fence(ev->vma); - __i915_vma_unpin(ev->vma); ev->flags &= ~__EXEC_OBJECT_RESERVED; } @@ -673,10 +669,8 @@ static int eb_reserve_vma(struct i915_execbuffer *eb, if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { err = i915_vma_pin_fence(vma); - if (unlikely(err)) { - i915_vma_unpin(vma); + if (unlikely(err)) return err; - } if (vma->fence) ev->flags |= __EXEC_OBJECT_HAS_FENCE; @@ -688,85 +682,95 @@ static int eb_reserve_vma(struct i915_execbuffer *eb, return 0; } -static int eb_reserve(struct i915_execbuffer *eb) +static bool eb_unbind(struct i915_execbuffer *eb, bool force) { const unsigned int count = eb->buffer_count; - unsigned int pin_flags = PIN_USER | PIN_NONBLOCK; + unsigned int i; struct list_head last; + bool unpinned = false; + + /* Resort *all* the objects into priority order */ + INIT_LIST_HEAD(&eb->unbound); + INIT_LIST_HEAD(&last); + + for (i = 0; i < count; i++) { + struct eb_vma *ev = &eb->vma[i]; + unsigned int flags = ev->flags; + + if (!force && flags & EXEC_OBJECT_PINNED && + flags & __EXEC_OBJECT_HAS_PIN) + continue; + + unpinned = true; + eb_unreserve_vma(ev); + + if (flags & EXEC_OBJECT_PINNED) + /* Pinned must have their slot */ + list_add(&ev->bind_link, &eb->unbound); + else if (flags & __EXEC_OBJECT_NEEDS_MAP) + /* Map require the lowest 256MiB (aperture) */ + list_add_tail(&ev->bind_link, &eb->unbound); + else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) + /* Prioritise 4GiB region for restricted bo */ + list_add(&ev->bind_link, &last); + else + list_add_tail(&ev->bind_link, &last); + } + + list_splice_tail(&last, &eb->unbound); + return unpinned; +} + +static int eb_reserve(struct i915_execbuffer *eb) +{ struct eb_vma *ev; - unsigned int i, pass; + unsigned int pass; int err = 0; + bool unpinned; /* * Attempt to pin all of the buffers into the GTT. - * This is done in 3 phases: + * This is done in 2 phases: * - * 1a. Unbind all objects that do not match the GTT constraints for - * the execbuffer (fenceable, mappable, alignment etc). - * 1b. Increment pin count for already bound objects. - * 2. Bind new objects. - * 3. Decrement pin count. + * 1. Unbind all objects that do not match the GTT constraints for + * the execbuffer (fenceable, mappable, alignment etc). + * 2. Bind new objects. * * This avoid unnecessary unbinding of later objects in order to make * room for the earlier objects *unless* we need to defragment. + * + * Defragmenting is skipped if all objects are pinned at a fixed location. */ - pass = 0; - do { - list_for_each_entry(ev, &eb->unbound, bind_link) { - err = eb_reserve_vma(eb, ev, pin_flags); - if (err) - break; - } - if (err != -ENOSPC) - return err; + for (pass = 0; pass <= 2; pass++) { + int pin_flags = PIN_USER | PIN_VALIDATE; - /* Resort *all* the objects into priority order */ - INIT_LIST_HEAD(&eb->unbound); - INIT_LIST_HEAD(&last); - for (i = 0; i < count; i++) { - unsigned int flags; + if (pass == 0) + pin_flags |= PIN_NONBLOCK; - ev = &eb->vma[i]; - flags = ev->flags; - if (flags & EXEC_OBJECT_PINNED && - flags & __EXEC_OBJECT_HAS_PIN) - continue; + if (pass >= 1) + unpinned = eb_unbind(eb, pass == 2); - eb_unreserve_vma(ev); - - if (flags & EXEC_OBJECT_PINNED) - /* Pinned must have their slot */ - list_add(&ev->bind_link, &eb->unbound); - else if (flags & __EXEC_OBJECT_NEEDS_MAP) - /* Map require the lowest 256MiB (aperture) */ - list_add_tail(&ev->bind_link, &eb->unbound); - else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) - /* Prioritise 4GiB region for restricted bo */ - list_add(&ev->bind_link, &last); - else - list_add_tail(&ev->bind_link, &last); - } - list_splice_tail(&last, &eb->unbound); - - switch (pass++) { - case 0: - break; - - case 1: - /* Too fragmented, unbind everything and retry */ - mutex_lock(&eb->context->vm->mutex); - err = i915_gem_evict_vm(eb->context->vm); - mutex_unlock(&eb->context->vm->mutex); + if (pass == 2) { + err = mutex_lock_interruptible(&eb->context->vm->mutex); + if (!err) { + err = i915_gem_evict_vm(eb->context->vm, &eb->ww); + mutex_unlock(&eb->context->vm->mutex); + } if (err) return err; - break; + } - default: - return -ENOSPC; + list_for_each_entry(ev, &eb->unbound, bind_link) { + err = eb_reserve_vma(eb, ev, pin_flags); + if (err) + break; } - pin_flags = PIN_USER; - } while (1); + if (err != -ENOSPC) + break; + } + + return err; } static int eb_select_context(struct i915_execbuffer *eb) @@ -1095,7 +1099,7 @@ static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) { struct drm_i915_private *i915 = container_of(cache, struct i915_execbuffer, reloc_cache)->i915; - return &i915->ggtt; + return to_gt(i915)->ggtt; } static void reloc_cache_unmap(struct reloc_cache *cache) @@ -1214,10 +1218,11 @@ static void *reloc_kmap(struct drm_i915_gem_object *obj, return vaddr; } -static void *reloc_iomap(struct drm_i915_gem_object *obj, +static void *reloc_iomap(struct i915_vma *batch, struct i915_execbuffer *eb, unsigned long page) { + struct drm_i915_gem_object *obj = batch->obj; struct reloc_cache *cache = &eb->reloc_cache; struct i915_ggtt *ggtt = cache_to_ggtt(cache); unsigned long offset; @@ -1227,7 +1232,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj, intel_gt_flush_ggtt_writes(ggtt->vm.gt); io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); } else { - struct i915_vma *vma; + struct i915_vma *vma = ERR_PTR(-ENODEV); int err; if (i915_gem_object_is_tiled(obj)) @@ -1240,10 +1245,23 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj, if (err) return ERR_PTR(err); - vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0, - PIN_MAPPABLE | - PIN_NONBLOCK /* NOWARN */ | - PIN_NOEVICT); + /* + * i915_gem_object_ggtt_pin_ww may attempt to remove the batch + * VMA from the object list because we no longer pin. + * + * Only attempt to pin the batch buffer to ggtt if the current batch + * is not inside ggtt, or the batch buffer is not misplaced. + */ + if (!i915_is_ggtt(batch->vm)) { + vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0, + PIN_MAPPABLE | + PIN_NONBLOCK /* NOWARN */ | + PIN_NOEVICT); + } else if (i915_vma_is_map_and_fenceable(batch)) { + __i915_vma_pin(batch); + vma = batch; + } + if (vma == ERR_PTR(-EDEADLK)) return vma; @@ -1281,7 +1299,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj, return vaddr; } -static void *reloc_vaddr(struct drm_i915_gem_object *obj, +static void *reloc_vaddr(struct i915_vma *vma, struct i915_execbuffer *eb, unsigned long page) { @@ -1293,9 +1311,9 @@ static void *reloc_vaddr(struct drm_i915_gem_object *obj, } else { vaddr = NULL; if ((cache->vaddr & KMAP) == 0) - vaddr = reloc_iomap(obj, eb, page); + vaddr = reloc_iomap(vma, eb, page); if (!vaddr) - vaddr = reloc_kmap(obj, cache, page); + vaddr = reloc_kmap(vma->obj, cache, page); } return vaddr; @@ -1336,7 +1354,7 @@ relocate_entry(struct i915_vma *vma, void *vaddr; repeat: - vaddr = reloc_vaddr(vma->obj, eb, + vaddr = reloc_vaddr(vma, eb, offset >> PAGE_SHIFT); if (IS_ERR(vaddr)) return PTR_ERR(vaddr); @@ -1411,7 +1429,7 @@ eb_relocate_entry(struct i915_execbuffer *eb, mutex_lock(&vma->vm->mutex); err = i915_vma_bind(target->vma, target->vma->obj->cache_level, - PIN_GLOBAL, NULL); + PIN_GLOBAL, NULL, NULL); mutex_unlock(&vma->vm->mutex); reloc_cache_remap(&eb->reloc_cache, ev->vma->obj); if (err) @@ -1941,7 +1959,6 @@ static void eb_capture_stage(struct i915_execbuffer *eb) { const unsigned int count = eb->buffer_count; unsigned int i = count, j; - struct i915_vma_snapshot *vsnap; while (i--) { struct eb_vma *ev = &eb->vma[i]; @@ -1951,11 +1968,6 @@ static void eb_capture_stage(struct i915_execbuffer *eb) if (!(flags & EXEC_OBJECT_CAPTURE)) continue; - vsnap = i915_vma_snapshot_alloc(GFP_KERNEL); - if (!vsnap) - continue; - - i915_vma_snapshot_init(vsnap, vma, "user"); for_each_batch_create_order(eb, j) { struct i915_capture_list *capture; @@ -1964,10 +1976,9 @@ static void eb_capture_stage(struct i915_execbuffer *eb) continue; capture->next = eb->capture_lists[j]; - capture->vma_snapshot = i915_vma_snapshot_get(vsnap); + capture->vma_res = i915_vma_resource_get(vma->resource); eb->capture_lists[j] = capture; } - i915_vma_snapshot_put(vsnap); } } @@ -2198,7 +2209,7 @@ shadow_batch_pin(struct i915_execbuffer *eb, if (IS_ERR(vma)) return vma; - err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags); + err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags | PIN_VALIDATE); if (err) return ERR_PTR(err); @@ -2212,7 +2223,7 @@ static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i9 * batch" bit. Hence we need to pin secure batches into the global gtt. * hsw should have this fixed, but bdw mucks it up again. */ if (eb->batch_flags & I915_DISPATCH_SECURE) - return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, 0); + return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, PIN_VALIDATE); return NULL; } @@ -2263,13 +2274,12 @@ static int eb_parse(struct i915_execbuffer *eb) err = i915_gem_object_lock(pool->obj, &eb->ww); if (err) - goto err; + return err; shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER); - if (IS_ERR(shadow)) { - err = PTR_ERR(shadow); - goto err; - } + if (IS_ERR(shadow)) + return PTR_ERR(shadow); + intel_gt_buffer_pool_mark_used(pool); i915_gem_object_set_readonly(shadow->obj); shadow->private = pool; @@ -2281,25 +2291,21 @@ static int eb_parse(struct i915_execbuffer *eb) shadow = shadow_batch_pin(eb, pool->obj, &eb->gt->ggtt->vm, PIN_GLOBAL); - if (IS_ERR(shadow)) { - err = PTR_ERR(shadow); - shadow = trampoline; - goto err_shadow; - } + if (IS_ERR(shadow)) + return PTR_ERR(shadow); + shadow->private = pool; eb->batch_flags |= I915_DISPATCH_SECURE; } batch = eb_dispatch_secure(eb, shadow); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto err_trampoline; - } + if (IS_ERR(batch)) + return PTR_ERR(batch); err = dma_resv_reserve_shared(shadow->obj->base.resv, 1); if (err) - goto err_trampoline; + return err; err = intel_engine_cmd_parser(eb->context->engine, eb->batches[0]->vma, @@ -2307,7 +2313,7 @@ static int eb_parse(struct i915_execbuffer *eb) eb->batch_len[0], shadow, trampoline); if (err) - goto err_unpin_batch; + return err; eb->batches[0] = &eb->vma[eb->buffer_count++]; eb->batches[0]->vma = i915_vma_get(shadow); @@ -2326,17 +2332,6 @@ secure_batch: eb->batches[0]->vma = i915_vma_get(batch); } return 0; - -err_unpin_batch: - if (batch) - i915_vma_unpin(batch); -err_trampoline: - if (trampoline) - i915_vma_unpin(trampoline); -err_shadow: - i915_vma_unpin(shadow); -err: - return err; } static int eb_request_submit(struct i915_execbuffer *eb, @@ -3275,9 +3270,8 @@ eb_requests_create(struct i915_execbuffer *eb, struct dma_fence *in_fence, * _onstack interface. */ if (eb->batches[i]->vma) - i915_vma_snapshot_init_onstack(&eb->requests[i]->batch_snapshot, - eb->batches[i]->vma, - "batch"); + eb->requests[i]->batch_res = + i915_vma_resource_get(eb->batches[i]->vma->resource); if (eb->batch_pool) { GEM_BUG_ON(intel_context_is_parallel(eb->context)); intel_gt_buffer_pool_mark_active(eb->batch_pool, @@ -3462,8 +3456,6 @@ err_request: err_vma: eb_release_vmas(&eb, true); - if (eb.trampoline) - i915_vma_unpin(eb.trampoline); WARN_ON(err == -EDEADLK); i915_gem_ww_ctx_fini(&eb.ww); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index c5150a1ee3d2..c698f95af15f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -10,6 +10,7 @@ #include "i915_drv.h" #include "i915_gem.h" +#include "i915_gem_internal.h" #include "i915_gem_object.h" #include "i915_scatterlist.h" #include "i915_utils.h" diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.h b/drivers/gpu/drm/i915/gem/i915_gem_internal.h new file mode 100644 index 000000000000..6664e06112fc --- /dev/null +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __I915_GEM_INTERNAL_H__ +#define __I915_GEM_INTERNAL_H__ + +#include <linux/types.h> + +struct drm_i915_gem_object; +struct drm_i915_gem_object_ops; +struct drm_i915_private; + +struct drm_i915_gem_object * +i915_gem_object_create_internal(struct drm_i915_private *i915, + phys_addr_t size); +struct drm_i915_gem_object * +__i915_gem_object_create_internal(struct drm_i915_private *i915, + const struct drm_i915_gem_object_ops *ops, + phys_addr_t size); + +#endif /* __I915_GEM_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 1478c02a82cb..c3ea243d414d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -9,10 +9,13 @@ #include <linux/pfn_t.h> #include <linux/sizes.h> +#include <drm/drm_cache.h> + #include "gt/intel_gt.h" #include "gt/intel_gt_requests.h" #include "i915_drv.h" +#include "i915_gem_evict.h" #include "i915_gem_gtt.h" #include "i915_gem_ioctls.h" #include "i915_gem_object.h" @@ -295,7 +298,7 @@ static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) struct drm_device *dev = obj->base.dev; struct drm_i915_private *i915 = to_i915(dev); struct intel_runtime_pm *rpm = &i915->runtime_pm; - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; bool write = area->vm_flags & VM_WRITE; struct i915_gem_ww_ctx ww; intel_wakeref_t wakeref; @@ -358,8 +361,21 @@ retry: vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); } - /* The entire mappable GGTT is pinned? Unexpected! */ - GEM_BUG_ON(vma == ERR_PTR(-ENOSPC)); + /* + * The entire mappable GGTT is pinned? Unexpected! + * Try to evict the object we locked too, as normally we skip it + * due to lack of short term pinning inside execbuf. + */ + if (vma == ERR_PTR(-ENOSPC)) { + ret = mutex_lock_interruptible(&ggtt->vm.mutex); + if (!ret) { + ret = i915_gem_evict_vm(&ggtt->vm, &ww); + mutex_unlock(&ggtt->vm.mutex); + } + if (ret) + goto err_reset; + vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); + } } if (IS_ERR(vma)) { ret = PTR_ERR(vma); @@ -388,16 +404,16 @@ retry: assert_rpm_wakelock_held(rpm); /* Mark as being mmapped into userspace for later revocation */ - mutex_lock(&i915->ggtt.vm.mutex); + mutex_lock(&to_gt(i915)->ggtt->vm.mutex); if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) - list_add(&obj->userfault_link, &i915->ggtt.userfault_list); - mutex_unlock(&i915->ggtt.vm.mutex); + list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list); + mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); /* Track the mmo associated with the fenced vma */ vma->mmo = mmo; if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) - intel_wakeref_auto(&i915->ggtt.userfault_wakeref, + intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref, msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); if (write) { @@ -439,7 +455,7 @@ vm_access(struct vm_area_struct *area, unsigned long addr, return -EACCES; addr -= area->vm_start; - if (addr >= obj->base.size) + if (range_overflows_t(u64, addr, len, obj->base.size)) return -EINVAL; i915_gem_ww_ctx_init(&ww, true); @@ -512,7 +528,7 @@ void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) * wakeref. */ wakeref = intel_runtime_pm_get(&i915->runtime_pm); - mutex_lock(&i915->ggtt.vm.mutex); + mutex_lock(&to_gt(i915)->ggtt->vm.mutex); if (!obj->userfault_count) goto out; @@ -530,7 +546,7 @@ void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) wmb(); out: - mutex_unlock(&i915->ggtt.vm.mutex); + mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); intel_runtime_pm_put(&i915->runtime_pm, wakeref); } @@ -736,13 +752,14 @@ i915_gem_dumb_mmap_offset(struct drm_file *file, u32 handle, u64 *offset) { + struct drm_i915_private *i915 = to_i915(dev); enum i915_mmap_type mmap_type; if (HAS_LMEM(to_i915(dev))) mmap_type = I915_MMAP_TYPE_FIXED; else if (pat_enabled()) mmap_type = I915_MMAP_TYPE_WC; - else if (!i915_ggtt_has_aperture(&to_i915(dev)->ggtt)) + else if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return -ENODEV; else mmap_type = I915_MMAP_TYPE_GTT; @@ -790,7 +807,7 @@ i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, switch (args->flags) { case I915_MMAP_OFFSET_GTT: - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return -ENODEV; type = I915_MMAP_TYPE_GTT; break; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index d87b508b59b1..372bc220faeb 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -24,11 +24,16 @@ #include <linux/sched/mm.h> +#include <drm/drm_cache.h> + #include "display/intel_frontbuffer.h" #include "pxp/intel_pxp.h" + #include "i915_drv.h" +#include "i915_file_private.h" #include "i915_gem_clflush.h" #include "i915_gem_context.h" +#include "i915_gem_dmabuf.h" #include "i915_gem_mman.h" #include "i915_gem_object.h" #include "i915_gem_ttm.h" @@ -267,12 +272,6 @@ void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) if (!list_empty(&obj->vma.list)) { struct i915_vma *vma; - /* - * Note that the vma keeps an object reference while - * it is active, so it *should* not sleep while we - * destroy it. Our debug code errs insits it *might*. - * For the moment, play along. - */ spin_lock(&obj->vma.lock); while ((vma = list_first_entry_or_null(&obj->vma.list, struct i915_vma, @@ -280,7 +279,7 @@ void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) GEM_BUG_ON(vma->obj != obj); spin_unlock(&obj->vma.lock); - __i915_vma_put(vma); + i915_vma_destroy(vma); spin_lock(&obj->vma.lock); } @@ -756,6 +755,18 @@ i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj) return dma_fence_get(i915_gem_to_ttm(obj)->moving); } +void i915_gem_object_set_moving_fence(struct drm_i915_gem_object *obj, + struct dma_fence *fence) +{ + struct dma_fence **moving = &i915_gem_to_ttm(obj)->moving; + + if (*moving == fence) + return; + + dma_fence_put(*moving); + *moving = dma_fence_get(fence); +} + /** * i915_gem_object_wait_moving_fence - Wait for the object's moving fence if any * @obj: The object whose moving fence to wait for. diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index f66d46882ea7..02c37fe4a535 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -459,7 +459,6 @@ i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj); int i915_gem_object_truncate(struct drm_i915_gem_object *obj); -void i915_gem_object_writeback(struct drm_i915_gem_object *obj); /** * i915_gem_object_pin_map - return a contiguous mapping of the entire object @@ -524,6 +523,9 @@ i915_gem_object_finish_access(struct drm_i915_gem_object *obj) struct dma_fence * i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj); +void i915_gem_object_set_moving_fence(struct drm_i915_gem_object *obj, + struct dma_fence *fence); + int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, bool intr); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 0dd107dcecc2..fd54eb8f4826 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -15,6 +15,7 @@ #include "i915_active.h" #include "i915_selftest.h" +#include "i915_vma_resource.h" struct drm_i915_gem_object; struct intel_fronbuffer; @@ -57,10 +58,26 @@ struct drm_i915_gem_object_ops { void (*put_pages)(struct drm_i915_gem_object *obj, struct sg_table *pages); int (*truncate)(struct drm_i915_gem_object *obj); - void (*writeback)(struct drm_i915_gem_object *obj); - int (*shrinker_release_pages)(struct drm_i915_gem_object *obj, - bool no_gpu_wait, - bool should_writeback); + /** + * shrink - Perform further backend specific actions to facilate + * shrinking. + * @obj: The gem object + * @flags: Extra flags to control shrinking behaviour in the backend + * + * Possible values for @flags: + * + * I915_GEM_OBJECT_SHRINK_WRITEBACK - Try to perform writeback of the + * backing pages, if supported. + * + * I915_GEM_OBJECT_SHRINK_NO_GPU_WAIT - Don't wait for the object to + * idle. Active objects can be considered later. The TTM backend for + * example might have aync migrations going on, which don't use any + * i915_vma to track the active GTT binding, and hence having an unbound + * object might not be enough. + */ +#define I915_GEM_OBJECT_SHRINK_WRITEBACK BIT(0) +#define I915_GEM_OBJECT_SHRINK_NO_GPU_WAIT BIT(1) + int (*shrink)(struct drm_i915_gem_object *obj, unsigned int flags); int (*pread)(struct drm_i915_gem_object *obj, const struct drm_i915_gem_pread *arg); @@ -302,16 +319,23 @@ struct drm_i915_gem_object { #define I915_BO_ALLOC_PM_VOLATILE BIT(4) /* Object needs to be restored early using memcpy during resume */ #define I915_BO_ALLOC_PM_EARLY BIT(5) +/* + * Object is likely never accessed by the CPU. This will prioritise the BO to be + * allocated in the non-mappable portion of lmem. This is merely a hint, and if + * dealing with userspace objects the CPU fault handler is free to ignore this. + */ +#define I915_BO_ALLOC_GPU_ONLY BIT(6) #define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \ I915_BO_ALLOC_VOLATILE | \ I915_BO_ALLOC_CPU_CLEAR | \ I915_BO_ALLOC_USER | \ I915_BO_ALLOC_PM_VOLATILE | \ - I915_BO_ALLOC_PM_EARLY) -#define I915_BO_READONLY BIT(6) -#define I915_TILING_QUIRK_BIT 7 /* unknown swizzling; do not release! */ -#define I915_BO_PROTECTED BIT(8) -#define I915_BO_WAS_BOUND_BIT 9 + I915_BO_ALLOC_PM_EARLY | \ + I915_BO_ALLOC_GPU_ONLY) +#define I915_BO_READONLY BIT(7) +#define I915_TILING_QUIRK_BIT 8 /* unknown swizzling; do not release! */ +#define I915_BO_PROTECTED BIT(9) +#define I915_BO_WAS_BOUND_BIT 10 /** * @mem_flags - Mutable placement-related flags * @@ -551,31 +575,7 @@ struct drm_i915_gem_object { struct sg_table *pages; void *mapping; - struct i915_page_sizes { - /** - * The sg mask of the pages sg_table. i.e the mask of - * of the lengths for each sg entry. - */ - unsigned int phys; - - /** - * The gtt page sizes we are allowed to use given the - * sg mask and the supported page sizes. This will - * express the smallest unit we can use for the whole - * object, as well as the larger sizes we may be able - * to use opportunistically. - */ - unsigned int sg; - - /** - * The actual gtt page size usage. Since we can have - * multiple vma associated with this object we need to - * prevent any trampling of state, hence a copy of this - * struct also lives in each vma, therefore the gtt - * value here should only be read/write through the vma. - */ - unsigned int gtt; - } page_sizes; + struct i915_page_sizes page_sizes; I915_SELFTEST_DECLARE(unsigned int page_mask); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index a50f884973bc..97c820eee115 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -4,6 +4,8 @@ * Copyright © 2014-2016 Intel Corporation */ +#include <drm/drm_cache.h> + #include "i915_drv.h" #include "i915_gem_object.h" #include "i915_scatterlist.h" @@ -169,16 +171,6 @@ int i915_gem_object_truncate(struct drm_i915_gem_object *obj) return 0; } -/* Try to discard unwanted pages */ -void i915_gem_object_writeback(struct drm_i915_gem_object *obj) -{ - assert_object_held_shared(obj); - GEM_BUG_ON(i915_gem_object_has_pages(obj)); - - if (obj->ops->writeback) - obj->ops->writeback(obj); -} - static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) { struct radix_tree_iter iter; @@ -366,6 +358,9 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, !i915_gem_object_has_iomem(obj)) return ERR_PTR(-ENXIO); + if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY)) + return ERR_PTR(-EINVAL); + assert_object_held(obj); pinned = !(type & I915_MAP_OVERRIDE); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c index ac56124760e1..00359ec9d58b 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c @@ -10,6 +10,7 @@ #include "gt/intel_gt_pm.h" #include "gt/intel_gt_requests.h" +#include "i915_driver.h" #include "i915_drv.h" #if defined(CONFIG_X86) @@ -23,7 +24,7 @@ void i915_gem_suspend(struct drm_i915_private *i915) { GEM_TRACE("%s\n", dev_name(i915->drm.dev)); - intel_wakeref_auto(&i915->ggtt.userfault_wakeref, 0); + intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref, 0); flush_workqueue(i915->wq); /* diff --git a/drivers/gpu/drm/i915/gem/i915_gem_region.c b/drivers/gpu/drm/i915/gem/i915_gem_region.c index a4350227e9ae..6cf94469d5a8 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_region.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_region.c @@ -45,6 +45,11 @@ i915_gem_object_create_region(struct intel_memory_region *mem, GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS); + if (WARN_ON_ONCE(flags & I915_BO_ALLOC_GPU_ONLY && + (flags & I915_BO_ALLOC_CPU_CLEAR || + flags & I915_BO_ALLOC_PM_EARLY))) + return ERR_PTR(-EINVAL); + if (!mem) return ERR_PTR(-ENODEV); @@ -67,6 +72,17 @@ i915_gem_object_create_region(struct intel_memory_region *mem, if (!obj) return ERR_PTR(-ENOMEM); + /* + * Anything smaller than the min_page_size can't be freely inserted into + * the GTT, due to alignemnt restrictions. For such special objects, + * make sure we force memcpy based suspend-resume. In the future we can + * revisit this, either by allowing special mis-aligned objects in the + * migration path, or by mapping all of LMEM upfront using cheap 1G + * GTT entries. + */ + if (default_page_size < mem->min_page_size) + flags |= I915_BO_ALLOC_PM_EARLY; + err = mem->ops->init_object(mem, obj, size, page_size, flags); if (err) goto err_object_free; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index cc9fe258fba7..3a1c782ed791 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -5,8 +5,11 @@ */ #include <linux/pagevec.h> +#include <linux/shmem_fs.h> #include <linux/swap.h> +#include <drm/drm_cache.h> + #include "gem/i915_gem_region.h" #include "i915_drv.h" #include "i915_gemfs.h" @@ -331,6 +334,21 @@ shmem_writeback(struct drm_i915_gem_object *obj) __shmem_writeback(obj->base.size, obj->base.filp->f_mapping); } +static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags) +{ + switch (obj->mm.madv) { + case I915_MADV_DONTNEED: + return i915_gem_object_truncate(obj); + case __I915_MADV_PURGED: + return 0; + } + + if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) + shmem_writeback(obj); + + return 0; +} + void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages, @@ -503,7 +521,7 @@ const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { .get_pages = shmem_get_pages, .put_pages = shmem_put_pages, .truncate = shmem_truncate, - .writeback = shmem_writeback, + .shrink = shmem_shrink, .pwrite = shmem_pwrite, .pread = shmem_pread, @@ -681,7 +699,7 @@ struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915, { return intel_memory_region_create(i915, 0, totalram_pages() << PAGE_SHIFT, - PAGE_SIZE, 0, + PAGE_SIZE, 0, 0, type, instance, &shmem_region_ops); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c index cc927e49d21f..6a6ff98a8746 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c @@ -57,21 +57,17 @@ static int drop_pages(struct drm_i915_gem_object *obj, static int try_to_writeback(struct drm_i915_gem_object *obj, unsigned int flags) { - if (obj->ops->shrinker_release_pages) - return obj->ops->shrinker_release_pages(obj, - !(flags & I915_SHRINK_ACTIVE), - flags & I915_SHRINK_WRITEBACK); - - switch (obj->mm.madv) { - case I915_MADV_DONTNEED: - i915_gem_object_truncate(obj); - return 0; - case __I915_MADV_PURGED: - return 0; - } + if (obj->ops->shrink) { + unsigned int shrink_flags = 0; + + if (!(flags & I915_SHRINK_ACTIVE)) + shrink_flags |= I915_GEM_OBJECT_SHRINK_NO_GPU_WAIT; - if (flags & I915_SHRINK_WRITEBACK) - i915_gem_object_writeback(obj); + if (flags & I915_SHRINK_WRITEBACK) + shrink_flags |= I915_GEM_OBJECT_SHRINK_WRITEBACK; + + return obj->ops->shrink(obj, shrink_flags); + } return 0; } @@ -401,9 +397,9 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr I915_SHRINK_VMAPS); /* We also want to clear any cached iomaps as they wrap vmap */ - mutex_lock(&i915->ggtt.vm.mutex); + mutex_lock(&to_gt(i915)->ggtt->vm.mutex); list_for_each_entry_safe(vma, next, - &i915->ggtt.vm.bound_list, vm_link) { + &to_gt(i915)->ggtt->vm.bound_list, vm_link) { unsigned long count = vma->node.size >> PAGE_SHIFT; struct drm_i915_gem_object *obj = vma->obj; @@ -418,7 +414,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr i915_gem_object_unlock(obj); } - mutex_unlock(&i915->ggtt.vm.mutex); + mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); *(unsigned long *)ptr += freed_pages; return NOTIFY_DONE; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index 7df50fd6cc7b..0bf8f61134af 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -14,7 +14,9 @@ #include "gem/i915_gem_region.h" #include "i915_drv.h" #include "i915_gem_stolen.h" +#include "i915_reg.h" #include "i915_vgpu.h" +#include "intel_mchbar_regs.h" /* * The BIOS typically reserves some of the system's memory for the exclusive @@ -71,7 +73,7 @@ void i915_gem_stolen_remove_node(struct drm_i915_private *i915, static int i915_adjust_stolen(struct drm_i915_private *i915, struct resource *dsm) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct intel_uncore *uncore = ggtt->vm.gt->uncore; struct resource *r; @@ -490,18 +492,22 @@ static int i915_gem_init_stolen(struct intel_memory_region *mem) /* Exclude the reserved region from driver use */ mem->region.end = reserved_base - 1; + mem->io_size = resource_size(&mem->region); /* It is possible for the reserved area to end before the end of stolen * memory, so just consider the start. */ reserved_total = stolen_top - reserved_base; + i915->stolen_usable_size = + resource_size(&i915->dsm) - reserved_total; + drm_dbg(&i915->drm, "Memory reserved for graphics device: %lluK, usable: %lluK\n", (u64)resource_size(&i915->dsm) >> 10, - ((u64)resource_size(&i915->dsm) - reserved_total) >> 10); + (u64)i915->stolen_usable_size >> 10); - i915->stolen_usable_size = - resource_size(&i915->dsm) - reserved_total; + if (i915->stolen_usable_size == 0) + return 0; /* Basic memrange allocator for stolen space. */ drm_mm_init(&i915->mm.stolen, 0, i915->stolen_usable_size); @@ -582,6 +588,7 @@ i915_pages_create_for_stolen(struct drm_device *dev, static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); struct sg_table *pages = i915_pages_create_for_stolen(obj->base.dev, obj->stolen->start, @@ -589,7 +596,7 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) if (IS_ERR(pages)) return PTR_ERR(pages); - dbg_poison(&to_i915(obj->base.dev)->ggtt, + dbg_poison(to_gt(i915)->ggtt, sg_dma_address(pages->sgl), sg_dma_len(pages->sgl), POISON_INUSE); @@ -602,9 +609,10 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj, struct sg_table *pages) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); /* Should only be called from i915_gem_object_release_stolen() */ - dbg_poison(&to_i915(obj->base.dev)->ggtt, + dbg_poison(to_gt(i915)->ggtt, sg_dma_address(pages->sgl), sg_dma_len(pages->sgl), POISON_FREE); @@ -744,7 +752,7 @@ static int init_stolen_lmem(struct intel_memory_region *mem) if (!io_mapping_init_wc(&mem->iomap, mem->io_start, - resource_size(&mem->region))) + mem->io_size)) return -EIO; /* @@ -799,7 +807,8 @@ i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type, I915_GTT_PAGE_SIZE_4K; mem = intel_memory_region_create(i915, lmem_base, lmem_size, - min_page_size, io_start, + min_page_size, + io_start, lmem_size, type, instance, &i915_region_stolen_lmem_ops); if (IS_ERR(mem)) @@ -830,7 +839,7 @@ i915_gem_stolen_smem_setup(struct drm_i915_private *i915, u16 type, mem = intel_memory_region_create(i915, intel_graphics_stolen_res.start, resource_size(&intel_graphics_stolen_res), - PAGE_SIZE, 0, type, instance, + PAGE_SIZE, 0, 0, type, instance, &i915_region_stolen_smem_ops); if (IS_ERR(mem)) return mem; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_throttle.c b/drivers/gpu/drm/i915/gem/i915_gem_throttle.c index 75501db71041..af85d0c28168 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_throttle.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_throttle.c @@ -9,6 +9,7 @@ #include <drm/drm_file.h> #include "i915_drv.h" +#include "i915_file_private.h" #include "i915_gem_context.h" #include "i915_gem_ioctls.h" #include "i915_gem_object.h" diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c index ef4d0f7dc118..d6adda5bf96b 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c @@ -12,6 +12,8 @@ #include "i915_gem_ioctls.h" #include "i915_gem_mman.h" #include "i915_gem_object.h" +#include "i915_gem_tiling.h" +#include "i915_reg.h" /** * DOC: buffer object tiling @@ -181,7 +183,8 @@ static int i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj, int tiling_mode, unsigned int stride) { - struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt; + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct i915_vma *vma, *vn; LIST_HEAD(unbind); int ret = 0; @@ -336,7 +339,7 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; int err; - if (!dev_priv->ggtt.num_fences) + if (!to_gt(dev_priv)->ggtt->num_fences) return -EOPNOTSUPP; obj = i915_gem_object_lookup(file, args->handle); @@ -362,9 +365,9 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, args->stride = 0; } else { if (args->tiling_mode == I915_TILING_X) - args->swizzle_mode = to_i915(dev)->ggtt.bit_6_swizzle_x; + args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x; else - args->swizzle_mode = to_i915(dev)->ggtt.bit_6_swizzle_y; + args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y; /* Hide bit 17 swizzling from the user. This prevents old Mesa * from aborting the application on sw fallbacks to bit 17, @@ -419,7 +422,7 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; int err = -ENOENT; - if (!dev_priv->ggtt.num_fences) + if (!to_gt(dev_priv)->ggtt->num_fences) return -EOPNOTSUPP; rcu_read_lock(); @@ -435,10 +438,10 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, switch (args->tiling_mode) { case I915_TILING_X: - args->swizzle_mode = dev_priv->ggtt.bit_6_swizzle_x; + args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x; break; case I915_TILING_Y: - args->swizzle_mode = dev_priv->ggtt.bit_6_swizzle_y; + args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y; break; default: case I915_TILING_NONE: diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.h b/drivers/gpu/drm/i915/gem/i915_gem_tiling.h new file mode 100644 index 000000000000..9924196a8139 --- /dev/null +++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __I915_GEM_TILING_H__ +#define __I915_GEM_TILING_H__ + +#include <linux/types.h> + +struct drm_i915_private; + +u32 i915_gem_fence_size(struct drm_i915_private *i915, u32 size, + unsigned int tiling, unsigned int stride); +u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size, + unsigned int tiling, unsigned int stride); + +#endif /* __I915_GEM_TILING_H__ */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 1f880c8c66e7..45cc5837ce00 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -3,10 +3,14 @@ * Copyright © 2021 Intel Corporation */ +#include <linux/shmem_fs.h> + #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> +#include <drm/drm_buddy.h> #include "i915_drv.h" +#include "i915_ttm_buddy_manager.h" #include "intel_memory_region.h" #include "intel_region_ttm.h" @@ -20,6 +24,7 @@ #define I915_TTM_PRIO_PURGE 0 #define I915_TTM_PRIO_NO_PAGES 1 #define I915_TTM_PRIO_HAS_PAGES 2 +#define I915_TTM_PRIO_NEEDS_CPU_ACCESS 3 /* * Size of struct ttm_place vector in on-stack struct ttm_placement allocs @@ -127,7 +132,15 @@ i915_ttm_place_from_region(const struct intel_memory_region *mr, place->mem_type = intel_region_to_ttm_type(mr); if (flags & I915_BO_ALLOC_CONTIGUOUS) - place->flags = TTM_PL_FLAG_CONTIGUOUS; + place->flags |= TTM_PL_FLAG_CONTIGUOUS; + if (mr->io_size && mr->io_size < mr->total) { + if (flags & I915_BO_ALLOC_GPU_ONLY) { + place->flags |= TTM_PL_FLAG_TOPDOWN; + } else { + place->fpfn = 0; + place->lpfn = mr->io_size >> PAGE_SHIFT; + } + } } static void @@ -329,6 +342,7 @@ static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) { struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); + struct ttm_resource *res = bo->resource; if (!obj) return false; @@ -342,7 +356,48 @@ static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo, return false; /* Will do for now. Our pinned objects are still on TTM's LRU lists */ - return i915_gem_object_evictable(obj); + if (!i915_gem_object_evictable(obj)) + return false; + + switch (res->mem_type) { + case I915_PL_LMEM0: { + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, res->mem_type); + struct i915_ttm_buddy_resource *bman_res = + to_ttm_buddy_resource(res); + struct drm_buddy *mm = bman_res->mm; + struct drm_buddy_block *block; + + if (!place->fpfn && !place->lpfn) + return true; + + GEM_BUG_ON(!place->lpfn); + + /* + * If we just want something mappable then we can quickly check + * if the current victim resource is using any of the CPU + * visible portion. + */ + if (!place->fpfn && + place->lpfn == i915_ttm_buddy_man_visible_size(man)) + return bman_res->used_visible_size > 0; + + /* Real range allocation */ + list_for_each_entry(block, &bman_res->blocks, link) { + unsigned long fpfn = + drm_buddy_block_offset(block) >> PAGE_SHIFT; + unsigned long lpfn = fpfn + + (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); + + if (place->fpfn < lpfn && place->lpfn > fpfn) + return true; + } + return false; + } default: + break; + } + + return true; } static void i915_ttm_evict_flags(struct ttm_buffer_object *bo, @@ -424,16 +479,14 @@ int i915_ttm_purge(struct drm_i915_gem_object *obj) return 0; } -static int i915_ttm_shrinker_release_pages(struct drm_i915_gem_object *obj, - bool no_wait_gpu, - bool should_writeback) +static int i915_ttm_shrink(struct drm_i915_gem_object *obj, unsigned int flags) { struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); struct i915_ttm_tt *i915_tt = container_of(bo->ttm, typeof(*i915_tt), ttm); struct ttm_operation_ctx ctx = { .interruptible = true, - .no_wait_gpu = no_wait_gpu, + .no_wait_gpu = flags & I915_GEM_OBJECT_SHRINK_NO_GPU_WAIT, }; struct ttm_placement place = {}; int ret; @@ -467,7 +520,7 @@ static int i915_ttm_shrinker_release_pages(struct drm_i915_gem_object *obj, return ret; } - if (should_writeback) + if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) __shmem_writeback(obj->base.size, i915_tt->filp->f_mapping); return 0; @@ -585,11 +638,24 @@ static void i915_ttm_swap_notify(struct ttm_buffer_object *bo) i915_ttm_purge(obj); } +static bool i915_ttm_resource_mappable(struct ttm_resource *res) +{ + struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); + + if (!i915_ttm_cpu_maps_iomem(res)) + return true; + + return bman_res->used_visible_size == bman_res->base.num_pages; +} + static int i915_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) { if (!i915_ttm_cpu_maps_iomem(mem)) return 0; + if (!i915_ttm_resource_mappable(mem)) + return -EINVAL; + mem->bus.caching = ttm_write_combined; mem->bus.is_iomem = true; @@ -728,14 +794,15 @@ static int i915_ttm_get_pages(struct drm_i915_gem_object *obj) * Gem forced migration using the i915_ttm_migrate() op, is allowed even * to regions that are not in the object's list of allowable placements. */ -static int i915_ttm_migrate(struct drm_i915_gem_object *obj, - struct intel_memory_region *mr) +static int __i915_ttm_migrate(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr, + unsigned int flags) { struct ttm_place requested; struct ttm_placement placement; int ret; - i915_ttm_place_from_region(mr, &requested, obj->flags); + i915_ttm_place_from_region(mr, &requested, flags); placement.num_placement = 1; placement.num_busy_placement = 1; placement.placement = &requested; @@ -758,6 +825,12 @@ static int i915_ttm_migrate(struct drm_i915_gem_object *obj, return 0; } +static int i915_ttm_migrate(struct drm_i915_gem_object *obj, + struct intel_memory_region *mr) +{ + return __i915_ttm_migrate(obj, mr, obj->flags); +} + static void i915_ttm_put_pages(struct drm_i915_gem_object *obj, struct sg_table *st) { @@ -844,7 +917,23 @@ void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj) } else if (!i915_gem_object_has_pages(obj)) { bo->priority = I915_TTM_PRIO_NO_PAGES; } else { - bo->priority = I915_TTM_PRIO_HAS_PAGES; + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, bo->resource->mem_type); + + /* + * If we need to place an LMEM resource which doesn't need CPU + * access then we should try not to victimize mappable objects + * first, since we likely end up stealing more of the mappable + * portion. And likewise when we try to find space for a mappble + * object, we know not to ever victimize objects that don't + * occupy any mappable pages. + */ + if (i915_ttm_cpu_maps_iomem(bo->resource) && + i915_ttm_buddy_man_visible_size(man) < man->size && + !(obj->flags & I915_BO_ALLOC_GPU_ONLY)) + bo->priority = I915_TTM_PRIO_NEEDS_CPU_ACCESS; + else + bo->priority = I915_TTM_PRIO_HAS_PAGES; } ttm_bo_move_to_lru_tail(bo, bo->resource, NULL); @@ -900,6 +989,31 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) return VM_FAULT_SIGBUS; } + if (!i915_ttm_resource_mappable(bo->resource)) { + int err = -ENODEV; + int i; + + for (i = 0; i < obj->mm.n_placements; i++) { + struct intel_memory_region *mr = obj->mm.placements[i]; + unsigned int flags; + + if (!mr->io_size && mr->type != INTEL_MEMORY_SYSTEM) + continue; + + flags = obj->flags; + flags &= ~I915_BO_ALLOC_GPU_ONLY; + err = __i915_ttm_migrate(obj, mr, flags); + if (!err) + break; + } + + if (err) { + drm_dbg(dev, "Unable to make resource CPU accessible\n"); + dma_resv_unlock(bo->base.resv); + return VM_FAULT_SIGBUS; + } + } + if (drm_dev_enter(dev, &idx)) { ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, TTM_BO_VM_NUM_PREFAULT); @@ -975,7 +1089,7 @@ static const struct drm_i915_gem_object_ops i915_gem_ttm_obj_ops = { .get_pages = i915_ttm_get_pages, .put_pages = i915_ttm_put_pages, .truncate = i915_ttm_truncate, - .shrinker_release_pages = i915_ttm_shrinker_release_pages, + .shrink = i915_ttm_shrink, .adjust_lru = i915_ttm_adjust_lru, .delayed_free = i915_ttm_delayed_free, @@ -1103,7 +1217,7 @@ i915_gem_ttm_system_setup(struct drm_i915_private *i915, mr = intel_memory_region_create(i915, 0, totalram_pages() << PAGE_SHIFT, - PAGE_SIZE, 0, + PAGE_SIZE, 0, 0, type, instance, &ttm_system_region_ops); if (IS_ERR(mr)) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c index e130c820ae4e..1ebe6e4086a1 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c @@ -142,7 +142,16 @@ int i915_ttm_move_notify(struct ttm_buffer_object *bo) struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); int ret; - ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); + /* + * Note: The async unbinding here will actually transform the + * blocking wait for unbind into a wait before finally submitting + * evict / migration blit and thus stall the migration timeline + * which may not be good for overall throughput. We should make + * sure we await the unbind fences *after* the migration blit + * instead of *before* as we currently do. + */ + ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE | + I915_GEM_OBJECT_UNBIND_ASYNC); if (ret) return ret; @@ -531,7 +540,7 @@ int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, return ret; } - migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, bo->ttm, + migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, ttm, dst_rsgt, true, &deps); i915_deps_fini(&deps); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 3cc01c30dd62..6d1a71d6404c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -42,6 +42,7 @@ #include "i915_drv.h" #include "i915_gem_ioctls.h" #include "i915_gem_object.h" +#include "i915_gem_userptr.h" #include "i915_scatterlist.h" #ifdef CONFIG_MMU_NOTIFIER diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.h b/drivers/gpu/drm/i915/gem/i915_gem_userptr.h new file mode 100644 index 000000000000..8dadb2f8436d --- /dev/null +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_GEM_USERPTR_H__ +#define __I915_GEM_USERPTR_H__ + +struct drm_i915_private; + +int i915_gem_init_userptr(struct drm_i915_private *dev_priv); +void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv); + +#endif /* __I915_GEM_USERPTR_H__ */ diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index 11f0aa65f8a3..7a84fa68a99c 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -8,9 +8,10 @@ #include "i915_selftest.h" -#include "gem/i915_gem_region.h" +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_pm.h" +#include "gem/i915_gem_region.h" #include "gt/intel_gt.h" @@ -370,9 +371,9 @@ static int igt_check_page_sizes(struct i915_vma *vma) err = -EINVAL; } - if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) { + if (!HAS_PAGE_SIZES(i915, vma->resource->page_sizes_gtt)) { pr_err("unsupported page_sizes.gtt=%u, supported=%u\n", - vma->page_sizes.gtt & ~supported, supported); + vma->resource->page_sizes_gtt & ~supported, supported); err = -EINVAL; } @@ -403,15 +404,9 @@ static int igt_check_page_sizes(struct i915_vma *vma) if (i915_gem_object_is_lmem(obj) && IS_ALIGNED(vma->node.start, SZ_2M) && vma->page_sizes.sg & SZ_2M && - vma->page_sizes.gtt < SZ_2M) { + vma->resource->page_sizes_gtt < SZ_2M) { pr_err("gtt pages mismatch for LMEM, expected 2M GTT pages, sg(%u), gtt(%u)\n", - vma->page_sizes.sg, vma->page_sizes.gtt); - err = -EINVAL; - } - - if (obj->mm.page_sizes.gtt) { - pr_err("obj->page_sizes.gtt(%u) should never be set\n", - obj->mm.page_sizes.gtt); + vma->page_sizes.sg, vma->resource->page_sizes_gtt); err = -EINVAL; } @@ -505,7 +500,7 @@ static int igt_mock_memory_region_huge_pages(void *arg) int bit; int err = 0; - mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0); if (IS_ERR(mem)) { pr_err("%s failed to create memory region\n", __func__); return PTR_ERR(mem); @@ -547,9 +542,9 @@ static int igt_mock_memory_region_huge_pages(void *arg) goto out_unpin; } - if (vma->page_sizes.gtt != page_size) { + if (vma->resource->page_sizes_gtt != page_size) { pr_err("%s page_sizes.gtt=%u, expected=%u\n", - __func__, vma->page_sizes.gtt, + __func__, vma->resource->page_sizes_gtt, page_size); err = -EINVAL; goto out_unpin; @@ -630,9 +625,9 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) err = igt_check_page_sizes(vma); - if (vma->page_sizes.gtt != page_size) { + if (vma->resource->page_sizes_gtt != page_size) { pr_err("page_sizes.gtt=%u, expected %u\n", - vma->page_sizes.gtt, page_size); + vma->resource->page_sizes_gtt, page_size); err = -EINVAL; } @@ -647,7 +642,7 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) * pages. */ for (offset = 4096; offset < page_size; offset += 4096) { - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) goto out_unpin; @@ -657,9 +652,10 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) err = igt_check_page_sizes(vma); - if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) { + if (vma->resource->page_sizes_gtt != I915_GTT_PAGE_SIZE_4K) { pr_err("page_sizes.gtt=%u, expected %llu\n", - vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K); + vma->resource->page_sizes_gtt, + I915_GTT_PAGE_SIZE_4K); err = -EINVAL; } @@ -805,9 +801,9 @@ static int igt_mock_ppgtt_huge_fill(void *arg) } } - if (vma->page_sizes.gtt != expected_gtt) { + if (vma->resource->page_sizes_gtt != expected_gtt) { pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n", - vma->page_sizes.gtt, expected_gtt, + vma->resource->page_sizes_gtt, expected_gtt, obj->base.size, yesno(!!single)); err = -EINVAL; break; @@ -961,10 +957,10 @@ static int igt_mock_ppgtt_64K(void *arg) } } - if (vma->page_sizes.gtt != expected_gtt) { + if (vma->resource->page_sizes_gtt != expected_gtt) { pr_err("gtt=%u, expected=%u, i=%d, single=%s\n", - vma->page_sizes.gtt, expected_gtt, i, - yesno(!!single)); + vma->resource->page_sizes_gtt, + expected_gtt, i, yesno(!!single)); err = -EINVAL; goto out_vma_unpin; } @@ -1349,7 +1345,7 @@ try_again: err = i915_gem_object_pin_pages_unlocked(obj); if (err) { - if (err == -ENXIO || err == -E2BIG) { + if (err == -ENXIO || err == -E2BIG || err == -ENOMEM) { i915_gem_object_put(obj); size >>= 1; goto try_again; @@ -1483,6 +1479,65 @@ out: return err; } +static int igt_ppgtt_compact(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct drm_i915_gem_object *obj; + int err; + + /* + * Simple test to catch issues with compact 64K pages -- since the pt is + * compacted to 256B that gives us 32 entries per pt, however since the + * backing page for the pt is 4K, any extra entries we might incorrectly + * write out should be ignored by the HW. If ever hit such a case this + * test should catch it since some of our writes would land in scratch. + */ + + if (!HAS_64K_PAGES(i915)) { + pr_info("device lacks compact 64K page support, skipping\n"); + return 0; + } + + if (!HAS_LMEM(i915)) { + pr_info("device lacks LMEM support, skipping\n"); + return 0; + } + + /* We want the range to cover multiple page-table boundaries. */ + obj = i915_gem_object_create_lmem(i915, SZ_4M, 0); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + err = i915_gem_object_pin_pages_unlocked(obj); + if (err) + goto out_put; + + if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) { + pr_info("LMEM compact unable to allocate huge-page(s)\n"); + goto out_unpin; + } + + /* + * Disable 2M GTT pages by forcing the page-size to 64K for the GTT + * insertion. + */ + obj->mm.page_sizes.sg = I915_GTT_PAGE_SIZE_64K; + + err = igt_write_huge(i915, obj); + if (err) + pr_err("LMEM compact write-huge failed\n"); + +out_unpin: + i915_gem_object_unpin_pages(obj); +out_put: + i915_gem_object_put(obj); + + if (err == -ENOMEM) + err = 0; + + return err; +} + static int igt_tmpfs_fallback(void *arg) { struct drm_i915_private *i915 = arg; @@ -1740,6 +1795,7 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_tmpfs_fallback), SUBTEST(igt_ppgtt_smoke_huge), SUBTEST(igt_ppgtt_sanity_check), + SUBTEST(igt_ppgtt_compact), }; if (!HAS_PPGTT(i915)) { diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c index 75947e9dada2..ddd0772fd828 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c @@ -7,8 +7,9 @@ #include "gt/intel_context.h" #include "gt/intel_engine_user.h" -#include "gt/intel_gt.h" #include "gt/intel_gpu_commands.h" +#include "gt/intel_gt.h" +#include "gt/intel_gt_regs.h" #include "gem/i915_gem_lmem.h" #include "selftests/igt_flush_test.h" @@ -39,6 +40,7 @@ struct tiled_blits { struct blit_buffer scratch; struct i915_vma *batch; u64 hole; + u64 align; u32 width; u32 height; }; @@ -318,7 +320,7 @@ static int pin_buffer(struct i915_vma *vma, u64 addr) int err; if (drm_mm_node_allocated(&vma->node) && vma->node.start != addr) { - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) return err; } @@ -410,14 +412,19 @@ tiled_blits_create(struct intel_engine_cs *engine, struct rnd_state *prng) goto err_free; } - hole_size = 2 * PAGE_ALIGN(WIDTH * HEIGHT * 4); + t->align = i915_vm_min_alignment(t->ce->vm, INTEL_MEMORY_LOCAL); + t->align = max(t->align, + i915_vm_min_alignment(t->ce->vm, INTEL_MEMORY_SYSTEM)); + + hole_size = 2 * round_up(WIDTH * HEIGHT * 4, t->align); hole_size *= 2; /* room to maneuver */ - hole_size += 2 * I915_GTT_MIN_ALIGNMENT; + hole_size += 2 * t->align; /* padding on either side */ mutex_lock(&t->ce->vm->mutex); memset(&hole, 0, sizeof(hole)); err = drm_mm_insert_node_in_range(&t->ce->vm->mm, &hole, - hole_size, 0, I915_COLOR_UNEVICTABLE, + hole_size, t->align, + I915_COLOR_UNEVICTABLE, 0, U64_MAX, DRM_MM_INSERT_BEST); if (!err) @@ -428,7 +435,7 @@ tiled_blits_create(struct intel_engine_cs *engine, struct rnd_state *prng) goto err_put; } - t->hole = hole.start + I915_GTT_MIN_ALIGNMENT; + t->hole = hole.start + t->align; pr_info("Using hole at %llx\n", t->hole); err = tiled_blits_create_buffers(t, WIDTH, HEIGHT, prng); @@ -455,7 +462,7 @@ static void tiled_blits_destroy(struct tiled_blits *t) static int tiled_blits_prepare(struct tiled_blits *t, struct rnd_state *prng) { - u64 offset = PAGE_ALIGN(t->width * t->height * 4); + u64 offset = round_up(t->width * t->height * 4, t->align); u32 *map; int err; int i; @@ -486,8 +493,7 @@ static int tiled_blits_prepare(struct tiled_blits *t, static int tiled_blits_bounce(struct tiled_blits *t, struct rnd_state *prng) { - u64 offset = - round_up(t->width * t->height * 4, 2 * I915_GTT_MIN_ALIGNMENT); + u64 offset = round_up(t->width * t->height * 4, 2 * t->align); int err; /* We want to check position invariant tiling across GTT eviction */ @@ -500,7 +506,7 @@ static int tiled_blits_bounce(struct tiled_blits *t, struct rnd_state *prng) /* Reposition so that we overlap the old addresses, and slightly off */ err = tiled_blit(t, - &t->buffers[2], t->hole + I915_GTT_MIN_ALIGNMENT, + &t->buffers[2], t->hole + t->align, &t->buffers[1], t->hole + 3 * offset / 2); if (err) return err; @@ -543,7 +549,7 @@ static bool has_bit17_swizzle(int sw) static bool bad_swizzling(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) return true; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index 3f41fe5ec9d4..7609db87df05 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -6,8 +6,10 @@ #include <linux/prime_numbers.h> +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gt/intel_engine_pm.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_gt.h" #include "gt/intel_gt_requests.h" #include "gt/intel_reset.h" @@ -883,7 +885,9 @@ out_file: return err; } -static int rpcs_query_batch(struct drm_i915_gem_object *rpcs, struct i915_vma *vma) +static int rpcs_query_batch(struct drm_i915_gem_object *rpcs, + struct i915_vma *vma, + struct intel_engine_cs *engine) { u32 *cmd; @@ -894,7 +898,7 @@ static int rpcs_query_batch(struct drm_i915_gem_object *rpcs, struct i915_vma *v return PTR_ERR(cmd); *cmd++ = MI_STORE_REGISTER_MEM_GEN8; - *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE); + *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE(engine->mmio_base)); *cmd++ = lower_32_bits(vma->node.start); *cmd++ = upper_32_bits(vma->node.start); *cmd = MI_BATCH_BUFFER_END; @@ -955,7 +959,7 @@ retry: if (err) goto err_vma; - err = rpcs_query_batch(rpcs, vma); + err = rpcs_query_batch(rpcs, vma, ce->engine); if (err) goto err_batch; @@ -1374,7 +1378,7 @@ static int igt_ctx_readonly(void *arg) goto out_file; } - vm = ctx->vm ?: &i915->ggtt.alias->vm; + vm = ctx->vm ?: &to_gt(i915)->ggtt->alias->vm; if (!vm || !vm->has_read_only) { err = 0; goto out_file; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c index 3cc74b0fed06..b071a58dd6da 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c @@ -266,7 +266,7 @@ static int igt_dmabuf_import(void *arg) struct drm_i915_gem_object *obj; struct dma_buf *dmabuf; void *obj_map, *dma_map; - struct dma_buf_map map; + struct iosys_map map; u32 pattern[] = { 0, 0xaa, 0xcc, 0x55, 0xff }; int err, i; @@ -349,7 +349,7 @@ static int igt_dmabuf_import_ownership(void *arg) struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj; struct dma_buf *dmabuf; - struct dma_buf_map map; + struct iosys_map map; void *ptr; int err; @@ -400,7 +400,7 @@ static int igt_dmabuf_export_vmap(void *arg) struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj; struct dma_buf *dmabuf; - struct dma_buf_map map; + struct iosys_map map; void *ptr; int err; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c index ecb691c81d1e..d534141b2cf7 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c @@ -4,8 +4,13 @@ */ #include "gt/intel_migrate.h" +#include "gt/intel_gpu_commands.h" #include "gem/i915_gem_ttm_move.h" +#include "i915_deps.h" + +#include "selftests/igt_spinner.h" + static int igt_fill_check_buffer(struct drm_i915_gem_object *obj, bool fill) { @@ -101,7 +106,8 @@ static int igt_same_create_migrate(void *arg) } static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww, - struct drm_i915_gem_object *obj) + struct drm_i915_gem_object *obj, + struct i915_vma *vma) { int err; @@ -109,6 +115,24 @@ static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww, if (err) return err; + if (vma) { + err = i915_vma_pin_ww(vma, ww, obj->base.size, 0, + 0UL | PIN_OFFSET_FIXED | + PIN_USER); + if (err) { + if (err != -EINTR && err != ERESTARTSYS && + err != -EDEADLK) + pr_err("Failed to pin vma.\n"); + return err; + } + + i915_vma_unpin(vma); + } + + /* + * Migration will implicitly unbind (asynchronously) any bound + * vmas. + */ if (i915_gem_object_is_lmem(obj)) { err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM); if (err) { @@ -149,11 +173,15 @@ static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww, return err; } -static int igt_lmem_pages_migrate(void *arg) +static int __igt_lmem_pages_migrate(struct intel_gt *gt, + struct i915_address_space *vm, + struct i915_deps *deps, + struct igt_spinner *spin, + struct dma_fence *spin_fence) { - struct intel_gt *gt = arg; struct drm_i915_private *i915 = gt->i915; struct drm_i915_gem_object *obj; + struct i915_vma *vma = NULL; struct i915_gem_ww_ctx ww; struct i915_request *rq; int err; @@ -165,6 +193,14 @@ static int igt_lmem_pages_migrate(void *arg) if (IS_ERR(obj)) return PTR_ERR(obj); + if (vm) { + vma = i915_vma_instance(obj, vm, NULL); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto out_put; + } + } + /* Initial GPU fill, sync, CPU initialization. */ for_i915_gem_ww(&ww, err, true) { err = i915_gem_object_lock(obj, &ww); @@ -175,25 +211,23 @@ static int igt_lmem_pages_migrate(void *arg) if (err) continue; - err = intel_migrate_clear(>->migrate, &ww, NULL, + err = intel_migrate_clear(>->migrate, &ww, deps, obj->mm.pages->sgl, obj->cache_level, i915_gem_object_is_lmem(obj), 0xdeadbeaf, &rq); if (rq) { dma_resv_add_excl_fence(obj->base.resv, &rq->fence); + i915_gem_object_set_moving_fence(obj, &rq->fence); i915_request_put(rq); } if (err) continue; - err = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE, - 5 * HZ); - if (err) - continue; - - err = igt_fill_check_buffer(obj, true); - if (err) - continue; + if (!vma) { + err = igt_fill_check_buffer(obj, true); + if (err) + continue; + } } if (err) goto out_put; @@ -204,7 +238,7 @@ static int igt_lmem_pages_migrate(void *arg) */ for (i = 1; i <= 5; ++i) { for_i915_gem_ww(&ww, err, true) - err = lmem_pages_migrate_one(&ww, obj); + err = lmem_pages_migrate_one(&ww, obj, vma); if (err) goto out_put; } @@ -213,12 +247,27 @@ static int igt_lmem_pages_migrate(void *arg) if (err) goto out_put; + if (spin) { + if (dma_fence_is_signaled(spin_fence)) { + pr_err("Spinner was terminated by hangcheck.\n"); + err = -EBUSY; + goto out_unlock; + } + igt_spinner_end(spin); + } + /* Finally sync migration and check content. */ err = i915_gem_object_wait_migration(obj, true); if (err) goto out_unlock; - err = igt_fill_check_buffer(obj, false); + if (vma) { + err = i915_vma_wait_for_bind(vma); + if (err) + goto out_unlock; + } else { + err = igt_fill_check_buffer(obj, false); + } out_unlock: i915_gem_object_unlock(obj); @@ -231,6 +280,7 @@ out_put: static int igt_lmem_pages_failsafe_migrate(void *arg) { int fail_gpu, fail_alloc, ret; + struct intel_gt *gt = arg; for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) { for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) { @@ -238,7 +288,118 @@ static int igt_lmem_pages_failsafe_migrate(void *arg) fail_gpu, fail_alloc); i915_ttm_migrate_set_failure_modes(fail_gpu, fail_alloc); - ret = igt_lmem_pages_migrate(arg); + ret = __igt_lmem_pages_migrate(gt, NULL, NULL, NULL, NULL); + if (ret) + goto out_err; + } + } + +out_err: + i915_ttm_migrate_set_failure_modes(false, false); + return ret; +} + +/* + * This subtest tests that unbinding at migration is indeed performed + * async. We launch a spinner and a number of migrations depending on + * that spinner to have terminated. Before each migration we bind a + * vma, which should then be async unbound by the migration operation. + * If we are able to schedule migrations without blocking while the + * spinner is still running, those unbinds are indeed async and non- + * blocking. + * + * Note that each async bind operation is awaiting the previous migration + * due to the moving fence resulting from the migration. + */ +static int igt_async_migrate(struct intel_gt *gt) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct i915_ppgtt *ppgtt; + struct igt_spinner spin; + int err; + + ppgtt = i915_ppgtt_create(gt, 0); + if (IS_ERR(ppgtt)) + return PTR_ERR(ppgtt); + + if (igt_spinner_init(&spin, gt)) { + err = -ENOMEM; + goto out_spin; + } + + for_each_engine(engine, gt, id) { + struct ttm_operation_ctx ctx = { + .interruptible = true + }; + struct dma_fence *spin_fence; + struct intel_context *ce; + struct i915_request *rq; + struct i915_deps deps; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + goto out_ce; + } + + /* + * Use MI_NOOP, making the spinner non-preemptible. If there + * is a code path where we fail async operation due to the + * running spinner, we will block and fail to end the + * spinner resulting in a deadlock. But with a non- + * preemptible spinner, hangcheck will terminate the spinner + * for us, and we will later detect that and fail the test. + */ + rq = igt_spinner_create_request(&spin, ce, MI_NOOP); + intel_context_put(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_ce; + } + + i915_deps_init(&deps, GFP_KERNEL); + err = i915_deps_add_dependency(&deps, &rq->fence, &ctx); + spin_fence = dma_fence_get(&rq->fence); + i915_request_add(rq); + if (err) + goto out_ce; + + err = __igt_lmem_pages_migrate(gt, &ppgtt->vm, &deps, &spin, + spin_fence); + i915_deps_fini(&deps); + dma_fence_put(spin_fence); + if (err) + goto out_ce; + } + +out_ce: + igt_spinner_fini(&spin); +out_spin: + i915_vm_put(&ppgtt->vm); + + return err; +} + +/* + * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while + * arming the migration error check and block async migration. This + * will cause us to deadlock and hangcheck will terminate the spinner + * causing the test to fail. + */ +#define ASYNC_FAIL_ALLOC 1 +static int igt_lmem_async_migrate(void *arg) +{ + int fail_gpu, fail_alloc, ret; + struct intel_gt *gt = arg; + + for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) { + for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) { + pr_info("Simulated failure modes: gpu: %d, alloc: %d\n", + fail_gpu, fail_alloc); + i915_ttm_migrate_set_failure_modes(fail_gpu, + fail_alloc); + ret = igt_async_migrate(gt); if (ret) goto out_err; } @@ -256,6 +417,7 @@ int i915_gem_migrate_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_lmem_create_migrate), SUBTEST(igt_same_create_migrate), SUBTEST(igt_lmem_pages_failsafe_migrate), + SUBTEST(igt_lmem_async_migrate), }; if (!HAS_LMEM(i915)) diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index c6291429b00c..a132e241c3ee 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -6,11 +6,16 @@ #include <linux/prime_numbers.h> +#include "gem/i915_gem_internal.h" +#include "gem/i915_gem_region.h" +#include "gem/i915_gem_ttm.h" #include "gt/intel_engine_pm.h" #include "gt/intel_gpu_commands.h" #include "gt/intel_gt.h" #include "gt/intel_gt_pm.h" -#include "gem/i915_gem_region.h" +#include "gt/intel_migrate.h" +#include "i915_ttm_buddy_manager.h" + #include "huge_gem_object.h" #include "i915_selftest.h" #include "selftests/i915_random.h" @@ -166,7 +171,9 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj, kunmap(p); out: - __i915_vma_put(vma); + i915_gem_object_lock(obj, NULL); + i915_vma_destroy(vma); + i915_gem_object_unlock(obj); return err; } @@ -261,7 +268,9 @@ static int check_partial_mappings(struct drm_i915_gem_object *obj, if (err) return err; - __i915_vma_put(vma); + i915_gem_object_lock(obj, NULL); + i915_vma_destroy(vma); + i915_gem_object_unlock(obj); if (igt_timeout(end_time, "%s: timed out after tiling=%d stride=%d\n", @@ -307,7 +316,7 @@ static int igt_partial_tiling(void *arg) int tiling; int err; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return 0; /* We want to check the page mapping and fencing of a large object @@ -320,7 +329,7 @@ static int igt_partial_tiling(void *arg) obj = huge_gem_object(i915, nreal << PAGE_SHIFT, - (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); + (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); if (IS_ERR(obj)) return PTR_ERR(obj); @@ -366,10 +375,10 @@ static int igt_partial_tiling(void *arg) tile.tiling = tiling; switch (tiling) { case I915_TILING_X: - tile.swizzle = i915->ggtt.bit_6_swizzle_x; + tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x; break; case I915_TILING_Y: - tile.swizzle = i915->ggtt.bit_6_swizzle_y; + tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y; break; } @@ -440,7 +449,7 @@ static int igt_smoke_tiling(void *arg) IGT_TIMEOUT(end); int err; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return 0; /* @@ -457,7 +466,7 @@ static int igt_smoke_tiling(void *arg) obj = huge_gem_object(i915, nreal << PAGE_SHIFT, - (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); + (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); if (IS_ERR(obj)) return PTR_ERR(obj); @@ -486,10 +495,10 @@ static int igt_smoke_tiling(void *arg) break; case I915_TILING_X: - tile.swizzle = i915->ggtt.bit_6_swizzle_x; + tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x; break; case I915_TILING_Y: - tile.swizzle = i915->ggtt.bit_6_swizzle_y; + tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y; break; } @@ -856,6 +865,7 @@ static int wc_check(struct drm_i915_gem_object *obj) static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); bool no_map; if (obj->ops->mmap_offset) @@ -864,7 +874,7 @@ static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) return false; if (type == I915_MMAP_TYPE_GTT && - !i915_ggtt_has_aperture(&to_i915(obj->base.dev)->ggtt)) + !i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return false; i915_gem_object_lock(obj, NULL); @@ -994,6 +1004,331 @@ static int igt_mmap(void *arg) return 0; } +static void igt_close_objects(struct drm_i915_private *i915, + struct list_head *objects) +{ + struct drm_i915_gem_object *obj, *on; + + list_for_each_entry_safe(obj, on, objects, st_link) { + i915_gem_object_lock(obj, NULL); + if (i915_gem_object_has_pinned_pages(obj)) + i915_gem_object_unpin_pages(obj); + /* No polluting the memory region between tests */ + __i915_gem_object_put_pages(obj); + i915_gem_object_unlock(obj); + list_del(&obj->st_link); + i915_gem_object_put(obj); + } + + cond_resched(); + + i915_gem_drain_freed_objects(i915); +} + +static void igt_make_evictable(struct list_head *objects) +{ + struct drm_i915_gem_object *obj; + + list_for_each_entry(obj, objects, st_link) { + i915_gem_object_lock(obj, NULL); + if (i915_gem_object_has_pinned_pages(obj)) + i915_gem_object_unpin_pages(obj); + i915_gem_object_unlock(obj); + } + + cond_resched(); +} + +static int igt_fill_mappable(struct intel_memory_region *mr, + struct list_head *objects) +{ + u64 size, total; + int err; + + total = 0; + size = mr->io_size; + do { + struct drm_i915_gem_object *obj; + + obj = i915_gem_object_create_region(mr, size, 0, 0); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto err_close; + } + + list_add(&obj->st_link, objects); + + err = i915_gem_object_pin_pages_unlocked(obj); + if (err) { + if (err != -ENXIO && err != -ENOMEM) + goto err_close; + + if (size == mr->min_page_size) { + err = 0; + break; + } + + size >>= 1; + continue; + } + + total += obj->base.size; + } while (1); + + pr_info("%s filled=%lluMiB\n", __func__, total >> 20); + return 0; + +err_close: + igt_close_objects(mr->i915, objects); + return err; +} + +static int ___igt_mmap_migrate(struct drm_i915_private *i915, + struct drm_i915_gem_object *obj, + unsigned long addr, + bool unfaultable) +{ + struct vm_area_struct *area; + int err = 0, i; + + pr_info("igt_mmap(%s, %d) @ %lx\n", + obj->mm.region->name, I915_MMAP_TYPE_FIXED, addr); + + mmap_read_lock(current->mm); + area = vma_lookup(current->mm, addr); + mmap_read_unlock(current->mm); + if (!area) { + pr_err("%s: Did not create a vm_area_struct for the mmap\n", + obj->mm.region->name); + err = -EINVAL; + goto out_unmap; + } + + for (i = 0; i < obj->base.size / sizeof(u32); i++) { + u32 __user *ux = u64_to_user_ptr((u64)(addr + i * sizeof(*ux))); + u32 x; + + if (get_user(x, ux)) { + err = -EFAULT; + if (!unfaultable) { + pr_err("%s: Unable to read from mmap, offset:%zd\n", + obj->mm.region->name, i * sizeof(x)); + goto out_unmap; + } + + continue; + } + + if (unfaultable) { + pr_err("%s: Faulted unmappable memory\n", + obj->mm.region->name); + err = -EINVAL; + goto out_unmap; + } + + if (x != expand32(POISON_INUSE)) { + pr_err("%s: Read incorrect value from mmap, offset:%zd, found:%x, expected:%x\n", + obj->mm.region->name, + i * sizeof(x), x, expand32(POISON_INUSE)); + err = -EINVAL; + goto out_unmap; + } + + x = expand32(POISON_FREE); + if (put_user(x, ux)) { + pr_err("%s: Unable to write to mmap, offset:%zd\n", + obj->mm.region->name, i * sizeof(x)); + err = -EFAULT; + goto out_unmap; + } + } + + if (unfaultable) { + if (err == -EFAULT) + err = 0; + } else { + obj->flags &= ~I915_BO_ALLOC_GPU_ONLY; + err = wc_check(obj); + } +out_unmap: + vm_munmap(addr, obj->base.size); + return err; +} + +#define IGT_MMAP_MIGRATE_TOPDOWN (1 << 0) +#define IGT_MMAP_MIGRATE_FILL (1 << 1) +#define IGT_MMAP_MIGRATE_EVICTABLE (1 << 2) +#define IGT_MMAP_MIGRATE_UNFAULTABLE (1 << 3) +static int __igt_mmap_migrate(struct intel_memory_region **placements, + int n_placements, + struct intel_memory_region *expected_mr, + unsigned int flags) +{ + struct drm_i915_private *i915 = placements[0]->i915; + struct drm_i915_gem_object *obj; + struct i915_request *rq = NULL; + unsigned long addr; + LIST_HEAD(objects); + u64 offset; + int err; + + obj = __i915_gem_object_create_user(i915, PAGE_SIZE, + placements, + n_placements); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + if (flags & IGT_MMAP_MIGRATE_TOPDOWN) + obj->flags |= I915_BO_ALLOC_GPU_ONLY; + + err = __assign_mmap_offset(obj, I915_MMAP_TYPE_FIXED, &offset, NULL); + if (err) + goto out_put; + + /* + * This will eventually create a GEM context, due to opening dummy drm + * file, which needs a tiny amount of mappable device memory for the top + * level paging structures(and perhaps scratch), so make sure we + * allocate early, to avoid tears. + */ + addr = igt_mmap_offset(i915, offset, obj->base.size, + PROT_WRITE, MAP_SHARED); + if (IS_ERR_VALUE(addr)) { + err = addr; + goto out_put; + } + + if (flags & IGT_MMAP_MIGRATE_FILL) { + err = igt_fill_mappable(placements[0], &objects); + if (err) + goto out_put; + } + + err = i915_gem_object_lock(obj, NULL); + if (err) + goto out_put; + + err = i915_gem_object_pin_pages(obj); + if (err) { + i915_gem_object_unlock(obj); + goto out_put; + } + + err = intel_context_migrate_clear(to_gt(i915)->migrate.context, NULL, + obj->mm.pages->sgl, obj->cache_level, + i915_gem_object_is_lmem(obj), + expand32(POISON_INUSE), &rq); + i915_gem_object_unpin_pages(obj); + if (rq) { + dma_resv_add_excl_fence(obj->base.resv, &rq->fence); + i915_gem_object_set_moving_fence(obj, &rq->fence); + i915_request_put(rq); + } + i915_gem_object_unlock(obj); + if (err) + goto out_put; + + if (flags & IGT_MMAP_MIGRATE_EVICTABLE) + igt_make_evictable(&objects); + + err = ___igt_mmap_migrate(i915, obj, addr, + flags & IGT_MMAP_MIGRATE_UNFAULTABLE); + if (!err && obj->mm.region != expected_mr) { + pr_err("%s region mismatch %s\n", __func__, expected_mr->name); + err = -EINVAL; + } + +out_put: + i915_gem_object_put(obj); + igt_close_objects(i915, &objects); + return err; +} + +static int igt_mmap_migrate(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_memory_region *system = i915->mm.regions[INTEL_REGION_SMEM]; + struct intel_memory_region *mr; + enum intel_region_id id; + + for_each_memory_region(mr, i915, id) { + struct intel_memory_region *mixed[] = { mr, system }; + struct intel_memory_region *single[] = { mr }; + struct ttm_resource_manager *man = mr->region_private; + resource_size_t saved_io_size; + int err; + + if (mr->private) + continue; + + if (!mr->io_size) + continue; + + /* + * For testing purposes let's force small BAR, if not already + * present. + */ + saved_io_size = mr->io_size; + if (mr->io_size == mr->total) { + resource_size_t io_size = mr->io_size; + + io_size = rounddown_pow_of_two(io_size >> 1); + if (io_size < PAGE_SIZE) + continue; + + mr->io_size = io_size; + i915_ttm_buddy_man_force_visible_size(man, + io_size >> PAGE_SHIFT); + } + + /* + * Allocate in the mappable portion, should be no suprises here. + */ + err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), mr, 0); + if (err) + goto out_io_size; + + /* + * Allocate in the non-mappable portion, but force migrating to + * the mappable portion on fault (LMEM -> LMEM) + */ + err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, + IGT_MMAP_MIGRATE_TOPDOWN | + IGT_MMAP_MIGRATE_FILL | + IGT_MMAP_MIGRATE_EVICTABLE); + if (err) + goto out_io_size; + + /* + * Allocate in the non-mappable portion, but force spilling into + * system memory on fault (LMEM -> SMEM) + */ + err = __igt_mmap_migrate(mixed, ARRAY_SIZE(mixed), system, + IGT_MMAP_MIGRATE_TOPDOWN | + IGT_MMAP_MIGRATE_FILL); + if (err) + goto out_io_size; + + /* + * Allocate in the non-mappable portion, but since the mappable + * portion is already full, and we can't spill to system memory, + * then we should expect the fault to fail. + */ + err = __igt_mmap_migrate(single, ARRAY_SIZE(single), mr, + IGT_MMAP_MIGRATE_TOPDOWN | + IGT_MMAP_MIGRATE_FILL | + IGT_MMAP_MIGRATE_UNFAULTABLE); +out_io_size: + mr->io_size = saved_io_size; + i915_ttm_buddy_man_force_visible_size(man, + mr->io_size >> PAGE_SHIFT); + if (err) + return err; + } + + return 0; +} + static const char *repr_mmap_type(enum i915_mmap_type type) { switch (type) { @@ -1351,7 +1686,9 @@ static int __igt_mmap_revoke(struct drm_i915_private *i915, * for other objects. Ergo we have to revoke the previous mmap PTE * access as it no longer points to the same object. */ + i915_gem_object_lock(obj, NULL); err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); + i915_gem_object_unlock(obj); if (err) { pr_err("Failed to unbind object!\n"); goto out_unmap; @@ -1417,6 +1754,7 @@ int i915_gem_mman_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_smoke_tiling), SUBTEST(igt_mmap_offset_exhaustion), SUBTEST(igt_mmap), + SUBTEST(igt_mmap_migrate), SUBTEST(igt_mmap_access), SUBTEST(igt_mmap_revoke), SUBTEST(igt_mmap_gpu), diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c index 740ee8086a27..fe0a890775e2 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c @@ -43,7 +43,7 @@ static int igt_gem_huge(void *arg) obj = huge_gem_object(i915, nreal * PAGE_SIZE, - i915->ggtt.vm.total + PAGE_SIZE); + to_gt(i915)->ggtt->vm.total + PAGE_SIZE); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c index b35c1219c852..3c55e77b0f1b 100644 --- a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c +++ b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c @@ -7,6 +7,7 @@ #include "igt_gem_utils.h" #include "gem/i915_gem_context.h" +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gt/intel_context.h" #include "gt/intel_gpu_commands.h" diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_context.c b/drivers/gpu/drm/i915/gem/selftests/mock_context.c index c0a8ef368044..6d6082b5f31f 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/mock_context.c @@ -4,6 +4,7 @@ * Copyright © 2016 Intel Corporation */ +#include "i915_file_private.h" #include "mock_context.h" #include "selftests/mock_drm.h" #include "selftests/mock_gtt.h" diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c b/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c index 2855d11c7a51..b2a5882b8f81 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/selftests/mock_dmabuf.c @@ -61,7 +61,7 @@ static void mock_dmabuf_release(struct dma_buf *dma_buf) kfree(mock); } -static int mock_dmabuf_vmap(struct dma_buf *dma_buf, struct dma_buf_map *map) +static int mock_dmabuf_vmap(struct dma_buf *dma_buf, struct iosys_map *map) { struct mock_dmabuf *mock = to_mock(dma_buf); void *vaddr; @@ -69,12 +69,12 @@ static int mock_dmabuf_vmap(struct dma_buf *dma_buf, struct dma_buf_map *map) vaddr = vm_map_ram(mock->pages, mock->npages, 0); if (!vaddr) return -ENOMEM; - dma_buf_map_set_vaddr(map, vaddr); + iosys_map_set_vaddr(map, vaddr); return 0; } -static void mock_dmabuf_vunmap(struct dma_buf *dma_buf, struct dma_buf_map *map) +static void mock_dmabuf_vunmap(struct dma_buf *dma_buf, struct iosys_map *map) { struct mock_dmabuf *mock = to_mock(dma_buf); diff --git a/drivers/gpu/drm/i915/gt/gen2_engine_cs.c b/drivers/gpu/drm/i915/gt/gen2_engine_cs.c index 61383830505e..1c82caf525c3 100644 --- a/drivers/gpu/drm/i915/gt/gen2_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/gen2_engine_cs.c @@ -5,7 +5,9 @@ #include "gen2_engine_cs.h" #include "i915_drv.h" +#include "i915_reg.h" #include "intel_engine.h" +#include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt.h" #include "intel_gt_irq.h" diff --git a/drivers/gpu/drm/i915/gt/gen6_engine_cs.c b/drivers/gpu/drm/i915/gt/gen6_engine_cs.c index b388ceeeb1c9..5e65550b4dfb 100644 --- a/drivers/gpu/drm/i915/gt/gen6_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/gen6_engine_cs.c @@ -5,6 +5,7 @@ #include "gen6_engine_cs.h" #include "intel_engine.h" +#include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt.h" #include "intel_gt_irq.h" diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index 6e9292918bfc..871fe7bda0e0 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -5,10 +5,14 @@ #include <linux/log2.h> +#include "gem/i915_gem_internal.h" + #include "gen6_ppgtt.h" #include "i915_scatterlist.h" #include "i915_trace.h" #include "i915_vgpu.h" +#include "intel_gt_regs.h" +#include "intel_engine_regs.h" #include "intel_gt.h" /* Write pde (index) from the page directory @pd to the page table @pt */ @@ -104,17 +108,17 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, } static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags) { struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_page_directory * const pd = ppgtt->pd; - unsigned int first_entry = vma->node.start / I915_GTT_PAGE_SIZE; + unsigned int first_entry = vma_res->start / I915_GTT_PAGE_SIZE; unsigned int act_pt = first_entry / GEN6_PTES; unsigned int act_pte = first_entry % GEN6_PTES; const u32 pte_encode = vm->pte_encode(0, cache_level, flags); - struct sgt_dma iter = sgt_dma(vma); + struct sgt_dma iter = sgt_dma(vma_res); gen6_pte_t *vaddr; GEM_BUG_ON(!pd->entry[act_pt]); @@ -140,7 +144,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, } } while (1); - vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; + vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; } static void gen6_flush_pd(struct gen6_ppgtt *ppgtt, u64 start, u64 end) @@ -271,13 +275,13 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) static void pd_vma_bind(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 unused) { struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - struct gen6_ppgtt *ppgtt = vma->private; - u32 ggtt_offset = i915_ggtt_offset(vma) / I915_GTT_PAGE_SIZE; + struct gen6_ppgtt *ppgtt = vma_res->private; + u32 ggtt_offset = vma_res->start / I915_GTT_PAGE_SIZE; ppgtt->pp_dir = ggtt_offset * sizeof(gen6_pte_t) << 10; ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset; @@ -285,9 +289,10 @@ static void pd_vma_bind(struct i915_address_space *vm, gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total); } -static void pd_vma_unbind(struct i915_address_space *vm, struct i915_vma *vma) +static void pd_vma_unbind(struct i915_address_space *vm, + struct i915_vma_resource *vma_res) { - struct gen6_ppgtt *ppgtt = vma->private; + struct gen6_ppgtt *ppgtt = vma_res->private; struct i915_page_directory * const pd = ppgtt->base.pd; struct i915_page_table *pt; unsigned int pde; diff --git a/drivers/gpu/drm/i915/gt/gen7_renderclear.c b/drivers/gpu/drm/i915/gt/gen7_renderclear.c index 21f08e53889c..317efb145787 100644 --- a/drivers/gpu/drm/i915/gt/gen7_renderclear.c +++ b/drivers/gpu/drm/i915/gt/gen7_renderclear.c @@ -6,6 +6,7 @@ #include "gen7_renderclear.h" #include "i915_drv.h" #include "intel_gpu_commands.h" +#include "intel_gt_regs.h" #define GT3_INLINE_DATA_DELAYS 0x1E00 #define batch_advance(Y, CS) GEM_BUG_ON((Y)->end != (CS)) diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c index e320610dd0b8..b1b9c3fd7bf9 100644 --- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c @@ -5,8 +5,9 @@ #include "gen8_engine_cs.h" #include "i915_drv.h" -#include "intel_lrc.h" #include "intel_gpu_commands.h" +#include "intel_gt_regs.h" +#include "intel_lrc.h" #include "intel_ring.h" int gen8_emit_flush_rcs(struct i915_request *rq, u32 mode) @@ -200,6 +201,8 @@ static u32 *gen12_emit_aux_table_inv(const i915_reg_t inv_reg, u32 *cs) int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) { + struct intel_engine_cs *engine = rq->engine; + if (mode & EMIT_FLUSH) { u32 flags = 0; u32 *cs; @@ -218,6 +221,9 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) flags |= PIPE_CONTROL_CS_STALL; + if (engine->class == COMPUTE_CLASS) + flags &= ~PIPE_CONTROL_3D_FLAGS; + cs = intel_ring_begin(rq, 6); if (IS_ERR(cs)) return PTR_ERR(cs); @@ -245,6 +251,9 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) flags |= PIPE_CONTROL_CS_STALL; + if (engine->class == COMPUTE_CLASS) + flags &= ~PIPE_CONTROL_3D_FLAGS; + cs = intel_ring_begin(rq, 8 + 4); if (IS_ERR(cs)) return PTR_ERR(cs); @@ -617,19 +626,27 @@ u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) { + struct drm_i915_private *i915 = rq->engine->i915; + u32 flags = (PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_TILE_CACHE_FLUSH | + PIPE_CONTROL_FLUSH_L3 | + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_DC_FLUSH_ENABLE | + PIPE_CONTROL_FLUSH_ENABLE); + + if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) + /* Wa_1409600907 */ + flags |= PIPE_CONTROL_DEPTH_STALL; + + if (rq->engine->class == COMPUTE_CLASS) + flags &= ~PIPE_CONTROL_3D_FLAGS; + cs = gen12_emit_ggtt_write_rcs(cs, rq->fence.seqno, hwsp_offset(rq), PIPE_CONTROL0_HDC_PIPELINE_FLUSH, - PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_TILE_CACHE_FLUSH | - PIPE_CONTROL_FLUSH_L3 | - PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | - PIPE_CONTROL_DEPTH_CACHE_FLUSH | - /* Wa_1409600907:tgl */ - PIPE_CONTROL_DEPTH_STALL | - PIPE_CONTROL_DC_FLUSH_ENABLE | - PIPE_CONTROL_FLUSH_ENABLE); + flags); return gen12_emit_fini_breadcrumb_tail(rq, cs); } diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index b012c50f7ce7..f574da00eff1 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -233,6 +233,8 @@ static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, start, end, lvl); } else { unsigned int count; + unsigned int pte = gen8_pd_index(start, 0); + unsigned int num_ptes; u64 *vaddr; count = gen8_pt_count(start, end); @@ -242,10 +244,18 @@ static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm, atomic_read(&pt->used)); GEM_BUG_ON(!count || count >= atomic_read(&pt->used)); + num_ptes = count; + if (pt->is_compact) { + GEM_BUG_ON(num_ptes % 16); + GEM_BUG_ON(pte % 16); + num_ptes /= 16; + pte /= 16; + } + vaddr = px_vaddr(pt); - memset64(vaddr + gen8_pd_index(start, 0), + memset64(vaddr + pte, vm->scratch[0]->encode, - count); + num_ptes); atomic_sub(count, &pt->used); start += count; @@ -453,20 +463,110 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, return idx; } -static void gen8_ppgtt_insert_huge(struct i915_vma *vma, +static void +xehpsdv_ppgtt_insert_huge(struct i915_address_space *vm, + struct i915_vma_resource *vma_res, + struct sgt_dma *iter, + enum i915_cache_level cache_level, + u32 flags) +{ + const gen8_pte_t pte_encode = vm->pte_encode(0, cache_level, flags); + unsigned int rem = sg_dma_len(iter->sg); + u64 start = vma_res->start; + + GEM_BUG_ON(!i915_vm_is_4lvl(vm)); + + do { + struct i915_page_directory * const pdp = + gen8_pdp_for_page_address(vm, start); + struct i915_page_directory * const pd = + i915_pd_entry(pdp, __gen8_pte_index(start, 2)); + struct i915_page_table *pt = + i915_pt_entry(pd, __gen8_pte_index(start, 1)); + gen8_pte_t encode = pte_encode; + unsigned int page_size; + gen8_pte_t *vaddr; + u16 index, max; + + max = I915_PDES; + + if (vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_2M && + IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && + rem >= I915_GTT_PAGE_SIZE_2M && + !__gen8_pte_index(start, 0)) { + index = __gen8_pte_index(start, 1); + encode |= GEN8_PDE_PS_2M; + page_size = I915_GTT_PAGE_SIZE_2M; + + vaddr = px_vaddr(pd); + } else { + if (encode & GEN12_PPGTT_PTE_LM) { + GEM_BUG_ON(__gen8_pte_index(start, 0) % 16); + GEM_BUG_ON(rem < I915_GTT_PAGE_SIZE_64K); + GEM_BUG_ON(!IS_ALIGNED(iter->dma, + I915_GTT_PAGE_SIZE_64K)); + + index = __gen8_pte_index(start, 0) / 16; + page_size = I915_GTT_PAGE_SIZE_64K; + + max /= 16; + + vaddr = px_vaddr(pd); + vaddr[__gen8_pte_index(start, 1)] |= GEN12_PDE_64K; + + pt->is_compact = true; + } else { + GEM_BUG_ON(pt->is_compact); + index = __gen8_pte_index(start, 0); + page_size = I915_GTT_PAGE_SIZE; + } + + vaddr = px_vaddr(pt); + } + + do { + GEM_BUG_ON(rem < page_size); + vaddr[index++] = encode | iter->dma; + + start += page_size; + iter->dma += page_size; + rem -= page_size; + if (iter->dma >= iter->max) { + iter->sg = __sg_next(iter->sg); + if (!iter->sg) + break; + + rem = sg_dma_len(iter->sg); + if (!rem) + break; + + iter->dma = sg_dma_address(iter->sg); + iter->max = iter->dma + rem; + + if (unlikely(!IS_ALIGNED(iter->dma, page_size))) + break; + } + } while (rem >= page_size && index < max); + + vma_res->page_sizes_gtt |= page_size; + } while (iter->sg && sg_dma_len(iter->sg)); +} + +static void gen8_ppgtt_insert_huge(struct i915_address_space *vm, + struct i915_vma_resource *vma_res, struct sgt_dma *iter, enum i915_cache_level cache_level, u32 flags) { const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags); unsigned int rem = sg_dma_len(iter->sg); - u64 start = vma->node.start; + u64 start = vma_res->start; - GEM_BUG_ON(!i915_vm_is_4lvl(vma->vm)); + GEM_BUG_ON(!i915_vm_is_4lvl(vm)); do { struct i915_page_directory * const pdp = - gen8_pdp_for_page_address(vma->vm, start); + gen8_pdp_for_page_address(vm, start); struct i915_page_directory * const pd = i915_pd_entry(pdp, __gen8_pte_index(start, 2)); gen8_pte_t encode = pte_encode; @@ -475,7 +575,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, gen8_pte_t *vaddr; u16 index; - if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M && + if (vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_2M && IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && rem >= I915_GTT_PAGE_SIZE_2M && !__gen8_pte_index(start, 0)) { @@ -492,7 +592,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, page_size = I915_GTT_PAGE_SIZE; if (!index && - vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K && + vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_64K && IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) || rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE)) @@ -541,9 +641,9 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, */ if (maybe_64K != -1 && (index == I915_PDES || - (i915_vm_has_scratch_64K(vma->vm) && - !iter->sg && IS_ALIGNED(vma->node.start + - vma->node.size, + (i915_vm_has_scratch_64K(vm) && + !iter->sg && IS_ALIGNED(vma_res->start + + vma_res->node_size, I915_GTT_PAGE_SIZE_2M)))) { vaddr = px_vaddr(pd); vaddr[maybe_64K] |= GEN8_PDE_IPS_64K; @@ -559,10 +659,10 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, * instead - which we detect as missing results during * selftests. */ - if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) { + if (I915_SELFTEST_ONLY(vm->scrub_64K)) { u16 i; - encode = vma->vm->scratch[0]->encode; + encode = vm->scratch[0]->encode; vaddr = px_vaddr(i915_pt_entry(pd, maybe_64K)); for (i = 1; i < index; i += 16) @@ -572,22 +672,25 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, } } - vma->page_sizes.gtt |= page_size; + vma_res->page_sizes_gtt |= page_size; } while (iter->sg && sg_dma_len(iter->sg)); } static void gen8_ppgtt_insert(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags) { struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm); - struct sgt_dma iter = sgt_dma(vma); + struct sgt_dma iter = sgt_dma(vma_res); - if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { - gen8_ppgtt_insert_huge(vma, &iter, cache_level, flags); + if (vma_res->bi.page_sizes.sg > I915_GTT_PAGE_SIZE) { + if (HAS_64K_PAGES(vm->i915)) + xehpsdv_ppgtt_insert_huge(vm, vma_res, &iter, cache_level, flags); + else + gen8_ppgtt_insert_huge(vm, vma_res, &iter, cache_level, flags); } else { - u64 idx = vma->node.start >> GEN8_PTE_SHIFT; + u64 idx = vma_res->start >> GEN8_PTE_SHIFT; do { struct i915_page_directory * const pdp = @@ -597,7 +700,7 @@ static void gen8_ppgtt_insert(struct i915_address_space *vm, cache_level, flags); } while (idx); - vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; + vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; } } @@ -612,13 +715,56 @@ static void gen8_ppgtt_insert_entry(struct i915_address_space *vm, gen8_pdp_for_page_index(vm, idx); struct i915_page_directory *pd = i915_pd_entry(pdp, gen8_pd_index(idx, 2)); + struct i915_page_table *pt = i915_pt_entry(pd, gen8_pd_index(idx, 1)); gen8_pte_t *vaddr; - vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1))); + GEM_BUG_ON(pt->is_compact); + + vaddr = px_vaddr(pt); vaddr[gen8_pd_index(idx, 0)] = gen8_pte_encode(addr, level, flags); clflush_cache_range(&vaddr[gen8_pd_index(idx, 0)], sizeof(*vaddr)); } +static void __xehpsdv_ppgtt_insert_entry_lm(struct i915_address_space *vm, + dma_addr_t addr, + u64 offset, + enum i915_cache_level level, + u32 flags) +{ + u64 idx = offset >> GEN8_PTE_SHIFT; + struct i915_page_directory * const pdp = + gen8_pdp_for_page_index(vm, idx); + struct i915_page_directory *pd = + i915_pd_entry(pdp, gen8_pd_index(idx, 2)); + struct i915_page_table *pt = i915_pt_entry(pd, gen8_pd_index(idx, 1)); + gen8_pte_t *vaddr; + + GEM_BUG_ON(!IS_ALIGNED(addr, SZ_64K)); + GEM_BUG_ON(!IS_ALIGNED(offset, SZ_64K)); + + if (!pt->is_compact) { + vaddr = px_vaddr(pd); + vaddr[gen8_pd_index(idx, 1)] |= GEN12_PDE_64K; + pt->is_compact = true; + } + + vaddr = px_vaddr(pt); + vaddr[gen8_pd_index(idx, 0) / 16] = gen8_pte_encode(addr, level, flags); +} + +static void xehpsdv_ppgtt_insert_entry(struct i915_address_space *vm, + dma_addr_t addr, + u64 offset, + enum i915_cache_level level, + u32 flags) +{ + if (flags & PTE_LM) + return __xehpsdv_ppgtt_insert_entry_lm(vm, addr, offset, + level, flags); + + return gen8_ppgtt_insert_entry(vm, addr, offset, level, flags); +} + static int gen8_init_scratch(struct i915_address_space *vm) { u32 pte_flags; @@ -818,7 +964,10 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt, ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; ppgtt->vm.insert_entries = gen8_ppgtt_insert; - ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; + if (HAS_64K_PAGES(gt->i915)) + ppgtt->vm.insert_page = xehpsdv_ppgtt_insert_entry; + else + ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; ppgtt->vm.clear_range = gen8_ppgtt_clear; ppgtt->vm.foreach = gen8_ppgtt_foreach; diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index ba083d800a08..5d0ec7c49b6a 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -79,7 +79,8 @@ static int intel_context_active_acquire(struct intel_context *ce) __i915_active_acquire(&ce->active); - if (intel_context_is_barrier(ce) || intel_engine_uses_guc(ce->engine)) + if (intel_context_is_barrier(ce) || intel_engine_uses_guc(ce->engine) || + intel_context_is_parallel(ce)) return 0; /* Preallocate tracking nodes */ @@ -563,7 +564,6 @@ void intel_context_bind_parent_child(struct intel_context *parent, * Callers responsibility to validate that this function is used * correctly but we use GEM_BUG_ON here ensure that they do. */ - GEM_BUG_ON(!intel_engine_uses_guc(parent->engine)); GEM_BUG_ON(intel_context_is_pinned(parent)); GEM_BUG_ON(intel_context_is_child(parent)); GEM_BUG_ON(intel_context_is_pinned(child)); diff --git a/drivers/gpu/drm/i915/gt/intel_context_sseu.c b/drivers/gpu/drm/i915/gt/intel_context_sseu.c index e86d8255feec..ece16c2b5b8e 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_sseu.c +++ b/drivers/gpu/drm/i915/gt/intel_context_sseu.c @@ -9,6 +9,7 @@ #include "intel_engine_pm.h" #include "intel_gpu_commands.h" #include "intel_lrc.h" +#include "intel_lrc_reg.h" #include "intel_ring.h" #include "intel_sseu.h" diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h index 08559ace0ada..1c0ab05c3c40 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine.h +++ b/drivers/gpu/drm/i915/gt/intel_engine.h @@ -11,7 +11,6 @@ #include <linux/seqlock.h> #include "i915_pmu.h" -#include "i915_reg.h" #include "i915_request.h" #include "i915_selftest.h" #include "intel_engine_types.h" @@ -183,6 +182,8 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value) #define I915_HWS_CSB_BUF0_INDEX 0x10 #define I915_HWS_CSB_WRITE_INDEX 0x1f #define ICL_HWS_CSB_WRITE_INDEX 0x2f +#define INTEL_HWS_CSB_WRITE_INDEX(__i915) \ + (GRAPHICS_VER(__i915) >= 11 ? ICL_HWS_CSB_WRITE_INDEX : I915_HWS_CSB_WRITE_INDEX) void intel_engine_stop(struct intel_engine_cs *engine); void intel_engine_cleanup(struct intel_engine_cs *engine); @@ -264,6 +265,8 @@ intel_engine_create_pinned_context(struct intel_engine_cs *engine, void intel_engine_destroy_pinned_context(struct intel_context *ce); +void xehp_enable_ccs_engines(struct intel_engine_cs *engine); + #define ENGINE_PHYSICAL 0 #define ENGINE_MOCK 1 #define ENGINE_VIRTUAL 2 diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 352254e001b4..e1aa78b20d2d 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -6,18 +6,22 @@ #include <drm/drm_print.h> #include "gem/i915_gem_context.h" +#include "gem/i915_gem_internal.h" +#include "gt/intel_gt_regs.h" +#include "i915_cmd_parser.h" #include "i915_drv.h" - #include "intel_breadcrumbs.h" #include "intel_context.h" #include "intel_engine.h" #include "intel_engine_pm.h" +#include "intel_engine_regs.h" #include "intel_engine_user.h" #include "intel_execlists_submission.h" #include "intel_gt.h" #include "intel_gt_requests.h" #include "intel_gt_pm.h" +#include "intel_lrc.h" #include "intel_lrc_reg.h" #include "intel_reset.h" #include "intel_ring.h" @@ -153,6 +157,34 @@ static const struct engine_info intel_engines[] = { { .graphics_ver = 12, .base = XEHP_VEBOX4_RING_BASE } }, }, + [CCS0] = { + .class = COMPUTE_CLASS, + .instance = 0, + .mmio_bases = { + { .graphics_ver = 12, .base = GEN12_COMPUTE0_RING_BASE } + } + }, + [CCS1] = { + .class = COMPUTE_CLASS, + .instance = 1, + .mmio_bases = { + { .graphics_ver = 12, .base = GEN12_COMPUTE1_RING_BASE } + } + }, + [CCS2] = { + .class = COMPUTE_CLASS, + .instance = 2, + .mmio_bases = { + { .graphics_ver = 12, .base = GEN12_COMPUTE2_RING_BASE } + } + }, + [CCS3] = { + .class = COMPUTE_CLASS, + .instance = 3, + .mmio_bases = { + { .graphics_ver = 12, .base = GEN12_COMPUTE3_RING_BASE } + } + }, }; /** @@ -177,6 +209,8 @@ u32 intel_engine_context_size(struct intel_gt *gt, u8 class) BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE); switch (class) { + case COMPUTE_CLASS: + fallthrough; case RENDER_CLASS: switch (GRAPHICS_VER(gt->i915)) { default: @@ -290,6 +324,50 @@ static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir) GEM_DEBUG_WARN_ON(iir); } +static u32 get_reset_domain(u8 ver, enum intel_engine_id id) +{ + u32 reset_domain; + + if (ver >= 11) { + static const u32 engine_reset_domains[] = { + [RCS0] = GEN11_GRDOM_RENDER, + [BCS0] = GEN11_GRDOM_BLT, + [VCS0] = GEN11_GRDOM_MEDIA, + [VCS1] = GEN11_GRDOM_MEDIA2, + [VCS2] = GEN11_GRDOM_MEDIA3, + [VCS3] = GEN11_GRDOM_MEDIA4, + [VCS4] = GEN11_GRDOM_MEDIA5, + [VCS5] = GEN11_GRDOM_MEDIA6, + [VCS6] = GEN11_GRDOM_MEDIA7, + [VCS7] = GEN11_GRDOM_MEDIA8, + [VECS0] = GEN11_GRDOM_VECS, + [VECS1] = GEN11_GRDOM_VECS2, + [VECS2] = GEN11_GRDOM_VECS3, + [VECS3] = GEN11_GRDOM_VECS4, + [CCS0] = GEN11_GRDOM_RENDER, + [CCS1] = GEN11_GRDOM_RENDER, + [CCS2] = GEN11_GRDOM_RENDER, + [CCS3] = GEN11_GRDOM_RENDER, + }; + GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || + !engine_reset_domains[id]); + reset_domain = engine_reset_domains[id]; + } else { + static const u32 engine_reset_domains[] = { + [RCS0] = GEN6_GRDOM_RENDER, + [BCS0] = GEN6_GRDOM_BLT, + [VCS0] = GEN6_GRDOM_MEDIA, + [VCS1] = GEN8_GRDOM_MEDIA2, + [VECS0] = GEN6_GRDOM_VECS, + }; + GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || + !engine_reset_domains[id]); + reset_domain = engine_reset_domains[id]; + } + + return reset_domain; +} + static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, u8 logical_instance) { @@ -325,38 +403,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, engine->id = id; engine->legacy_idx = INVALID_ENGINE; engine->mask = BIT(id); - if (GRAPHICS_VER(gt->i915) >= 11) { - static const u32 engine_reset_domains[] = { - [RCS0] = GEN11_GRDOM_RENDER, - [BCS0] = GEN11_GRDOM_BLT, - [VCS0] = GEN11_GRDOM_MEDIA, - [VCS1] = GEN11_GRDOM_MEDIA2, - [VCS2] = GEN11_GRDOM_MEDIA3, - [VCS3] = GEN11_GRDOM_MEDIA4, - [VCS4] = GEN11_GRDOM_MEDIA5, - [VCS5] = GEN11_GRDOM_MEDIA6, - [VCS6] = GEN11_GRDOM_MEDIA7, - [VCS7] = GEN11_GRDOM_MEDIA8, - [VECS0] = GEN11_GRDOM_VECS, - [VECS1] = GEN11_GRDOM_VECS2, - [VECS2] = GEN11_GRDOM_VECS3, - [VECS3] = GEN11_GRDOM_VECS4, - }; - GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || - !engine_reset_domains[id]); - engine->reset_domain = engine_reset_domains[id]; - } else { - static const u32 engine_reset_domains[] = { - [RCS0] = GEN6_GRDOM_RENDER, - [BCS0] = GEN6_GRDOM_BLT, - [VCS0] = GEN6_GRDOM_MEDIA, - [VCS1] = GEN8_GRDOM_MEDIA2, - [VECS0] = GEN6_GRDOM_VECS, - }; - GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || - !engine_reset_domains[id]); - engine->reset_domain = engine_reset_domains[id]; - } + engine->reset_domain = get_reset_domain(GRAPHICS_VER(gt->i915), + id); engine->i915 = i915; engine->gt = gt; engine->uncore = gt->uncore; @@ -386,6 +434,12 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id, if (GRAPHICS_VER(i915) == 12 && engine->class == RENDER_CLASS) engine->props.preempt_timeout_ms = 0; + /* features common between engines sharing EUs */ + if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) { + engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE; + engine->flags |= I915_ENGINE_HAS_EU_PRIORITY; + } + engine->defaults = engine->props; /* never to change again */ engine->context_size = intel_engine_context_size(gt, engine->class); @@ -538,6 +592,29 @@ bool gen11_vdbox_has_sfc(struct intel_gt *gt, return false; } +static void engine_mask_apply_compute_fuses(struct intel_gt *gt) +{ + struct drm_i915_private *i915 = gt->i915; + struct intel_gt_info *info = >->info; + int ss_per_ccs = info->sseu.max_subslices / I915_MAX_CCS; + unsigned long ccs_mask; + unsigned int i; + + if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) + return; + + ccs_mask = intel_slicemask_from_dssmask(intel_sseu_get_compute_subslices(&info->sseu), + ss_per_ccs); + /* + * If all DSS in a quadrant are fused off, the corresponding CCS + * engine is not available for use. + */ + for_each_clear_bit(i, &ccs_mask, I915_MAX_CCS) { + info->engine_mask &= ~BIT(_CCS(i)); + drm_dbg(&i915->drm, "ccs%u fused off\n", i); + } +} + /* * Determine which engines are fused off in our particular hardware. * Note that we have a catch-22 situation where we need to be able to access @@ -619,6 +696,8 @@ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) vebox_mask, VEBOX_MASK(gt)); GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt)); + engine_mask_apply_compute_fuses(gt); + return info->engine_mask; } @@ -1227,17 +1306,6 @@ void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine) ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); } -const char *i915_cache_level_str(struct drm_i915_private *i915, int type) -{ - switch (type) { - case I915_CACHE_NONE: return " uncached"; - case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped"; - case I915_CACHE_L3_LLC: return " L3+LLC"; - case I915_CACHE_WT: return " WT"; - default: return ""; - } -} - static u32 read_subslice_reg(const struct intel_engine_cs *engine, int slice, int subslice, i915_reg_t reg) @@ -1708,18 +1776,15 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, static void print_request_ring(struct drm_printer *m, struct i915_request *rq) { - struct i915_vma_snapshot *vsnap = &rq->batch_snapshot; + struct i915_vma_resource *vma_res = rq->batch_res; void *ring; int size; - if (!i915_vma_snapshot_present(vsnap)) - vsnap = NULL; - drm_printf(m, "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n", rq->head, rq->postfix, rq->tail, - vsnap ? upper_32_bits(vsnap->gtt_offset) : ~0u, - vsnap ? lower_32_bits(vsnap->gtt_offset) : ~0u); + vma_res ? upper_32_bits(vma_res->start) : ~0u, + vma_res ? lower_32_bits(vma_res->start) : ~0u); size = rq->tail - rq->head; if (rq->tail < rq->head) @@ -2030,6 +2095,23 @@ intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine) return active; } +void xehp_enable_ccs_engines(struct intel_engine_cs *engine) +{ + /* + * If there are any non-fused-off CCS engines, we need to enable CCS + * support in the RCU_MODE register. This only needs to be done once, + * so for simplicity we'll take care of this in the RCS engine's + * resume handler; since the RCS and all CCS engines belong to the + * same reset domain and are reset together, this will also take care + * of re-applying the setting after i915-triggered resets. + */ + if (!CCS_MASK(engine->gt)) + return; + + intel_uncore_write(engine->uncore, GEN12_RCU_MODE, + _MASKED_BIT_ENABLE(GEN12_RCU_MODE_CCS_ENABLE)); +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "mock_engine.c" #include "selftest_engine.c" diff --git a/drivers/gpu/drm/i915/gt/intel_engine_regs.h b/drivers/gpu/drm/i915/gt/intel_engine_regs.h new file mode 100644 index 000000000000..0bf8b45c9319 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_engine_regs.h @@ -0,0 +1,246 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_ENGINE_REGS__ +#define __INTEL_ENGINE_REGS__ + +#include "i915_reg_defs.h" + +#define RING_TAIL(base) _MMIO((base) + 0x30) +#define TAIL_ADDR 0x001FFFF8 +#define RING_HEAD(base) _MMIO((base) + 0x34) +#define HEAD_WRAP_COUNT 0xFFE00000 +#define HEAD_WRAP_ONE 0x00200000 +#define HEAD_ADDR 0x001FFFFC +#define RING_START(base) _MMIO((base) + 0x38) +#define RING_CTL(base) _MMIO((base) + 0x3c) +#define RING_CTL_SIZE(size) ((size) - PAGE_SIZE) /* in bytes -> pages */ +#define RING_NR_PAGES 0x001FF000 +#define RING_REPORT_MASK 0x00000006 +#define RING_REPORT_64K 0x00000002 +#define RING_REPORT_128K 0x00000004 +#define RING_NO_REPORT 0x00000000 +#define RING_VALID_MASK 0x00000001 +#define RING_VALID 0x00000001 +#define RING_INVALID 0x00000000 +#define RING_WAIT_I8XX (1 << 0) /* gen2, PRBx_HEAD */ +#define RING_WAIT (1 << 11) /* gen3+, PRBx_CTL */ +#define RING_WAIT_SEMAPHORE (1 << 10) /* gen6+ */ +#define RING_SYNC_0(base) _MMIO((base) + 0x40) +#define RING_SYNC_1(base) _MMIO((base) + 0x44) +#define RING_SYNC_2(base) _MMIO((base) + 0x48) +#define GEN6_RVSYNC (RING_SYNC_0(RENDER_RING_BASE)) +#define GEN6_RBSYNC (RING_SYNC_1(RENDER_RING_BASE)) +#define GEN6_RVESYNC (RING_SYNC_2(RENDER_RING_BASE)) +#define GEN6_VBSYNC (RING_SYNC_0(GEN6_BSD_RING_BASE)) +#define GEN6_VRSYNC (RING_SYNC_1(GEN6_BSD_RING_BASE)) +#define GEN6_VVESYNC (RING_SYNC_2(GEN6_BSD_RING_BASE)) +#define GEN6_BRSYNC (RING_SYNC_0(BLT_RING_BASE)) +#define GEN6_BVSYNC (RING_SYNC_1(BLT_RING_BASE)) +#define GEN6_BVESYNC (RING_SYNC_2(BLT_RING_BASE)) +#define GEN6_VEBSYNC (RING_SYNC_0(VEBOX_RING_BASE)) +#define GEN6_VERSYNC (RING_SYNC_1(VEBOX_RING_BASE)) +#define GEN6_VEVSYNC (RING_SYNC_2(VEBOX_RING_BASE)) +#define RING_PSMI_CTL(base) _MMIO((base) + 0x50) +#define GEN8_RC_SEMA_IDLE_MSG_DISABLE REG_BIT(12) +#define GEN8_FF_DOP_CLOCK_GATE_DISABLE REG_BIT(10) +#define GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE REG_BIT(7) +#define GEN6_BSD_GO_INDICATOR REG_BIT(4) +#define GEN6_BSD_SLEEP_INDICATOR REG_BIT(3) +#define GEN6_BSD_SLEEP_FLUSH_DISABLE REG_BIT(2) +#define GEN6_PSMI_SLEEP_MSG_DISABLE REG_BIT(0) +#define RING_MAX_IDLE(base) _MMIO((base) + 0x54) +#define PWRCTX_MAXCNT(base) _MMIO((base) + 0x54) +#define IDLE_TIME_MASK 0xFFFFF +#define RING_ACTHD_UDW(base) _MMIO((base) + 0x5c) +#define RING_DMA_FADD_UDW(base) _MMIO((base) + 0x60) /* gen8+ */ +#define RING_IPEIR(base) _MMIO((base) + 0x64) +#define RING_IPEHR(base) _MMIO((base) + 0x68) +#define RING_INSTDONE(base) _MMIO((base) + 0x6c) +#define RING_INSTPS(base) _MMIO((base) + 0x70) +#define RING_DMA_FADD(base) _MMIO((base) + 0x78) +#define RING_ACTHD(base) _MMIO((base) + 0x74) +#define RING_HWS_PGA(base) _MMIO((base) + 0x80) +#define RING_CMD_BUF_CCTL(base) _MMIO((base) + 0x84) +#define IPEIR(base) _MMIO((base) + 0x88) +#define IPEHR(base) _MMIO((base) + 0x8c) +#define RING_ID(base) _MMIO((base) + 0x8c) +#define RING_NOPID(base) _MMIO((base) + 0x94) +#define RING_HWSTAM(base) _MMIO((base) + 0x98) +#define RING_MI_MODE(base) _MMIO((base) + 0x9c) +#define ASYNC_FLIP_PERF_DISABLE REG_BIT(14) +#define MI_FLUSH_ENABLE REG_BIT(12) +#define TGL_NESTED_BB_EN REG_BIT(12) +#define MODE_IDLE REG_BIT(9) +#define STOP_RING REG_BIT(8) +#define VS_TIMER_DISPATCH REG_BIT(6) +#define RING_IMR(base) _MMIO((base) + 0xa8) +#define RING_EIR(base) _MMIO((base) + 0xb0) +#define RING_EMR(base) _MMIO((base) + 0xb4) +#define RING_ESR(base) _MMIO((base) + 0xb8) +#define RING_INSTPM(base) _MMIO((base) + 0xc0) +#define RING_CMD_CCTL(base) _MMIO((base) + 0xc4) +#define ACTHD(base) _MMIO((base) + 0xc8) +#define GEN8_R_PWR_CLK_STATE(base) _MMIO((base) + 0xc8) +#define GEN8_RPCS_ENABLE (1 << 31) +#define GEN8_RPCS_S_CNT_ENABLE (1 << 18) +#define GEN8_RPCS_S_CNT_SHIFT 15 +#define GEN8_RPCS_S_CNT_MASK (0x7 << GEN8_RPCS_S_CNT_SHIFT) +#define GEN11_RPCS_S_CNT_SHIFT 12 +#define GEN11_RPCS_S_CNT_MASK (0x3f << GEN11_RPCS_S_CNT_SHIFT) +#define GEN8_RPCS_SS_CNT_ENABLE (1 << 11) +#define GEN8_RPCS_SS_CNT_SHIFT 8 +#define GEN8_RPCS_SS_CNT_MASK (0x7 << GEN8_RPCS_SS_CNT_SHIFT) +#define GEN8_RPCS_EU_MAX_SHIFT 4 +#define GEN8_RPCS_EU_MAX_MASK (0xf << GEN8_RPCS_EU_MAX_SHIFT) +#define GEN8_RPCS_EU_MIN_SHIFT 0 +#define GEN8_RPCS_EU_MIN_MASK (0xf << GEN8_RPCS_EU_MIN_SHIFT) + +#define RING_RESET_CTL(base) _MMIO((base) + 0xd0) +#define RESET_CTL_CAT_ERROR REG_BIT(2) +#define RESET_CTL_READY_TO_RESET REG_BIT(1) +#define RESET_CTL_REQUEST_RESET REG_BIT(0) +#define DMA_FADD_I8XX(base) _MMIO((base) + 0xd0) +#define RING_BBSTATE(base) _MMIO((base) + 0x110) +#define RING_BB_PPGTT (1 << 5) +#define RING_SBBADDR(base) _MMIO((base) + 0x114) /* hsw+ */ +#define RING_SBBSTATE(base) _MMIO((base) + 0x118) /* hsw+ */ +#define RING_SBBADDR_UDW(base) _MMIO((base) + 0x11c) /* gen8+ */ +#define RING_BBADDR(base) _MMIO((base) + 0x140) +#define RING_BBADDR_UDW(base) _MMIO((base) + 0x168) /* gen8+ */ +#define CCID(base) _MMIO((base) + 0x180) +#define CCID_EN BIT(0) +#define CCID_EXTENDED_STATE_RESTORE BIT(2) +#define CCID_EXTENDED_STATE_SAVE BIT(3) +#define RING_BB_PER_CTX_PTR(base) _MMIO((base) + 0x1c0) /* gen8+ */ +#define RING_INDIRECT_CTX(base) _MMIO((base) + 0x1c4) /* gen8+ */ +#define RING_INDIRECT_CTX_OFFSET(base) _MMIO((base) + 0x1c8) /* gen8+ */ +#define ECOSKPD(base) _MMIO((base) + 0x1d0) +#define ECO_CONSTANT_BUFFER_SR_DISABLE REG_BIT(4) +#define ECO_GATING_CX_ONLY REG_BIT(3) +#define GEN6_BLITTER_FBC_NOTIFY REG_BIT(3) +#define ECO_FLIP_DONE REG_BIT(0) +#define GEN6_BLITTER_LOCK_SHIFT 16 + +#define BLIT_CCTL(base) _MMIO((base) + 0x204) +#define BLIT_CCTL_DST_MOCS_MASK REG_GENMASK(14, 8) +#define BLIT_CCTL_SRC_MOCS_MASK REG_GENMASK(6, 0) +#define BLIT_CCTL_MASK (BLIT_CCTL_DST_MOCS_MASK | \ + BLIT_CCTL_SRC_MOCS_MASK) +#define BLIT_CCTL_MOCS(dst, src) \ + (REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, (dst) << 1) | \ + REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, (src) << 1)) + +/* + * CMD_CCTL read/write fields take a MOCS value and _not_ a table index. + * The lsb of each can be considered a separate enabling bit for encryption. + * 6:0 == default MOCS value for reads => 6:1 == table index for reads. + * 13:7 == default MOCS value for writes => 13:8 == table index for writes. + * 15:14 == Reserved => 31:30 are set to 0. + */ +#define CMD_CCTL_WRITE_OVERRIDE_MASK REG_GENMASK(13, 7) +#define CMD_CCTL_READ_OVERRIDE_MASK REG_GENMASK(6, 0) +#define CMD_CCTL_MOCS_MASK (CMD_CCTL_WRITE_OVERRIDE_MASK | \ + CMD_CCTL_READ_OVERRIDE_MASK) +#define CMD_CCTL_MOCS_OVERRIDE(write, read) \ + (REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, (write) << 1) | \ + REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, (read) << 1)) + +#define MI_PREDICATE_RESULT_2(base) _MMIO((base) + 0x3bc) +#define LOWER_SLICE_ENABLED (1 << 0) +#define LOWER_SLICE_DISABLED (0 << 0) +#define MI_PREDICATE_SRC0(base) _MMIO((base) + 0x400) +#define MI_PREDICATE_SRC0_UDW(base) _MMIO((base) + 0x400 + 4) +#define MI_PREDICATE_SRC1(base) _MMIO((base) + 0x408) +#define MI_PREDICATE_SRC1_UDW(base) _MMIO((base) + 0x408 + 4) +#define MI_PREDICATE_DATA(base) _MMIO((base) + 0x410) +#define MI_PREDICATE_RESULT(base) _MMIO((base) + 0x418) +#define MI_PREDICATE_RESULT_1(base) _MMIO((base) + 0x41c) + +#define RING_PP_DIR_DCLV(base) _MMIO((base) + 0x220) +#define PP_DIR_DCLV_2G 0xffffffff +#define RING_PP_DIR_BASE(base) _MMIO((base) + 0x228) +#define RING_ELSP(base) _MMIO((base) + 0x230) +#define RING_EXECLIST_STATUS_LO(base) _MMIO((base) + 0x234) +#define RING_EXECLIST_STATUS_HI(base) _MMIO((base) + 0x234 + 4) +#define RING_CONTEXT_CONTROL(base) _MMIO((base) + 0x244) +#define CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT REG_BIT(0) +#define CTX_CTRL_RS_CTX_ENABLE REG_BIT(1) +#define CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT REG_BIT(2) +#define CTX_CTRL_INHIBIT_SYN_CTX_SWITCH REG_BIT(3) +#define GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE REG_BIT(8) +#define RING_SEMA_WAIT_POLL(base) _MMIO((base) + 0x24c) +#define GEN8_RING_PDP_UDW(base, n) _MMIO((base) + 0x270 + (n) * 8 + 4) +#define GEN8_RING_PDP_LDW(base, n) _MMIO((base) + 0x270 + (n) * 8) +#define RING_MODE_GEN7(base) _MMIO((base) + 0x29c) +#define GFX_RUN_LIST_ENABLE (1 << 15) +#define GFX_INTERRUPT_STEERING (1 << 14) +#define GFX_TLB_INVALIDATE_EXPLICIT (1 << 13) +#define GFX_SURFACE_FAULT_ENABLE (1 << 12) +#define GFX_REPLAY_MODE (1 << 11) +#define GFX_PSMI_GRANULARITY (1 << 10) +#define GFX_PPGTT_ENABLE (1 << 9) +#define GEN8_GFX_PPGTT_48B (1 << 7) +#define GFX_FORWARD_VBLANK_MASK (3 << 5) +#define GFX_FORWARD_VBLANK_NEVER (0 << 5) +#define GFX_FORWARD_VBLANK_ALWAYS (1 << 5) +#define GFX_FORWARD_VBLANK_COND (2 << 5) +#define GEN11_GFX_DISABLE_LEGACY_MODE (1 << 3) +#define RING_TIMESTAMP(base) _MMIO((base) + 0x358) +#define RING_TIMESTAMP_UDW(base) _MMIO((base) + 0x358 + 4) +#define RING_CONTEXT_STATUS_PTR(base) _MMIO((base) + 0x3a0) +#define RING_CTX_TIMESTAMP(base) _MMIO((base) + 0x3a8) /* gen8+ */ +#define RING_FORCE_TO_NONPRIV(base, i) _MMIO(((base) + 0x4D0) + (i) * 4) +#define RING_FORCE_TO_NONPRIV_ADDRESS_MASK REG_GENMASK(25, 2) +#define RING_FORCE_TO_NONPRIV_ACCESS_RW (0 << 28) /* CFL+ & Gen11+ */ +#define RING_FORCE_TO_NONPRIV_ACCESS_RD (1 << 28) +#define RING_FORCE_TO_NONPRIV_ACCESS_WR (2 << 28) +#define RING_FORCE_TO_NONPRIV_ACCESS_INVALID (3 << 28) +#define RING_FORCE_TO_NONPRIV_ACCESS_MASK (3 << 28) +#define RING_FORCE_TO_NONPRIV_RANGE_1 (0 << 0) /* CFL+ & Gen11+ */ +#define RING_FORCE_TO_NONPRIV_RANGE_4 (1 << 0) +#define RING_FORCE_TO_NONPRIV_RANGE_16 (2 << 0) +#define RING_FORCE_TO_NONPRIV_RANGE_64 (3 << 0) +#define RING_FORCE_TO_NONPRIV_RANGE_MASK (3 << 0) +#define RING_FORCE_TO_NONPRIV_MASK_VALID \ + (RING_FORCE_TO_NONPRIV_RANGE_MASK | RING_FORCE_TO_NONPRIV_ACCESS_MASK) +#define RING_MAX_NONPRIV_SLOTS 12 + +#define RING_EXECLIST_SQ_CONTENTS(base) _MMIO((base) + 0x510) +#define RING_PP_DIR_BASE_READ(base) _MMIO((base) + 0x518) +#define RING_EXECLIST_CONTROL(base) _MMIO((base) + 0x550) +#define EL_CTRL_LOAD REG_BIT(0) + +/* There are 16 64-bit CS General Purpose Registers per-engine on Gen8+ */ +#define GEN8_RING_CS_GPR(base, n) _MMIO((base) + 0x600 + (n) * 8) +#define GEN8_RING_CS_GPR_UDW(base, n) _MMIO((base) + 0x600 + (n) * 8 + 4) + +#define GEN11_VCS_SFC_FORCED_LOCK(base) _MMIO((base) + 0x88c) +#define GEN11_VCS_SFC_FORCED_LOCK_BIT (1 << 0) +#define GEN11_VCS_SFC_LOCK_STATUS(base) _MMIO((base) + 0x890) +#define GEN11_VCS_SFC_USAGE_BIT (1 << 0) +#define GEN11_VCS_SFC_LOCK_ACK_BIT (1 << 1) + +#define GEN11_VECS_SFC_FORCED_LOCK(base) _MMIO((base) + 0x201c) +#define GEN11_VECS_SFC_FORCED_LOCK_BIT (1 << 0) +#define GEN11_VECS_SFC_LOCK_ACK(base) _MMIO((base) + 0x2018) +#define GEN11_VECS_SFC_LOCK_ACK_BIT (1 << 0) +#define GEN11_VECS_SFC_USAGE(base) _MMIO((base) + 0x2014) +#define GEN11_VECS_SFC_USAGE_BIT (1 << 0) + +#define RING_HWS_PGA_GEN6(base) _MMIO((base) + 0x2080) + +#define GEN12_HCP_SFC_LOCK_STATUS(base) _MMIO((base) + 0x2914) +#define GEN12_HCP_SFC_LOCK_ACK_BIT REG_BIT(1) +#define GEN12_HCP_SFC_USAGE_BIT REG_BIT(0) + +#define VDBOX_CGCTL3F10(base) _MMIO((base) + 0x3f10) +#define IECPUNIT_CLKGATE_DIS REG_BIT(22) + +#define VDBOX_CGCTL3F18(base) _MMIO((base) + 0x3f18) +#define ALNUNIT_CLKGATE_DIS REG_BIT(13) + + +#endif /* __INTEL_ENGINE_REGS__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 36365bdbe1ee..19ff8758e34d 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -33,7 +33,8 @@ #define VIDEO_ENHANCEMENT_CLASS 2 #define COPY_ENGINE_CLASS 3 #define OTHER_CLASS 4 -#define MAX_ENGINE_CLASS 4 +#define COMPUTE_CLASS 5 +#define MAX_ENGINE_CLASS 5 #define MAX_ENGINE_INSTANCE 7 #define I915_MAX_SLICES 3 @@ -95,6 +96,7 @@ struct i915_ctx_workarounds { #define I915_MAX_VCS 8 #define I915_MAX_VECS 4 +#define I915_MAX_CCS 4 /* * Engine IDs definitions. @@ -117,6 +119,11 @@ enum intel_engine_id { VECS2, VECS3, #define _VECS(n) (VECS0 + (n)) + CCS0, + CCS1, + CCS2, + CCS3, +#define _CCS(n) (CCS0 + (n)) I915_NUM_ENGINES #define INVALID_ENGINE ((enum intel_engine_id)-1) }; @@ -517,6 +524,8 @@ struct intel_engine_cs { #define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6) #define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7) #define I915_ENGINE_WANT_FORCED_PREEMPTION BIT(8) +#define I915_ENGINE_HAS_RCS_REG_STATE BIT(9) +#define I915_ENGINE_HAS_EU_PRIORITY BIT(10) unsigned int flags; /* diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c index 9ce85a845105..b8c9b6b89003 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_user.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c @@ -47,6 +47,7 @@ static const u8 uabi_classes[] = { [COPY_ENGINE_CLASS] = I915_ENGINE_CLASS_COPY, [VIDEO_DECODE_CLASS] = I915_ENGINE_CLASS_VIDEO, [VIDEO_ENHANCEMENT_CLASS] = I915_ENGINE_CLASS_VIDEO_ENHANCE, + /* TODO: Add COMPUTE_CLASS mapping once ABI is available */ }; static int engine_cmp(void *priv, const struct list_head *A, @@ -139,6 +140,7 @@ const char *intel_engine_class_repr(u8 class) [COPY_ENGINE_CLASS] = "bcs", [VIDEO_DECODE_CLASS] = "vcs", [VIDEO_ENHANCEMENT_CLASS] = "vecs", + [COMPUTE_CLASS] = "ccs", }; if (class >= ARRAY_SIZE(uabi_names) || !uabi_names[class]) @@ -162,6 +164,7 @@ static int legacy_ring_idx(const struct legacy_ring *ring) [COPY_ENGINE_CLASS] = { BCS0, 1 }, [VIDEO_DECODE_CLASS] = { VCS0, I915_MAX_VCS }, [VIDEO_ENHANCEMENT_CLASS] = { VECS0, I915_MAX_VECS }, + [COMPUTE_CLASS] = { CCS0, I915_MAX_CCS }, }; if (GEM_DEBUG_WARN_ON(ring->class >= ARRAY_SIZE(map))) @@ -190,7 +193,7 @@ static void add_legacy_ring(struct legacy_ring *ring, void intel_engines_driver_register(struct drm_i915_private *i915) { struct legacy_ring ring = {}; - u8 uabi_instances[4] = {}; + u8 uabi_instances[5] = {}; struct list_head *it, *next; struct rb_node **p, *prev; LIST_HEAD(engines); diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index a69df5e9e77a..3e0c81f06bd0 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -116,11 +116,13 @@ #include "intel_context.h" #include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" +#include "intel_engine_regs.h" #include "intel_engine_stats.h" #include "intel_execlists_submission.h" #include "intel_gt.h" #include "intel_gt_irq.h" #include "intel_gt_pm.h" +#include "intel_gt_regs.h" #include "intel_gt_requests.h" #include "intel_lrc.h" #include "intel_lrc_reg.h" @@ -663,9 +665,13 @@ static inline void execlists_schedule_out(struct i915_request *rq) static u64 execlists_update_context(struct i915_request *rq) { struct intel_context *ce = rq->context; - u64 desc = ce->lrc.desc; + u64 desc; u32 tail, prev; + desc = ce->lrc.desc; + if (rq->engine->flags & I915_ENGINE_HAS_EU_PRIORITY) + desc |= lrc_desc_priority(rq_prio(rq)); + /* * WaIdleLiteRestore:bdw,skl * @@ -2599,6 +2605,43 @@ static void execlists_context_cancel_request(struct intel_context *ce, current->comm); } +static struct intel_context * +execlists_create_parallel(struct intel_engine_cs **engines, + unsigned int num_siblings, + unsigned int width) +{ + struct intel_context *parent = NULL, *ce, *err; + int i; + + GEM_BUG_ON(num_siblings != 1); + + for (i = 0; i < width; ++i) { + ce = intel_context_create(engines[i]); + if (IS_ERR(ce)) { + err = ce; + goto unwind; + } + + if (i == 0) + parent = ce; + else + intel_context_bind_parent_child(parent, ce); + } + + parent->parallel.fence_context = dma_fence_context_alloc(1); + + intel_context_set_nopreempt(parent); + for_each_child(parent, ce) + intel_context_set_nopreempt(ce); + + return parent; + +unwind: + if (parent) + intel_context_put(parent); + return err; +} + static const struct intel_context_ops execlists_context_ops = { .flags = COPS_HAS_INFLIGHT, @@ -2617,6 +2660,7 @@ static const struct intel_context_ops execlists_context_ops = { .reset = lrc_reset, .destroy = lrc_destroy, + .create_parallel = execlists_create_parallel, .create_virtual = execlists_create_virtual, }; @@ -2867,6 +2911,9 @@ static int execlists_resume(struct intel_engine_cs *engine) enable_execlists(engine); + if (engine->class == RENDER_CLASS) + xehp_enable_ccs_engines(engine); + return 0; } @@ -3440,7 +3487,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) logical_ring_default_vfuncs(engine); logical_ring_default_irqs(engine); - if (engine->class == RENDER_CLASS) + if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) rcs_submission_override(engine); lrc_init_wa_ctx(engine); @@ -3463,7 +3510,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) (u64 *)&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; execlists->csb_write = - &engine->status_page.addr[intel_hws_csb_write_index(i915)]; + &engine->status_page.addr[INTEL_HWS_CSB_WRITE_INDEX(i915)]; if (GRAPHICS_VER(i915) < 11) execlists->csb_size = GEN8_CSB_ENTRIES; diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 5263dda7f8d5..8850d4e0f9cc 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -15,6 +15,7 @@ #include "gem/i915_gem_lmem.h" #include "intel_gt.h" +#include "intel_gt_regs.h" #include "i915_drv.h" #include "i915_scatterlist.h" #include "i915_vgpu.h" @@ -86,7 +87,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *i915) * beyond the end of the batch buffer, across the page boundary, * and beyond the end of the GTT if we do not provide a guard. */ - ret = ggtt_init_hw(&i915->ggtt); + ret = ggtt_init_hw(to_gt(i915)->ggtt); if (ret) return ret; @@ -129,22 +130,51 @@ void i915_ggtt_suspend_vm(struct i915_address_space *vm) drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); +retry: + i915_gem_drain_freed_objects(vm->i915); + mutex_lock(&vm->mutex); /* Skip rewriting PTE on VMA unbind. */ open = atomic_xchg(&vm->open, 0); list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { + struct drm_i915_gem_object *obj = vma->obj; + GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); - i915_vma_wait_for_bind(vma); - if (i915_vma_is_pinned(vma)) + if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) continue; + /* unlikely to race when GPU is idle, so no worry about slowpath.. */ + if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) { + /* + * No dead objects should appear here, GPU should be + * completely idle, and userspace suspended + */ + i915_gem_object_get(obj); + + atomic_set(&vm->open, open); + mutex_unlock(&vm->mutex); + + i915_gem_object_lock(obj, NULL); + open = i915_vma_unbind(vma); + i915_gem_object_unlock(obj); + + GEM_WARN_ON(open); + + i915_gem_object_put(obj); + goto retry; + } + if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) { - __i915_vma_evict(vma); + i915_vma_wait_for_bind(vma); + + __i915_vma_evict(vma, false); drm_mm_remove_node(&vma->node); } + + i915_gem_object_unlock(obj); } vm->clear_range(vm, 0, vm->total); @@ -235,7 +265,7 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm, } static void gen8_ggtt_insert_entries(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level level, u32 flags) { @@ -252,10 +282,10 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, */ gte = (gen8_pte_t __iomem *)ggtt->gsm; - gte += vma->node.start / I915_GTT_PAGE_SIZE; - end = gte + vma->node.size / I915_GTT_PAGE_SIZE; + gte += vma_res->start / I915_GTT_PAGE_SIZE; + end = gte + vma_res->node_size / I915_GTT_PAGE_SIZE; - for_each_sgt_daddr(addr, iter, vma->pages) + for_each_sgt_daddr(addr, iter, vma_res->bi.pages) gen8_set_pte(gte++, pte_encode | addr); GEM_BUG_ON(gte > end); @@ -292,7 +322,7 @@ static void gen6_ggtt_insert_page(struct i915_address_space *vm, * through the GMADR mapped BAR (i915->mm.gtt->gtt). */ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level level, u32 flags) { @@ -303,10 +333,10 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, dma_addr_t addr; gte = (gen6_pte_t __iomem *)ggtt->gsm; - gte += vma->node.start / I915_GTT_PAGE_SIZE; - end = gte + vma->node.size / I915_GTT_PAGE_SIZE; + gte += vma_res->start / I915_GTT_PAGE_SIZE; + end = gte + vma_res->node_size / I915_GTT_PAGE_SIZE; - for_each_sgt_daddr(addr, iter, vma->pages) + for_each_sgt_daddr(addr, iter, vma_res->bi.pages) iowrite32(vm->pte_encode(addr, level, flags), gte++); GEM_BUG_ON(gte > end); @@ -389,7 +419,7 @@ static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, struct insert_entries { struct i915_address_space *vm; - struct i915_vma *vma; + struct i915_vma_resource *vma_res; enum i915_cache_level level; u32 flags; }; @@ -398,18 +428,18 @@ static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) { struct insert_entries *arg = _arg; - gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags); + gen8_ggtt_insert_entries(arg->vm, arg->vma_res, arg->level, arg->flags); bxt_vtd_ggtt_wa(arg->vm); return 0; } static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level level, u32 flags) { - struct insert_entries arg = { vm, vma, level, flags }; + struct insert_entries arg = { vm, vma_res, level, flags }; stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); } @@ -448,14 +478,14 @@ static void i915_ggtt_insert_page(struct i915_address_space *vm, } static void i915_ggtt_insert_entries(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 unused) { unsigned int flags = (cache_level == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; - intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT, + intel_gtt_insert_sg_entries(vma_res->bi.pages, vma_res->start >> PAGE_SHIFT, flags); } @@ -467,30 +497,32 @@ static void i915_ggtt_clear_range(struct i915_address_space *vm, static void ggtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags) { - struct drm_i915_gem_object *obj = vma->obj; u32 pte_flags; - if (i915_vma_is_bound(vma, ~flags & I915_VMA_BIND_MASK)) + if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK)) return; + vma_res->bound_flags |= flags; + /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ pte_flags = 0; - if (i915_gem_object_is_readonly(obj)) + if (vma_res->bi.readonly) pte_flags |= PTE_READ_ONLY; - if (i915_gem_object_is_lmem(obj)) + if (vma_res->bi.lmem) pte_flags |= PTE_LM; - vm->insert_entries(vm, vma, cache_level, pte_flags); - vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; + vm->insert_entries(vm, vma_res, cache_level, pte_flags); + vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; } -static void ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) +static void ggtt_unbind_vma(struct i915_address_space *vm, + struct i915_vma_resource *vma_res) { - vm->clear_range(vm, vma->node.start, vma->size); + vm->clear_range(vm, vma_res->start, vma_res->vma_size); } static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) @@ -504,7 +536,7 @@ static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP); size = ggtt->vm.total - GUC_GGTT_TOP; - ret = i915_gem_gtt_reserve(&ggtt->vm, &ggtt->uc_fw, size, + ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw, size, GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE, PIN_NOEVICT); if (ret) @@ -623,7 +655,7 @@ err: static void aliasing_gtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags) { @@ -631,25 +663,27 @@ static void aliasing_gtt_bind_vma(struct i915_address_space *vm, /* Currently applicable only to VLV */ pte_flags = 0; - if (i915_gem_object_is_readonly(vma->obj)) + if (vma_res->bi.readonly) pte_flags |= PTE_READ_ONLY; if (flags & I915_VMA_LOCAL_BIND) ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, - stash, vma, cache_level, flags); + stash, vma_res, cache_level, flags); if (flags & I915_VMA_GLOBAL_BIND) - vm->insert_entries(vm, vma, cache_level, pte_flags); + vm->insert_entries(vm, vma_res, cache_level, pte_flags); + + vma_res->bound_flags |= flags; } static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, - struct i915_vma *vma) + struct i915_vma_resource *vma_res) { - if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) - vm->clear_range(vm, vma->node.start, vma->size); + if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND) + vm->clear_range(vm, vma_res->start, vma_res->vma_size); - if (i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) - ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma); + if (vma_res->bound_flags & I915_VMA_LOCAL_BIND) + ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res); } static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) @@ -722,14 +756,14 @@ int i915_init_ggtt(struct drm_i915_private *i915) { int ret; - ret = init_ggtt(&i915->ggtt); + ret = init_ggtt(to_gt(i915)->ggtt); if (ret) return ret; if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { - ret = init_aliasing_ppgtt(&i915->ggtt); + ret = init_aliasing_ppgtt(to_gt(i915)->ggtt); if (ret) - cleanup_init_ggtt(&i915->ggtt); + cleanup_init_ggtt(to_gt(i915)->ggtt); } return 0; @@ -742,11 +776,21 @@ static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) atomic_set(&ggtt->vm.open, 0); flush_workqueue(ggtt->vm.i915->wq); + i915_gem_drain_freed_objects(ggtt->vm.i915); mutex_lock(&ggtt->vm.mutex); - list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) + list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { + struct drm_i915_gem_object *obj = vma->obj; + bool trylock; + + trylock = i915_gem_object_trylock(obj, NULL); + WARN_ON(!trylock); + WARN_ON(__i915_vma_unbind(vma)); + if (trylock) + i915_gem_object_unlock(obj); + } if (drm_mm_node_allocated(&ggtt->error_capture)) drm_mm_remove_node(&ggtt->error_capture); @@ -772,7 +816,7 @@ static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) */ void i915_ggtt_driver_release(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; fini_aliasing_ppgtt(ggtt); @@ -787,7 +831,7 @@ void i915_ggtt_driver_release(struct drm_i915_private *i915) */ void i915_ggtt_driver_late_release(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1); dma_resv_fini(&ggtt->vm._resv); @@ -1208,7 +1252,7 @@ int i915_ggtt_probe_hw(struct drm_i915_private *i915) { int ret; - ret = ggtt_probe_hw(&i915->ggtt, to_gt(i915)); + ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915)); if (ret) return ret; @@ -1280,7 +1324,7 @@ bool i915_ggtt_resume_vm(struct i915_address_space *vm) atomic_read(&vma->flags) & I915_VMA_BIND_MASK; GEM_BUG_ON(!was_bound); - vma->ops->bind_vma(vm, NULL, vma, + vma->ops->bind_vma(vm, NULL, vma->resource, obj ? obj->cache_level : 0, was_bound); if (obj) { /* only used during resume => exclusive access */ diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index f8948de72036..76880fb8fc19 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -4,9 +4,12 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "i915_scatterlist.h" #include "i915_pvinfo.h" #include "i915_vgpu.h" +#include "intel_gt_regs.h" +#include "intel_mchbar_regs.h" /** * DOC: fence register handling @@ -425,7 +428,6 @@ int i915_vma_pin_fence(struct i915_vma *vma) * must keep the device awake whilst using the fence. */ assert_rpm_wakelock_held(vma->vm->gt->uncore->rpm); - GEM_BUG_ON(!i915_vma_is_pinned(vma)); GEM_BUG_ON(!i915_vma_is_ggtt(vma)); err = mutex_lock_interruptible(&vma->vm->mutex); @@ -728,8 +730,8 @@ static void detect_bit_6_swizzle(struct i915_ggtt *ggtt) swizzle_y = I915_BIT_6_SWIZZLE_NONE; } - i915->ggtt.bit_6_swizzle_x = swizzle_x; - i915->ggtt.bit_6_swizzle_y = swizzle_y; + to_gt(i915)->ggtt->bit_6_swizzle_x = swizzle_x; + to_gt(i915)->ggtt->bit_6_swizzle_y = swizzle_y; } /* @@ -896,7 +898,7 @@ void intel_gt_init_swizzling(struct intel_gt *gt) struct intel_uncore *uncore = gt->uncore; if (GRAPHICS_VER(i915) < 5 || - i915->ggtt.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) + to_gt(i915)->ggtt->bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) return; intel_uncore_rmw(uncore, DISP_ARB_CTL, 0, DISP_TILE_SURFACE_SWIZZLING); diff --git a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h index f8253012d166..d112ffd56418 100644 --- a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h +++ b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h @@ -228,11 +228,14 @@ #define PIPE_CONTROL_COMMAND_CACHE_INVALIDATE (1<<29) /* gen11+ */ #define PIPE_CONTROL_TILE_CACHE_FLUSH (1<<28) /* gen11+ */ #define PIPE_CONTROL_FLUSH_L3 (1<<27) +#define PIPE_CONTROL_AMFS_FLUSH (1<<25) /* gen12+ */ #define PIPE_CONTROL_GLOBAL_GTT_IVB (1<<24) /* gen7+ */ #define PIPE_CONTROL_MMIO_WRITE (1<<23) #define PIPE_CONTROL_STORE_DATA_INDEX (1<<21) #define PIPE_CONTROL_CS_STALL (1<<20) +#define PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET (1<<19) #define PIPE_CONTROL_TLB_INVALIDATE (1<<18) +#define PIPE_CONTROL_PSD_SYNC (1<<17) /* gen11+ */ #define PIPE_CONTROL_MEDIA_STATE_CLEAR (1<<16) #define PIPE_CONTROL_WRITE_TIMESTAMP (3<<14) #define PIPE_CONTROL_QW_WRITE (1<<14) @@ -254,6 +257,18 @@ #define PIPE_CONTROL_DEPTH_CACHE_FLUSH (1<<0) #define PIPE_CONTROL_GLOBAL_GTT (1<<2) /* in addr dword */ +/* 3D-related flags can't be set on compute engine */ +#define PIPE_CONTROL_3D_FLAGS (\ + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ + PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ + PIPE_CONTROL_TILE_CACHE_FLUSH | \ + PIPE_CONTROL_DEPTH_STALL | \ + PIPE_CONTROL_STALL_AT_SCOREBOARD | \ + PIPE_CONTROL_PSD_SYNC | \ + PIPE_CONTROL_AMFS_FLUSH | \ + PIPE_CONTROL_VF_CACHE_INVALIDATE | \ + PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) + #define MI_MATH(x) MI_INSTR(0x1a, (x) - 1) #define MI_MATH_INSTR(opcode, op1, op2) ((opcode) << 20 | (op1) << 10 | (op2)) /* Opcodes for MI_MATH_INSTR */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index 35d0fcd3a86c..8a2483ccbfb9 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -3,17 +3,22 @@ * Copyright © 2019 Intel Corporation */ +#include <drm/drm_managed.h> #include <drm/intel-gtt.h> -#include "intel_gt_debugfs.h" - +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_lmem.h" +#include "pxp/intel_pxp.h" + #include "i915_drv.h" #include "intel_context.h" +#include "intel_engine_regs.h" #include "intel_gt.h" #include "intel_gt_buffer_pool.h" #include "intel_gt_clock_utils.h" +#include "intel_gt_debugfs.h" #include "intel_gt_pm.h" +#include "intel_gt_regs.h" #include "intel_gt_requests.h" #include "intel_migrate.h" #include "intel_mocs.h" @@ -23,7 +28,6 @@ #include "intel_rps.h" #include "intel_uncore.h" #include "shmem_utils.h" -#include "pxp/intel_pxp.h" void __intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915) { @@ -61,8 +65,6 @@ int intel_gt_probe_lmem(struct intel_gt *gt) int err; mem = intel_gt_setup_lmem(gt); - if (mem == ERR_PTR(-ENODEV)) - mem = intel_gt_setup_fake_lmem(gt); if (IS_ERR(mem)) { err = PTR_ERR(mem); if (err == -ENODEV) @@ -87,9 +89,11 @@ int intel_gt_probe_lmem(struct intel_gt *gt) return 0; } -void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt) +int intel_gt_assign_ggtt(struct intel_gt *gt) { - gt->ggtt = ggtt; + gt->ggtt = drmm_kzalloc(>->i915->drm, sizeof(*gt->ggtt), GFP_KERNEL); + + return gt->ggtt ? 0 : -ENOMEM; } static const struct intel_mmio_range icl_l3bank_steering_table[] = { @@ -207,7 +211,7 @@ int intel_gt_init_hw(struct intel_gt *gt) if (IS_HASWELL(i915)) intel_uncore_write(uncore, - MI_PREDICATE_RESULT_2, + HSW_MI_PREDICATE_RESULT_2, IS_HSW_GT3(i915) ? LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); @@ -452,7 +456,9 @@ static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) struct i915_vma *vma; int ret; - obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE); + obj = i915_gem_object_create_lmem(i915, size, + I915_BO_ALLOC_VOLATILE | + I915_BO_ALLOC_GPU_ONLY); if (IS_ERR(obj)) obj = i915_gem_object_create_stolen(i915, size); if (IS_ERR(obj)) @@ -907,6 +913,25 @@ u32 intel_gt_read_register_fw(struct intel_gt *gt, i915_reg_t reg) return intel_uncore_read_fw(gt->uncore, reg); } +u32 intel_gt_read_register(struct intel_gt *gt, i915_reg_t reg) +{ + int type; + u8 sliceid, subsliceid; + + for (type = 0; type < NUM_STEERING_TYPES; type++) { + if (intel_gt_reg_needs_read_steering(gt, reg, type)) { + intel_gt_get_valid_steering(gt, type, &sliceid, + &subsliceid); + return intel_uncore_read_with_mcr_steering(gt->uncore, + reg, + sliceid, + subsliceid); + } + } + + return intel_uncore_read(gt->uncore, reg); +} + void intel_gt_info_print(const struct intel_gt_info *info, struct drm_printer *p) { diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h index a913fb6ffec3..0f571c8ee22b 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.h +++ b/drivers/gpu/drm/i915/gt/intel_gt.h @@ -36,7 +36,7 @@ static inline struct intel_gt *huc_to_gt(struct intel_huc *huc) void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915); void __intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915); -void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt); +int intel_gt_assign_ggtt(struct intel_gt *gt); int intel_gt_probe_lmem(struct intel_gt *gt); int intel_gt_init_mmio(struct intel_gt *gt); int __must_check intel_gt_init_hw(struct intel_gt *gt); @@ -85,6 +85,7 @@ static inline bool intel_gt_needs_read_steering(struct intel_gt *gt, } u32 intel_gt_read_register_fw(struct intel_gt *gt, i915_reg_t reg); +u32 intel_gt_read_register(struct intel_gt *gt, i915_reg_t reg); void intel_gt_info_print(const struct intel_gt_info *info, struct drm_printer *p); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c index 9db3dcbd917f..cadfd85785b1 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c @@ -3,6 +3,7 @@ * Copyright © 2014-2018 Intel Corporation */ +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_object.h" #include "i915_drv.h" diff --git a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c index 3513d6f90747..0db822c3b7e5 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c @@ -4,8 +4,10 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "intel_gt.h" #include "intel_gt_clock_utils.h" +#include "intel_gt_regs.h" static u32 read_reference_ts_freq(struct intel_uncore *uncore) { diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index 699a74582d32..e443ac4c8059 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -10,7 +10,7 @@ #include "intel_breadcrumbs.h" #include "intel_gt.h" #include "intel_gt_irq.h" -#include "intel_lrc_reg.h" +#include "intel_gt_regs.h" #include "intel_uncore.h" #include "intel_rps.h" #include "pxp/intel_pxp_irq.h" @@ -100,7 +100,7 @@ gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity) if (unlikely(!intr)) return; - if (class <= COPY_ENGINE_CLASS) + if (class <= COPY_ENGINE_CLASS || class == COMPUTE_CLASS) return gen11_engine_irq_handler(gt, class, instance, intr); if (class == OTHER_CLASS) @@ -182,6 +182,8 @@ void gen11_gt_irq_reset(struct intel_gt *gt) /* Disable RCS, BCS, VCS and VECS class engines. */ intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0); intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0); + if (CCS_MASK(gt)) + intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, 0); /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0); @@ -195,6 +197,10 @@ void gen11_gt_irq_reset(struct intel_gt *gt) intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0); if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0); + if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1)) + intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~0); + if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3)) + intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~0); intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0); @@ -225,6 +231,8 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) /* Enable RCS, BCS, VCS and VECS class interrupts. */ intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask); intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask); + if (CCS_MASK(gt)) + intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, smask); /* Unmask irqs on RCS, BCS, VCS and VECS engines. */ intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask); @@ -238,6 +246,11 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask); if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask); + if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1)) + intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~dmask); + if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3)) + intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~dmask); + /* * RPS interrupts will get enabled/disabled on demand when RPS itself * is enabled/disabled. diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c index 404dfa7673c6..37765919fe32 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c @@ -7,12 +7,15 @@ #include <linux/seq_file.h> #include "i915_drv.h" +#include "i915_reg.h" #include "intel_gt.h" #include "intel_gt_clock_utils.h" #include "intel_gt_debugfs.h" #include "intel_gt_pm.h" #include "intel_gt_pm_debugfs.h" +#include "intel_gt_regs.h" #include "intel_llc.h" +#include "intel_mchbar_regs.h" #include "intel_pcode.h" #include "intel_rc6.h" #include "intel_rps.h" @@ -134,8 +137,7 @@ static int gen6_drpc(struct seq_file *m) } if (GRAPHICS_VER(i915) <= 7) - sandybridge_pcode_read(i915, GEN6_PCODE_READ_RC6VIDS, - &rc6vids, NULL); + snb_pcode_read(i915, GEN6_PCODE_READ_RC6VIDS, &rc6vids, NULL); seq_printf(m, "RC1e Enabled: %s\n", yesno(rcctl1 & GEN6_RC_CTL_RC1e_ENABLE)); @@ -557,9 +559,8 @@ static int llc_show(struct seq_file *m, void *data) wakeref = intel_runtime_pm_get(gt->uncore->rpm); for (gpu_freq = min_gpu_freq; gpu_freq <= max_gpu_freq; gpu_freq++) { ia_freq = gpu_freq; - sandybridge_pcode_read(i915, - GEN6_PCODE_READ_MIN_FREQ_TABLE, - &ia_freq, NULL); + snb_pcode_read(i915, GEN6_PCODE_READ_MIN_FREQ_TABLE, + &ia_freq, NULL); seq_printf(m, "%d\t\t%d\t\t\t\t%d\n", intel_gpu_freq(rps, (gpu_freq * diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c index fe51f894b073..11060f5a4c89 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c @@ -8,6 +8,7 @@ #include "intel_gt.h" #include "intel_gt_irq.h" #include "intel_gt_pm_irq.h" +#include "intel_gt_regs.h" static void write_pm_imr(struct intel_gt *gt) { diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h new file mode 100644 index 000000000000..19cd34f24263 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h @@ -0,0 +1,1506 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_GT_REGS__ +#define __INTEL_GT_REGS__ + +#include "i915_reg_defs.h" + +/* RPM unit config (Gen8+) */ +#define RPM_CONFIG0 _MMIO(0xd00) +#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT 3 +#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK (1 << GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT) +#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ 0 +#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ 1 +#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT 3 +#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK (0x7 << GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT) +#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ 0 +#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ 1 +#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ 2 +#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ 3 +#define GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT 1 +#define GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK (0x3 << GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT) + +#define RPM_CONFIG1 _MMIO(0xd04) +#define GEN10_GT_NOA_ENABLE (1 << 9) + +/* RCP unit config (Gen8+) */ +#define RCP_CONFIG _MMIO(0xd08) + +#define RC6_LOCATION _MMIO(0xd40) +#define RC6_CTX_IN_DRAM (1 << 0) +#define RC6_CTX_BASE _MMIO(0xd48) +#define RC6_CTX_BASE_MASK 0xFFFFFFF0 + +#define FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(n) _MMIO(0xd50 + (n) * 4) +#define FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(n) _MMIO(0xd70 + (n) * 4) +#define FORCEWAKE_ACK_RENDER_GEN9 _MMIO(0xd84) +#define FORCEWAKE_ACK_MEDIA_GEN9 _MMIO(0xd88) + +#define MCFG_MCR_SELECTOR _MMIO(0xfd0) +#define SF_MCR_SELECTOR _MMIO(0xfd8) +#define GEN8_MCR_SELECTOR _MMIO(0xfdc) +#define GEN8_MCR_SLICE(slice) (((slice) & 3) << 26) +#define GEN8_MCR_SLICE_MASK GEN8_MCR_SLICE(3) +#define GEN8_MCR_SUBSLICE(subslice) (((subslice) & 3) << 24) +#define GEN8_MCR_SUBSLICE_MASK GEN8_MCR_SUBSLICE(3) +#define GEN11_MCR_SLICE(slice) (((slice) & 0xf) << 27) +#define GEN11_MCR_SLICE_MASK GEN11_MCR_SLICE(0xf) +#define GEN11_MCR_SUBSLICE(subslice) (((subslice) & 0x7) << 24) +#define GEN11_MCR_SUBSLICE_MASK GEN11_MCR_SUBSLICE(0x7) + +#define IPEIR_I965 _MMIO(0x2064) +#define IPEHR_I965 _MMIO(0x2068) + +/* + * On GEN4, only the render ring INSTDONE exists and has a different + * layout than the GEN7+ version. + * The GEN2 counterpart of this register is GEN2_INSTDONE. + */ +#define INSTPS _MMIO(0x2070) /* 965+ only */ +#define GEN4_INSTDONE1 _MMIO(0x207c) /* 965+ only, aka INSTDONE_2 on SNB */ +#define ACTHD_I965 _MMIO(0x2074) +#define HWS_PGA _MMIO(0x2080) +#define HWS_ADDRESS_MASK 0xfffff000 +#define HWS_START_ADDRESS_SHIFT 4 + +#define _3D_CHICKEN _MMIO(0x2084) +#define _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB (1 << 10) + +#define PWRCTXA _MMIO(0x2088) /* 965GM+ only */ +#define PWRCTX_EN (1 << 0) + +#define FF_SLICE_CHICKEN _MMIO(0x2088) +#define FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX (1 << 1) + +/* GM45+ chicken bits -- debug workaround bits that may be required + * for various sorts of correct behavior. The top 16 bits of each are + * the enables for writing to the corresponding low bit. + */ +#define _3D_CHICKEN2 _MMIO(0x208c) +/* Disables pipelining of read flushes past the SF-WIZ interface. + * Required on all Ironlake steppings according to the B-Spec, but the + * particular danger of not doing so is not specified. + */ +#define _3D_CHICKEN2_WM_READ_PIPELINED (1 << 14) + +#define _3D_CHICKEN3 _MMIO(0x2090) +#define _3D_CHICKEN_SF_PROVOKING_VERTEX_FIX (1 << 12) +#define _3D_CHICKEN_SF_DISABLE_OBJEND_CULL (1 << 10) +#define _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE (1 << 5) +#define _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL (1 << 5) +#define _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x) ((x) << 1) /* gen8+ */ +#define _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH (1 << 1) /* gen6 */ + +#define GEN2_INSTDONE _MMIO(0x2090) +#define NOPID _MMIO(0x2094) +#define HWSTAM _MMIO(0x2098) + +#define WAIT_FOR_RC6_EXIT _MMIO(0x20cc) +/* HSW only */ +#define HSW_SELECTIVE_READ_ADDRESSING_SHIFT 2 +#define HSW_SELECTIVE_READ_ADDRESSING_MASK (0x3 << HSW_SLECTIVE_READ_ADDRESSING_SHIFT) +#define HSW_SELECTIVE_WRITE_ADDRESS_SHIFT 4 +#define HSW_SELECTIVE_WRITE_ADDRESS_MASK (0x7 << HSW_SELECTIVE_WRITE_ADDRESS_SHIFT) +/* HSW+ */ +#define HSW_WAIT_FOR_RC6_EXIT_ENABLE (1 << 0) +#define HSW_RCS_CONTEXT_ENABLE (1 << 7) +#define HSW_RCS_INHIBIT (1 << 8) +/* Gen8 */ +#define GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT 4 +#define GEN8_SELECTIVE_WRITE_ADDRESS_MASK (0x3 << GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT) +#define GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT 4 +#define GEN8_SELECTIVE_WRITE_ADDRESS_MASK (0x3 << GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT) +#define GEN8_SELECTIVE_WRITE_ADDRESSING_ENABLE (1 << 6) +#define GEN8_SELECTIVE_READ_SUBSLICE_SELECT_SHIFT 9 +#define GEN8_SELECTIVE_READ_SUBSLICE_SELECT_MASK (0x3 << GEN8_SELECTIVE_READ_SUBSLICE_SELECT_SHIFT) +#define GEN8_SELECTIVE_READ_SLICE_SELECT_SHIFT 11 +#define GEN8_SELECTIVE_READ_SLICE_SELECT_MASK (0x3 << GEN8_SELECTIVE_READ_SLICE_SELECT_SHIFT) +#define GEN8_SELECTIVE_READ_ADDRESSING_ENABLE (1 << 13) + +#define GEN6_GT_MODE _MMIO(0x20d0) +#define GEN6_WIZ_HASHING(hi, lo) (((hi) << 9) | ((lo) << 7)) +#define GEN6_WIZ_HASHING_8x8 GEN6_WIZ_HASHING(0, 0) +#define GEN6_WIZ_HASHING_8x4 GEN6_WIZ_HASHING(0, 1) +#define GEN6_WIZ_HASHING_16x4 GEN6_WIZ_HASHING(1, 0) +#define GEN6_WIZ_HASHING_MASK GEN6_WIZ_HASHING(1, 1) +#define GEN6_TD_FOUR_ROW_DISPATCH_DISABLE (1 << 5) + +/* chicken reg for WaConextSwitchWithConcurrentTLBInvalidate */ +#define GEN9_CSFE_CHICKEN1_RCS _MMIO(0x20d4) +#define GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE (1 << 2) +#define GEN11_ENABLE_32_PLANE_MODE (1 << 7) + +#define GEN7_FF_SLICE_CS_CHICKEN1 _MMIO(0x20e0) +#define GEN9_FFSC_PERCTX_PREEMPT_CTRL (1 << 14) + +#define FF_SLICE_CS_CHICKEN2 _MMIO(0x20e4) +#define GEN9_TSG_BARRIER_ACK_DISABLE (1 << 8) +#define GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE (1 << 10) + +#define GEN9_CS_DEBUG_MODE1 _MMIO(0x20ec) +#define FF_DOP_CLOCK_GATE_DISABLE REG_BIT(1) +#define GEN12_CS_DEBUG_MODE1_CCCSUNIT_BE_COMMON _MMIO(0x20ec) +#define GEN12_REPLAY_MODE_GRANULARITY REG_BIT(0) + +/* WaClearTdlStateAckDirtyBits */ +#define GEN8_STATE_ACK _MMIO(0x20f0) +#define GEN9_STATE_ACK_SLICE1 _MMIO(0x20f8) +#define GEN9_STATE_ACK_SLICE2 _MMIO(0x2100) +#define GEN9_STATE_ACK_TDL0 (1 << 12) +#define GEN9_STATE_ACK_TDL1 (1 << 13) +#define GEN9_STATE_ACK_TDL2 (1 << 14) +#define GEN9_STATE_ACK_TDL3 (1 << 15) +#define GEN9_SUBSLICE_TDL_ACK_BITS \ + (GEN9_STATE_ACK_TDL3 | GEN9_STATE_ACK_TDL2 | \ + GEN9_STATE_ACK_TDL1 | GEN9_STATE_ACK_TDL0) + +#define CACHE_MODE_0 _MMIO(0x2120) /* 915+ only */ +#define CM0_PIPELINED_RENDER_FLUSH_DISABLE (1 << 8) +#define CM0_IZ_OPT_DISABLE (1 << 6) +#define CM0_ZR_OPT_DISABLE (1 << 5) +#define CM0_STC_EVICT_DISABLE_LRA_SNB (1 << 5) +#define CM0_DEPTH_EVICT_DISABLE (1 << 4) +#define CM0_COLOR_EVICT_DISABLE (1 << 3) +#define CM0_DEPTH_WRITE_DISABLE (1 << 1) +#define CM0_RC_OP_FLUSH_DISABLE (1 << 0) + +#define GFX_FLSH_CNTL _MMIO(0x2170) /* 915+ only */ + +/* + * Logical Context regs + */ +/* + * Notes on SNB/IVB/VLV context size: + * - Power context is saved elsewhere (LLC or stolen) + * - Ring/execlist context is saved on SNB, not on IVB + * - Extended context size already includes render context size + * - We always need to follow the extended context size. + * SNB BSpec has comments indicating that we should use the + * render context size instead if execlists are disabled, but + * based on empirical testing that's just nonsense. + * - Pipelined/VF state is saved on SNB/IVB respectively + * - GT1 size just indicates how much of render context + * doesn't need saving on GT1 + */ +#define CXT_SIZE _MMIO(0x21a0) +#define GEN6_CXT_POWER_SIZE(cxt_reg) (((cxt_reg) >> 24) & 0x3f) +#define GEN6_CXT_RING_SIZE(cxt_reg) (((cxt_reg) >> 18) & 0x3f) +#define GEN6_CXT_RENDER_SIZE(cxt_reg) (((cxt_reg) >> 12) & 0x3f) +#define GEN6_CXT_EXTENDED_SIZE(cxt_reg) (((cxt_reg) >> 6) & 0x3f) +#define GEN6_CXT_PIPELINE_SIZE(cxt_reg) (((cxt_reg) >> 0) & 0x3f) +#define GEN6_CXT_TOTAL_SIZE(cxt_reg) (GEN6_CXT_RING_SIZE(cxt_reg) + \ + GEN6_CXT_EXTENDED_SIZE(cxt_reg) + \ + GEN6_CXT_PIPELINE_SIZE(cxt_reg)) +#define GEN7_CXT_SIZE _MMIO(0x21a8) +#define GEN7_CXT_POWER_SIZE(ctx_reg) (((ctx_reg) >> 25) & 0x7f) +#define GEN7_CXT_RING_SIZE(ctx_reg) (((ctx_reg) >> 22) & 0x7) +#define GEN7_CXT_RENDER_SIZE(ctx_reg) (((ctx_reg) >> 16) & 0x3f) +#define GEN7_CXT_EXTENDED_SIZE(ctx_reg) (((ctx_reg) >> 9) & 0x7f) +#define GEN7_CXT_GT1_SIZE(ctx_reg) (((ctx_reg) >> 6) & 0x7) +#define GEN7_CXT_VFSTATE_SIZE(ctx_reg) (((ctx_reg) >> 0) & 0x3f) +#define GEN7_CXT_TOTAL_SIZE(ctx_reg) (GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \ + GEN7_CXT_VFSTATE_SIZE(ctx_reg)) + +#define HSW_MI_PREDICATE_RESULT_2 _MMIO(0x2214) + +#define GEN9_CTX_PREEMPT_REG _MMIO(0x2248) +#define GEN12_DISABLE_POSH_BUSY_FF_DOP_CG REG_BIT(11) + +#define GPGPU_THREADS_DISPATCHED _MMIO(0x2290) +#define GPGPU_THREADS_DISPATCHED_UDW _MMIO(0x2290 + 4) + +#define GEN9_RCS_FE_FSM2 _MMIO(0x22a4) +#define GEN6_RCS_PWR_FSM _MMIO(0x22ac) + +#define HS_INVOCATION_COUNT _MMIO(0x2300) +#define HS_INVOCATION_COUNT_UDW _MMIO(0x2300 + 4) +#define DS_INVOCATION_COUNT _MMIO(0x2308) +#define DS_INVOCATION_COUNT_UDW _MMIO(0x2308 + 4) +#define IA_VERTICES_COUNT _MMIO(0x2310) +#define IA_VERTICES_COUNT_UDW _MMIO(0x2310 + 4) +#define IA_PRIMITIVES_COUNT _MMIO(0x2318) +#define IA_PRIMITIVES_COUNT_UDW _MMIO(0x2318 + 4) +#define VS_INVOCATION_COUNT _MMIO(0x2320) +#define VS_INVOCATION_COUNT_UDW _MMIO(0x2320 + 4) +#define GS_INVOCATION_COUNT _MMIO(0x2328) +#define GS_INVOCATION_COUNT_UDW _MMIO(0x2328 + 4) +#define GS_PRIMITIVES_COUNT _MMIO(0x2330) +#define GS_PRIMITIVES_COUNT_UDW _MMIO(0x2330 + 4) +#define CL_INVOCATION_COUNT _MMIO(0x2338) +#define CL_INVOCATION_COUNT_UDW _MMIO(0x2338 + 4) +#define CL_PRIMITIVES_COUNT _MMIO(0x2340) +#define CL_PRIMITIVES_COUNT_UDW _MMIO(0x2340 + 4) +#define PS_INVOCATION_COUNT _MMIO(0x2348) +#define PS_INVOCATION_COUNT_UDW _MMIO(0x2348 + 4) +#define PS_DEPTH_COUNT _MMIO(0x2350) +#define PS_DEPTH_COUNT_UDW _MMIO(0x2350 + 4) +#define GEN7_3DPRIM_END_OFFSET _MMIO(0x2420) +#define GEN7_3DPRIM_START_VERTEX _MMIO(0x2430) +#define GEN7_3DPRIM_VERTEX_COUNT _MMIO(0x2434) +#define GEN7_3DPRIM_INSTANCE_COUNT _MMIO(0x2438) +#define GEN7_3DPRIM_START_INSTANCE _MMIO(0x243c) +#define GEN7_3DPRIM_BASE_VERTEX _MMIO(0x2440) +#define GEN7_GPGPU_DISPATCHDIMX _MMIO(0x2500) +#define GEN7_GPGPU_DISPATCHDIMY _MMIO(0x2504) +#define GEN7_GPGPU_DISPATCHDIMZ _MMIO(0x2508) + +#define GFX_MODE _MMIO(0x2520) + +#define GEN8_CS_CHICKEN1 _MMIO(0x2580) +#define GEN9_PREEMPT_3D_OBJECT_LEVEL (1 << 0) +#define GEN9_PREEMPT_GPGPU_LEVEL(hi, lo) (((hi) << 2) | ((lo) << 1)) +#define GEN9_PREEMPT_GPGPU_MID_THREAD_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(0, 0) +#define GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(0, 1) +#define GEN9_PREEMPT_GPGPU_COMMAND_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(1, 0) +#define GEN9_PREEMPT_GPGPU_LEVEL_MASK GEN9_PREEMPT_GPGPU_LEVEL(1, 1) + +#define GEN12_GLOBAL_MOCS(i) _MMIO(0x4000 + (i) * 4) /* Global MOCS regs */ + +#define RENDER_HWS_PGA_GEN7 _MMIO(0x4080) + +#define GEN8_GAMW_ECO_DEV_RW_IA _MMIO(0x4080) +#define GAMW_ECO_ENABLE_64K_IPS_FIELD 0xF +#define GAMW_ECO_DEV_CTX_RELOAD_DISABLE (1 << 7) + +#define GAM_ECOCHK _MMIO(0x4090) +#define BDW_DISABLE_HDC_INVALIDATION (1 << 25) +#define ECOCHK_SNB_BIT (1 << 10) +#define ECOCHK_DIS_TLB (1 << 8) +#define HSW_ECOCHK_ARB_PRIO_SOL (1 << 6) +#define ECOCHK_PPGTT_CACHE64B (0x3 << 3) +#define ECOCHK_PPGTT_CACHE4B (0x0 << 3) +#define ECOCHK_PPGTT_GFDT_IVB (0x1 << 4) +#define ECOCHK_PPGTT_LLC_IVB (0x1 << 3) +#define ECOCHK_PPGTT_UC_HSW (0x1 << 3) +#define ECOCHK_PPGTT_WT_HSW (0x2 << 3) +#define ECOCHK_PPGTT_WB_HSW (0x3 << 3) + +#define GEN8_RING_FAULT_REG _MMIO(0x4094) +#define _RING_FAULT_REG_RCS 0x4094 +#define _RING_FAULT_REG_VCS 0x4194 +#define _RING_FAULT_REG_BCS 0x4294 +#define _RING_FAULT_REG_VECS 0x4394 +#define RING_FAULT_REG(engine) _MMIO(_PICK((engine)->class, \ + _RING_FAULT_REG_RCS, \ + _RING_FAULT_REG_VCS, \ + _RING_FAULT_REG_VECS, \ + _RING_FAULT_REG_BCS)) + +#define ERROR_GEN6 _MMIO(0x40a0) + +#define DONE_REG _MMIO(0x40b0) +#define GEN8_PRIVATE_PAT_LO _MMIO(0x40e0) +#define GEN8_PRIVATE_PAT_HI _MMIO(0x40e0 + 4) +#define GEN10_PAT_INDEX(index) _MMIO(0x40e0 + (index) * 4) +#define BSD_HWS_PGA_GEN7 _MMIO(0x4180) +#define GEN12_GFX_CCS_AUX_NV _MMIO(0x4208) +#define GEN12_VD0_AUX_NV _MMIO(0x4218) +#define GEN12_VD1_AUX_NV _MMIO(0x4228) + +#define GEN8_RTCR _MMIO(0x4260) +#define GEN8_M1TCR _MMIO(0x4264) +#define GEN8_M2TCR _MMIO(0x4268) +#define GEN8_BTCR _MMIO(0x426c) +#define GEN8_VTCR _MMIO(0x4270) + +#define GEN12_VD2_AUX_NV _MMIO(0x4298) +#define GEN12_VD3_AUX_NV _MMIO(0x42a8) +#define GEN12_VE0_AUX_NV _MMIO(0x4238) + +#define BLT_HWS_PGA_GEN7 _MMIO(0x4280) + +#define GEN12_VE1_AUX_NV _MMIO(0x42b8) +#define AUX_INV REG_BIT(0) +#define VEBOX_HWS_PGA_GEN7 _MMIO(0x4380) + +#define GEN12_AUX_ERR_DBG _MMIO(0x43f4) + +#define GEN7_TLB_RD_ADDR _MMIO(0x4700) + +#define GEN12_PAT_INDEX(index) _MMIO(0x4800 + (index) * 4) + +#define XEHPSDV_FLAT_CCS_BASE_ADDR _MMIO(0x4910) +#define XEHPSDV_CCS_BASE_SHIFT 8 + +#define GAMTARBMODE _MMIO(0x4a08) +#define ARB_MODE_BWGTLB_DISABLE (1 << 9) +#define ARB_MODE_SWIZZLE_BDW (1 << 1) + +#define GEN9_GAMT_ECO_REG_RW_IA _MMIO(0x4ab0) +#define GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS (1 << 18) + +#define GAMT_CHKN_BIT_REG _MMIO(0x4ab8) +#define GAMT_CHKN_DISABLE_L3_COH_PIPE (1 << 31) +#define GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING (1 << 28) +#define GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT (1 << 24) + +#define GEN8_FAULT_TLB_DATA0 _MMIO(0x4b10) +#define GEN8_FAULT_TLB_DATA1 _MMIO(0x4b14) + +#define GEN11_GACB_PERF_CTRL _MMIO(0x4b80) +#define GEN11_HASH_CTRL_MASK (0x3 << 12 | 0xf << 0) +#define GEN11_HASH_CTRL_BIT0 (1 << 0) +#define GEN11_HASH_CTRL_BIT4 (1 << 12) + +/* gamt regs */ +#define GEN8_L3_LRA_1_GPGPU _MMIO(0x4dd4) +#define GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW 0x67F1427F /* max/min for LRA1/2 */ +#define GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV 0x5FF101FF /* max/min for LRA1/2 */ +#define GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL 0x67F1427F /* " " */ +#define GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT 0x5FF101FF /* " " */ + +#define MMCD_MISC_CTRL _MMIO(0x4ddc) /* skl+ */ +#define MMCD_PCLA (1 << 31) +#define MMCD_HOTSPOT_EN (1 << 27) + +/* There are the 4 64-bit counter registers, one for each stream output */ +#define GEN7_SO_NUM_PRIMS_WRITTEN(n) _MMIO(0x5200 + (n) * 8) +#define GEN7_SO_NUM_PRIMS_WRITTEN_UDW(n) _MMIO(0x5200 + (n) * 8 + 4) + +#define GEN7_SO_PRIM_STORAGE_NEEDED(n) _MMIO(0x5240 + (n) * 8) +#define GEN7_SO_PRIM_STORAGE_NEEDED_UDW(n) _MMIO(0x5240 + (n) * 8 + 4) + +#define GEN9_WM_CHICKEN3 _MMIO(0x5588) +#define GEN9_FACTOR_IN_CLR_VAL_HIZ (1 << 9) + +#define VFLSKPD _MMIO(0x62a8) +#define DIS_OVER_FETCH_CACHE REG_BIT(1) +#define DIS_MULT_MISS_RD_SQUASH REG_BIT(0) + +#define FF_MODE2 _MMIO(0x6604) +#define FF_MODE2_GS_TIMER_MASK REG_GENMASK(31, 24) +#define FF_MODE2_GS_TIMER_224 REG_FIELD_PREP(FF_MODE2_GS_TIMER_MASK, 224) +#define FF_MODE2_TDS_TIMER_MASK REG_GENMASK(23, 16) +#define FF_MODE2_TDS_TIMER_128 REG_FIELD_PREP(FF_MODE2_TDS_TIMER_MASK, 4) + +#define XEHPG_INSTDONE_GEOM_SVG _MMIO(0x666c) + +#define CACHE_MODE_0_GEN7 _MMIO(0x7000) /* IVB+ */ +#define RC_OP_FLUSH_ENABLE (1 << 0) +#define HIZ_RAW_STALL_OPT_DISABLE (1 << 2) +#define CACHE_MODE_1 _MMIO(0x7004) /* IVB+ */ +#define PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1 << 6) +#define GEN8_4x4_STC_OPTIMIZATION_DISABLE (1 << 6) +#define GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE (1 << 1) + +#define GEN7_GT_MODE _MMIO(0x7008) +#define GEN9_IZ_HASHING_MASK(slice) (0x3 << ((slice) * 2)) +#define GEN9_IZ_HASHING(slice, val) ((val) << ((slice) * 2)) + +/* GEN7 chicken */ +#define GEN7_COMMON_SLICE_CHICKEN1 _MMIO(0x7010) +#define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC (1 << 10) +#define GEN9_RHWO_OPTIMIZATION_DISABLE (1 << 14) + +#define COMMON_SLICE_CHICKEN2 _MMIO(0x7014) +#define GEN9_PBE_COMPRESSED_HASH_SELECTION (1 << 13) +#define GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE (1 << 12) +#define GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION (1 << 8) +#define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE (1 << 0) + +#define HIZ_CHICKEN _MMIO(0x7018) +#define CHV_HZ_8X8_MODE_IN_1X REG_BIT(15) +#define DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE REG_BIT(14) +#define BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE REG_BIT(3) + +#define GEN8_L3CNTLREG _MMIO(0x7034) +#define GEN8_ERRDETBCTRL (1 << 9) + +#define GEN7_SC_INSTDONE _MMIO(0x7100) +#define GEN12_SC_INSTDONE_EXTRA _MMIO(0x7104) +#define GEN12_SC_INSTDONE_EXTRA2 _MMIO(0x7108) + +/* GEN8 chicken */ +#define HDC_CHICKEN0 _MMIO(0x7300) +#define HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE (1 << 15) +#define HDC_FENCE_DEST_SLM_DISABLE (1 << 14) +#define HDC_DONOT_FETCH_MEM_WHEN_MASKED (1 << 11) +#define HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT (1 << 5) +#define HDC_FORCE_NON_COHERENT (1 << 4) +#define HDC_BARRIER_PERFORMANCE_DISABLE (1 << 10) + +#define GEN8_HDC_CHICKEN1 _MMIO(0x7304) + +#define GEN11_COMMON_SLICE_CHICKEN3 _MMIO(0x7304) +#define DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN REG_BIT(12) +#define XEHP_DUAL_SIMD8_SEQ_MERGE_DISABLE REG_BIT(12) +#define GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC REG_BIT(11) +#define GEN12_DISABLE_CPS_AWARE_COLOR_PIPE REG_BIT(9) + +/* GEN9 chicken */ +#define SLICE_ECO_CHICKEN0 _MMIO(0x7308) +#define PIXEL_MASK_CAMMING_DISABLE (1 << 14) + +#define GEN9_SLICE_COMMON_ECO_CHICKEN0 _MMIO(0x7308) +#define DISABLE_PIXEL_MASK_CAMMING (1 << 14) + +#define GEN9_SLICE_COMMON_ECO_CHICKEN1 _MMIO(0x731c) +#define GEN11_STATE_CACHE_REDIRECT_TO_CS (1 << 11) + +#define SLICE_COMMON_ECO_CHICKEN1 _MMIO(0x731c) +#define MSC_MSAA_REODER_BUF_BYPASS_DISABLE REG_BIT(14) + +#define GEN9_SLICE_PGCTL_ACK(slice) _MMIO(0x804c + (slice) * 0x4) +#define GEN10_SLICE_PGCTL_ACK(slice) _MMIO(0x804c + ((slice) / 3) * 0x34 + \ + ((slice) % 3) * 0x4) +#define GEN9_PGCTL_SLICE_ACK (1 << 0) +#define GEN9_PGCTL_SS_ACK(subslice) (1 << (2 + (subslice) * 2)) +#define GEN10_PGCTL_VALID_SS_MASK(slice) ((slice) == 0 ? 0x7F : 0x1F) + +#define GEN9_SS01_EU_PGCTL_ACK(slice) _MMIO(0x805c + (slice) * 0x8) +#define GEN10_SS01_EU_PGCTL_ACK(slice) _MMIO(0x805c + ((slice) / 3) * 0x30 + \ + ((slice) % 3) * 0x8) +#define GEN9_SS23_EU_PGCTL_ACK(slice) _MMIO(0x8060 + (slice) * 0x8) +#define GEN10_SS23_EU_PGCTL_ACK(slice) _MMIO(0x8060 + ((slice) / 3) * 0x30 + \ + ((slice) % 3) * 0x8) +#define GEN9_PGCTL_SSA_EU08_ACK (1 << 0) +#define GEN9_PGCTL_SSA_EU19_ACK (1 << 2) +#define GEN9_PGCTL_SSA_EU210_ACK (1 << 4) +#define GEN9_PGCTL_SSA_EU311_ACK (1 << 6) +#define GEN9_PGCTL_SSB_EU08_ACK (1 << 8) +#define GEN9_PGCTL_SSB_EU19_ACK (1 << 10) +#define GEN9_PGCTL_SSB_EU210_ACK (1 << 12) +#define GEN9_PGCTL_SSB_EU311_ACK (1 << 14) + +#define VF_PREEMPTION _MMIO(0x83a4) +#define PREEMPTION_VERTEX_COUNT REG_GENMASK(15, 0) + +#define GEN8_RC6_CTX_INFO _MMIO(0x8504) + +#define GEN12_SQCM _MMIO(0x8724) +#define EN_32B_ACCESS REG_BIT(30) + +#define HSW_IDICR _MMIO(0x9008) +#define IDIHASHMSK(x) (((x) & 0x3f) << 16) + +#define GEN6_MBCUNIT_SNPCR _MMIO(0x900c) /* for LLC config */ +#define GEN6_MBC_SNPCR_SHIFT 21 +#define GEN6_MBC_SNPCR_MASK (3 << 21) +#define GEN6_MBC_SNPCR_MAX (0 << 21) +#define GEN6_MBC_SNPCR_MED (1 << 21) +#define GEN6_MBC_SNPCR_LOW (2 << 21) +#define GEN6_MBC_SNPCR_MIN (3 << 21) /* only 1/16th of the cache is shared */ + +#define VLV_G3DCTL _MMIO(0x9024) +#define VLV_GSCKGCTL _MMIO(0x9028) + +/* WaCatErrorRejectionIssue */ +#define GEN7_SQ_CHICKEN_MBCUNIT_CONFIG _MMIO(0x9030) +#define GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB (1 << 11) + +#define FBC_LLC_READ_CTRL _MMIO(0x9044) +#define FBC_LLC_FULLY_OPEN REG_BIT(30) + +#define GEN6_MBCTL _MMIO(0x907c) +#define GEN6_MBCTL_ENABLE_BOOT_FETCH (1 << 4) +#define GEN6_MBCTL_CTX_FETCH_NEEDED (1 << 3) +#define GEN6_MBCTL_BME_UPDATE_ENABLE (1 << 2) +#define GEN6_MBCTL_MAE_UPDATE_ENABLE (1 << 1) +#define GEN6_MBCTL_BOOT_FETCH_MECH (1 << 0) + +/* Fuse readout registers for GT */ +#define GEN10_MIRROR_FUSE3 _MMIO(0x9118) +#define GEN10_L3BANK_PAIR_COUNT 4 +#define GEN10_L3BANK_MASK 0x0F +/* on Xe_HP the same fuses indicates mslices instead of L3 banks */ +#define GEN12_MAX_MSLICES 4 +#define GEN12_MEML3_EN_MASK 0x0F + +#define HSW_PAVP_FUSE1 _MMIO(0x911c) +#define XEHP_SFC_ENABLE_MASK REG_GENMASK(27, 24) +#define HSW_F1_EU_DIS_MASK REG_GENMASK(17, 16) +#define HSW_F1_EU_DIS_10EUS 0 +#define HSW_F1_EU_DIS_8EUS 1 +#define HSW_F1_EU_DIS_6EUS 2 + +#define GEN8_FUSE2 _MMIO(0x9120) +#define GEN8_F2_SS_DIS_SHIFT 21 +#define GEN8_F2_SS_DIS_MASK (0x7 << GEN8_F2_SS_DIS_SHIFT) +#define GEN8_F2_S_ENA_SHIFT 25 +#define GEN8_F2_S_ENA_MASK (0x7 << GEN8_F2_S_ENA_SHIFT) +#define GEN9_F2_SS_DIS_SHIFT 20 +#define GEN9_F2_SS_DIS_MASK (0xf << GEN9_F2_SS_DIS_SHIFT) +#define GEN10_F2_S_ENA_SHIFT 22 +#define GEN10_F2_S_ENA_MASK (0x3f << GEN10_F2_S_ENA_SHIFT) +#define GEN10_F2_SS_DIS_SHIFT 18 +#define GEN10_F2_SS_DIS_MASK (0xf << GEN10_F2_SS_DIS_SHIFT) + +#define GEN8_EU_DISABLE0 _MMIO(0x9134) +#define GEN9_EU_DISABLE(slice) _MMIO(0x9134 + (slice) * 0x4) +#define GEN11_EU_DISABLE _MMIO(0x9134) +#define GEN8_EU_DIS0_S0_MASK 0xffffff +#define GEN8_EU_DIS0_S1_SHIFT 24 +#define GEN8_EU_DIS0_S1_MASK (0xff << GEN8_EU_DIS0_S1_SHIFT) +#define GEN11_EU_DIS_MASK 0xFF +#define XEHP_EU_ENABLE _MMIO(0x9134) +#define XEHP_EU_ENA_MASK 0xFF + +#define GEN8_EU_DISABLE1 _MMIO(0x9138) +#define GEN8_EU_DIS1_S1_MASK 0xffff +#define GEN8_EU_DIS1_S2_SHIFT 16 +#define GEN8_EU_DIS1_S2_MASK (0xffff << GEN8_EU_DIS1_S2_SHIFT) + +#define GEN11_GT_SLICE_ENABLE _MMIO(0x9138) +#define GEN11_GT_S_ENA_MASK 0xFF + +#define GEN8_EU_DISABLE2 _MMIO(0x913c) +#define GEN8_EU_DIS2_S2_MASK 0xff + +#define GEN11_GT_SUBSLICE_DISABLE _MMIO(0x913c) +#define GEN12_GT_GEOMETRY_DSS_ENABLE _MMIO(0x913c) + +#define GEN10_EU_DISABLE3 _MMIO(0x9140) +#define GEN10_EU_DIS_SS_MASK 0xff +#define GEN11_GT_VEBOX_VDBOX_DISABLE _MMIO(0x9140) +#define GEN11_GT_VDBOX_DISABLE_MASK 0xff +#define GEN11_GT_VEBOX_DISABLE_SHIFT 16 +#define GEN11_GT_VEBOX_DISABLE_MASK (0x0f << GEN11_GT_VEBOX_DISABLE_SHIFT) + +#define GEN12_GT_COMPUTE_DSS_ENABLE _MMIO(0x9144) + +#define GEN6_UCGCTL1 _MMIO(0x9400) +#define GEN6_GAMUNIT_CLOCK_GATE_DISABLE (1 << 22) +#define GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE (1 << 16) +#define GEN6_BLBUNIT_CLOCK_GATE_DISABLE (1 << 5) +#define GEN6_CSUNIT_CLOCK_GATE_DISABLE (1 << 7) + +#define GEN6_UCGCTL2 _MMIO(0x9404) +#define GEN6_VFUNIT_CLOCK_GATE_DISABLE (1 << 31) +#define GEN7_VDSUNIT_CLOCK_GATE_DISABLE (1 << 30) +#define GEN7_TDLUNIT_CLOCK_GATE_DISABLE (1 << 22) +#define GEN6_RCZUNIT_CLOCK_GATE_DISABLE (1 << 13) +#define GEN6_RCPBUNIT_CLOCK_GATE_DISABLE (1 << 12) +#define GEN6_RCCUNIT_CLOCK_GATE_DISABLE (1 << 11) + +#define GEN6_UCGCTL3 _MMIO(0x9408) +#define GEN6_OACSUNIT_CLOCK_GATE_DISABLE (1 << 20) + +#define GEN7_UCGCTL4 _MMIO(0x940c) +#define GEN7_L3BANK2X_CLOCK_GATE_DISABLE (1 << 25) +#define GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE (1 << 14) + +#define GEN6_RCGCTL1 _MMIO(0x9410) +#define GEN6_RCGCTL2 _MMIO(0x9414) + +#define GEN6_GDRST _MMIO(0x941c) +#define GEN6_GRDOM_FULL (1 << 0) +#define GEN6_GRDOM_RENDER (1 << 1) +#define GEN6_GRDOM_MEDIA (1 << 2) +#define GEN6_GRDOM_BLT (1 << 3) +#define GEN6_GRDOM_VECS (1 << 4) +#define GEN9_GRDOM_GUC (1 << 5) +#define GEN8_GRDOM_MEDIA2 (1 << 7) +/* GEN11 changed all bit defs except for FULL & RENDER */ +#define GEN11_GRDOM_FULL GEN6_GRDOM_FULL +#define GEN11_GRDOM_RENDER GEN6_GRDOM_RENDER +#define GEN11_GRDOM_BLT (1 << 2) +#define GEN11_GRDOM_GUC (1 << 3) +#define GEN11_GRDOM_MEDIA (1 << 5) +#define GEN11_GRDOM_MEDIA2 (1 << 6) +#define GEN11_GRDOM_MEDIA3 (1 << 7) +#define GEN11_GRDOM_MEDIA4 (1 << 8) +#define GEN11_GRDOM_MEDIA5 (1 << 9) +#define GEN11_GRDOM_MEDIA6 (1 << 10) +#define GEN11_GRDOM_MEDIA7 (1 << 11) +#define GEN11_GRDOM_MEDIA8 (1 << 12) +#define GEN11_GRDOM_VECS (1 << 13) +#define GEN11_GRDOM_VECS2 (1 << 14) +#define GEN11_GRDOM_VECS3 (1 << 15) +#define GEN11_GRDOM_VECS4 (1 << 16) +#define GEN11_GRDOM_SFC0 (1 << 17) +#define GEN11_GRDOM_SFC1 (1 << 18) +#define GEN11_GRDOM_SFC2 (1 << 19) +#define GEN11_GRDOM_SFC3 (1 << 20) +#define GEN11_VCS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << ((instance) >> 1)) +#define GEN11_VECS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << (instance)) + +#define GEN6_RSTCTL _MMIO(0x9420) + +#define GEN7_MISCCPCTL _MMIO(0x9424) +#define GEN7_DOP_CLOCK_GATE_ENABLE (1 << 0) +#define GEN8_DOP_CLOCK_GATE_CFCLK_ENABLE (1 << 2) +#define GEN8_DOP_CLOCK_GATE_GUC_ENABLE (1 << 4) +#define GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE (1 << 6) + +#define GEN8_UCGCTL6 _MMIO(0x9430) +#define GEN8_GAPSUNIT_CLOCK_GATE_DISABLE (1 << 24) +#define GEN8_SDEUNIT_CLOCK_GATE_DISABLE (1 << 14) +#define GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ (1 << 28) + +#define UNSLCGCTL9430 _MMIO(0x9430) +#define MSQDUNIT_CLKGATE_DIS REG_BIT(3) + +#define UNSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9434) +#define VFUNIT_CLKGATE_DIS REG_BIT(20) +#define TSGUNIT_CLKGATE_DIS REG_BIT(17) /* XEHPSDV */ +#define CG3DDISCFEG_CLKGATE_DIS REG_BIT(17) /* DG2 */ +#define GAMEDIA_CLKGATE_DIS REG_BIT(11) +#define HSUNIT_CLKGATE_DIS REG_BIT(8) +#define VSUNIT_CLKGATE_DIS REG_BIT(3) + +#define UNSLCGCTL9440 _MMIO(0x9440) +#define GAMTLBOACS_CLKGATE_DIS REG_BIT(28) +#define GAMTLBVDBOX5_CLKGATE_DIS REG_BIT(27) +#define GAMTLBVDBOX6_CLKGATE_DIS REG_BIT(26) +#define GAMTLBVDBOX3_CLKGATE_DIS REG_BIT(24) +#define GAMTLBVDBOX4_CLKGATE_DIS REG_BIT(23) +#define GAMTLBVDBOX7_CLKGATE_DIS REG_BIT(22) +#define GAMTLBVDBOX2_CLKGATE_DIS REG_BIT(21) +#define GAMTLBVDBOX0_CLKGATE_DIS REG_BIT(17) +#define GAMTLBKCR_CLKGATE_DIS REG_BIT(16) +#define GAMTLBGUC_CLKGATE_DIS REG_BIT(15) +#define GAMTLBBLT_CLKGATE_DIS REG_BIT(14) +#define GAMTLBVDBOX1_CLKGATE_DIS REG_BIT(6) + +#define UNSLCGCTL9444 _MMIO(0x9444) +#define GAMTLBGFXA0_CLKGATE_DIS REG_BIT(30) +#define GAMTLBGFXA1_CLKGATE_DIS REG_BIT(29) +#define GAMTLBCOMPA0_CLKGATE_DIS REG_BIT(28) +#define GAMTLBCOMPA1_CLKGATE_DIS REG_BIT(27) +#define GAMTLBCOMPB0_CLKGATE_DIS REG_BIT(26) +#define GAMTLBCOMPB1_CLKGATE_DIS REG_BIT(25) +#define GAMTLBCOMPC0_CLKGATE_DIS REG_BIT(24) +#define GAMTLBCOMPC1_CLKGATE_DIS REG_BIT(23) +#define GAMTLBCOMPD0_CLKGATE_DIS REG_BIT(22) +#define GAMTLBCOMPD1_CLKGATE_DIS REG_BIT(21) +#define GAMTLBMERT_CLKGATE_DIS REG_BIT(20) +#define GAMTLBVEBOX3_CLKGATE_DIS REG_BIT(19) +#define GAMTLBVEBOX2_CLKGATE_DIS REG_BIT(18) +#define GAMTLBVEBOX1_CLKGATE_DIS REG_BIT(17) +#define GAMTLBVEBOX0_CLKGATE_DIS REG_BIT(16) +#define LTCDD_CLKGATE_DIS REG_BIT(10) + +#define SLICE_UNIT_LEVEL_CLKGATE _MMIO(0x94d4) +#define SARBUNIT_CLKGATE_DIS (1 << 5) +#define RCCUNIT_CLKGATE_DIS (1 << 7) +#define MSCUNIT_CLKGATE_DIS (1 << 10) +#define NODEDSS_CLKGATE_DIS REG_BIT(12) +#define L3_CLKGATE_DIS REG_BIT(16) +#define L3_CR2X_CLKGATE_DIS REG_BIT(17) + +#define SCCGCTL94DC _MMIO(0x94dc) +#define CG3DDISURB REG_BIT(14) + +#define UNSLICE_UNIT_LEVEL_CLKGATE2 _MMIO(0x94e4) +#define VSUNIT_CLKGATE_DIS_TGL REG_BIT(19) +#define PSDUNIT_CLKGATE_DIS REG_BIT(5) + +#define SUBSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9524) +#define DSS_ROUTER_CLKGATE_DIS REG_BIT(28) +#define GWUNIT_CLKGATE_DIS REG_BIT(16) + +#define SUBSLICE_UNIT_LEVEL_CLKGATE2 _MMIO(0x9528) +#define CPSSUNIT_CLKGATE_DIS REG_BIT(9) + +#define SSMCGCTL9530 _MMIO(0x9530) +#define RTFUNIT_CLKGATE_DIS REG_BIT(18) + +#define GEN10_DFR_RATIO_EN_AND_CHICKEN _MMIO(0x9550) +#define DFR_DISABLE (1 << 9) + +#define INF_UNIT_LEVEL_CLKGATE _MMIO(0x9560) +#define CGPSF_CLKGATE_DIS (1 << 3) + +#define MICRO_BP0_0 _MMIO(0x9800) +#define MICRO_BP0_2 _MMIO(0x9804) +#define MICRO_BP0_1 _MMIO(0x9808) +#define MICRO_BP1_0 _MMIO(0x980c) +#define MICRO_BP1_2 _MMIO(0x9810) +#define MICRO_BP1_1 _MMIO(0x9814) +#define MICRO_BP2_0 _MMIO(0x9818) +#define MICRO_BP2_2 _MMIO(0x981c) +#define MICRO_BP2_1 _MMIO(0x9820) +#define MICRO_BP3_0 _MMIO(0x9824) +#define MICRO_BP3_2 _MMIO(0x9828) +#define MICRO_BP3_1 _MMIO(0x982c) +#define MICRO_BP_TRIGGER _MMIO(0x9830) +#define MICRO_BP3_COUNT_STATUS01 _MMIO(0x9834) +#define MICRO_BP3_COUNT_STATUS23 _MMIO(0x9838) +#define MICRO_BP_FIRED_ARMED _MMIO(0x983c) + +#define GEN6_GFXPAUSE _MMIO(0xa000) +#define GEN6_RPNSWREQ _MMIO(0xa008) +#define GEN6_TURBO_DISABLE (1 << 31) +#define GEN6_FREQUENCY(x) ((x) << 25) +#define HSW_FREQUENCY(x) ((x) << 24) +#define GEN9_FREQUENCY(x) ((x) << 23) +#define GEN6_OFFSET(x) ((x) << 19) +#define GEN6_AGGRESSIVE_TURBO (0 << 15) +#define GEN9_SW_REQ_UNSLICE_RATIO_SHIFT 23 +#define GEN9_IGNORE_SLICE_RATIO (0 << 0) + +#define GEN6_RC_VIDEO_FREQ _MMIO(0xa00c) +#define GEN6_RC_CTL_RC6pp_ENABLE (1 << 16) +#define GEN6_RC_CTL_RC6p_ENABLE (1 << 17) +#define GEN6_RC_CTL_RC6_ENABLE (1 << 18) +#define GEN6_RC_CTL_RC1e_ENABLE (1 << 20) +#define GEN6_RC_CTL_RC7_ENABLE (1 << 22) +#define VLV_RC_CTL_CTX_RST_PARALLEL (1 << 24) +#define GEN7_RC_CTL_TO_MODE (1 << 28) +#define GEN6_RC_CTL_EI_MODE(x) ((x) << 27) +#define GEN6_RC_CTL_HW_ENABLE (1 << 31) +#define GEN6_RP_DOWN_TIMEOUT _MMIO(0xa010) +#define GEN6_RP_INTERRUPT_LIMITS _MMIO(0xa014) +#define GEN6_RPSTAT1 _MMIO(0xa01c) +#define GEN6_CAGF_SHIFT 8 +#define HSW_CAGF_SHIFT 7 +#define GEN9_CAGF_SHIFT 23 +#define GEN6_CAGF_MASK (0x7f << GEN6_CAGF_SHIFT) +#define HSW_CAGF_MASK (0x7f << HSW_CAGF_SHIFT) +#define GEN9_CAGF_MASK (0x1ff << GEN9_CAGF_SHIFT) +#define GEN6_RP_CONTROL _MMIO(0xa024) +#define GEN6_RP_MEDIA_TURBO (1 << 11) +#define GEN6_RP_MEDIA_MODE_MASK (3 << 9) +#define GEN6_RP_MEDIA_HW_TURBO_MODE (3 << 9) +#define GEN6_RP_MEDIA_HW_NORMAL_MODE (2 << 9) +#define GEN6_RP_MEDIA_HW_MODE (1 << 9) +#define GEN6_RP_MEDIA_SW_MODE (0 << 9) +#define GEN6_RP_MEDIA_IS_GFX (1 << 8) +#define GEN6_RP_ENABLE (1 << 7) +#define GEN6_RP_UP_IDLE_MIN (0x1 << 3) +#define GEN6_RP_UP_BUSY_AVG (0x2 << 3) +#define GEN6_RP_UP_BUSY_CONT (0x4 << 3) +#define GEN6_RP_DOWN_IDLE_AVG (0x2 << 0) +#define GEN6_RP_DOWN_IDLE_CONT (0x1 << 0) +#define GEN6_RPSWCTL_SHIFT 9 +#define GEN9_RPSWCTL_ENABLE (0x2 << GEN6_RPSWCTL_SHIFT) +#define GEN9_RPSWCTL_DISABLE (0x0 << GEN6_RPSWCTL_SHIFT) +#define GEN6_RP_UP_THRESHOLD _MMIO(0xa02c) +#define GEN6_RP_DOWN_THRESHOLD _MMIO(0xa030) +#define GEN6_RP_CUR_UP_EI _MMIO(0xa050) +#define GEN6_RP_EI_MASK 0xffffff +#define GEN6_CURICONT_MASK GEN6_RP_EI_MASK +#define GEN6_RP_CUR_UP _MMIO(0xa054) +#define GEN6_CURBSYTAVG_MASK GEN6_RP_EI_MASK +#define GEN6_RP_PREV_UP _MMIO(0xa058) +#define GEN6_RP_CUR_DOWN_EI _MMIO(0xa05c) +#define GEN6_CURIAVG_MASK GEN6_RP_EI_MASK +#define GEN6_RP_CUR_DOWN _MMIO(0xa060) +#define GEN6_RP_PREV_DOWN _MMIO(0xa064) +#define GEN6_RP_UP_EI _MMIO(0xa068) +#define GEN6_RP_DOWN_EI _MMIO(0xa06c) +#define GEN6_RP_IDLE_HYSTERSIS _MMIO(0xa070) +#define GEN6_RPDEUHWTC _MMIO(0xa080) +#define GEN6_RPDEUC _MMIO(0xa084) +#define GEN6_RPDEUCSW _MMIO(0xa088) +#define GEN6_RC_CONTROL _MMIO(0xa090) +#define GEN6_RC_STATE _MMIO(0xa094) +#define RC_SW_TARGET_STATE_SHIFT 16 +#define RC_SW_TARGET_STATE_MASK (7 << RC_SW_TARGET_STATE_SHIFT) +#define GEN6_RC1_WAKE_RATE_LIMIT _MMIO(0xa098) +#define GEN6_RC6_WAKE_RATE_LIMIT _MMIO(0xa09c) +#define GEN6_RC6pp_WAKE_RATE_LIMIT _MMIO(0xa0a0) +#define GEN10_MEDIA_WAKE_RATE_LIMIT _MMIO(0xa0a0) +#define GEN6_RC_EVALUATION_INTERVAL _MMIO(0xa0a8) +#define GEN6_RC_IDLE_HYSTERSIS _MMIO(0xa0ac) +#define GEN6_RC_SLEEP _MMIO(0xa0b0) +#define GEN6_RCUBMABDTMR _MMIO(0xa0b0) +#define GEN6_RC1e_THRESHOLD _MMIO(0xa0b4) +#define GEN6_RC6_THRESHOLD _MMIO(0xa0b8) +#define GEN6_RC6p_THRESHOLD _MMIO(0xa0bc) +#define VLV_RCEDATA _MMIO(0xa0bc) +#define GEN6_RC6pp_THRESHOLD _MMIO(0xa0c0) +#define GEN9_MEDIA_PG_IDLE_HYSTERESIS _MMIO(0xa0c4) +#define GEN9_RENDER_PG_IDLE_HYSTERESIS _MMIO(0xa0c8) + +#define GEN6_PMINTRMSK _MMIO(0xa168) +#define GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC (1 << 31) +#define ARAT_EXPIRED_INTRMSK (1 << 9) + +#define GEN8_MISC_CTRL0 _MMIO(0xa180) + +#define ECOBUS _MMIO(0xa180) +#define FORCEWAKE_MT_ENABLE (1 << 5) + +#define FORCEWAKE_MT _MMIO(0xa188) /* multi-threaded */ +#define FORCEWAKE_GT_GEN9 _MMIO(0xa188) +#define FORCEWAKE _MMIO(0xa18c) + +#define VLV_SPAREG2H _MMIO(0xa194) + +#define GEN9_PG_ENABLE _MMIO(0xa210) +#define GEN9_RENDER_PG_ENABLE REG_BIT(0) +#define GEN9_MEDIA_PG_ENABLE REG_BIT(1) +#define GEN11_MEDIA_SAMPLER_PG_ENABLE REG_BIT(2) +#define VDN_HCP_POWERGATE_ENABLE(n) REG_BIT(3 + 2 * (n)) +#define VDN_MFX_POWERGATE_ENABLE(n) REG_BIT(4 + 2 * (n)) + +#define GEN8_PUSHBUS_CONTROL _MMIO(0xa248) +#define GEN8_PUSHBUS_ENABLE _MMIO(0xa250) +#define GEN8_PUSHBUS_SHIFT _MMIO(0xa25c) + +/* GPM unit config (Gen9+) */ +#define CTC_MODE _MMIO(0xa26c) +#define CTC_SOURCE_PARAMETER_MASK 1 +#define CTC_SOURCE_CRYSTAL_CLOCK 0 +#define CTC_SOURCE_DIVIDE_LOGIC 1 +#define CTC_SHIFT_PARAMETER_SHIFT 1 +#define CTC_SHIFT_PARAMETER_MASK (0x3 << CTC_SHIFT_PARAMETER_SHIFT) + +#define FORCEWAKE_MEDIA_GEN9 _MMIO(0xa270) +#define FORCEWAKE_RENDER_GEN9 _MMIO(0xa278) + +#define VLV_PWRDWNUPCTL _MMIO(0xa294) + +#define GEN9_PWRGT_DOMAIN_STATUS _MMIO(0xa2a0) +#define GEN9_PWRGT_MEDIA_STATUS_MASK (1 << 0) +#define GEN9_PWRGT_RENDER_STATUS_MASK (1 << 1) + +#define MISC_STATUS0 _MMIO(0xa500) +#define MISC_STATUS1 _MMIO(0xa504) + +#define FORCEWAKE_MEDIA_VDBOX_GEN11(n) _MMIO(0xa540 + (n) * 4) +#define FORCEWAKE_MEDIA_VEBOX_GEN11(n) _MMIO(0xa560 + (n) * 4) + +#define CHV_POWER_SS0_SIG1 _MMIO(0xa720) +#define CHV_POWER_SS0_SIG2 _MMIO(0xa724) +#define CHV_POWER_SS1_SIG1 _MMIO(0xa728) +#define CHV_SS_PG_ENABLE (1 << 1) +#define CHV_EU08_PG_ENABLE (1 << 9) +#define CHV_EU19_PG_ENABLE (1 << 17) +#define CHV_EU210_PG_ENABLE (1 << 25) +#define CHV_POWER_SS1_SIG2 _MMIO(0xa72c) +#define CHV_EU311_PG_ENABLE (1 << 1) + +#define GEN7_SARCHKMD _MMIO(0xb000) +#define GEN7_DISABLE_DEMAND_PREFETCH (1 << 31) +#define GEN7_DISABLE_SAMPLER_PREFETCH (1 << 30) + +#define GEN8_GARBCNTL _MMIO(0xb004) +#define GEN9_GAPS_TSV_CREDIT_DISABLE (1 << 7) +#define GEN11_ARBITRATION_PRIO_ORDER_MASK (0x3f << 22) +#define GEN11_HASH_CTRL_EXCL_MASK (0x7f << 0) +#define GEN11_HASH_CTRL_EXCL_BIT0 (1 << 0) + +#define GEN9_SCRATCH_LNCF1 _MMIO(0xb008) +#define GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE REG_BIT(0) + +#define GEN7_L3SQCREG1 _MMIO(0xb010) +#define VLV_B0_WA_L3SQCREG1_VALUE 0x00D30000 + +#define GEN7_L3CNTLREG1 _MMIO(0xb01c) +#define GEN7_WA_FOR_GEN7_L3_CONTROL 0x3C47FF8C +#define GEN7_L3AGDIS (1 << 19) +#define GEN7_L3CNTLREG2 _MMIO(0xb020) + +/* MOCS (Memory Object Control State) registers */ +#define GEN9_LNCFCMOCS(i) _MMIO(0xb020 + (i) * 4) /* L3 Cache Control */ +#define GEN9_LNCFCMOCS_REG_COUNT 32 + +#define GEN7_L3CNTLREG3 _MMIO(0xb024) + +#define GEN7_L3_CHICKEN_MODE_REGISTER _MMIO(0xb030) +#define GEN7_WA_L3_CHICKEN_MODE 0x20000000 + +#define GEN7_L3SQCREG4 _MMIO(0xb034) +#define L3SQ_URB_READ_CAM_MATCH_DISABLE (1 << 27) + +#define HSW_SCRATCH1 _MMIO(0xb038) +#define HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE (1 << 27) + +#define GEN7_L3LOG(slice, i) _MMIO(0xb070 + (slice) * 0x200 + (i) * 4) +#define GEN7_L3LOG_SIZE 0x80 + +#define GEN10_SCRATCH_LNCF2 _MMIO(0xb0a0) +#define PMFLUSHDONE_LNICRSDROP (1 << 20) +#define PMFLUSH_GAPL3UNBLOCK (1 << 21) +#define PMFLUSHDONE_LNEBLK (1 << 22) + +#define XEHP_L3NODEARBCFG _MMIO(0xb0b4) +#define XEHP_LNESPARE REG_BIT(19) + +#define GEN8_L3SQCREG1 _MMIO(0xb100) +/* + * Note that on CHV the following has an off-by-one error wrt. to BSpec. + * Using the formula in BSpec leads to a hang, while the formula here works + * fine and matches the formulas for all other platforms. A BSpec change + * request has been filed to clarify this. + */ +#define L3_GENERAL_PRIO_CREDITS(x) (((x) >> 1) << 19) +#define L3_HIGH_PRIO_CREDITS(x) (((x) >> 1) << 14) +#define L3_PRIO_CREDITS_MASK ((0x1f << 19) | (0x1f << 14)) + +#define GEN10_L3_CHICKEN_MODE_REGISTER _MMIO(0xb114) +#define GEN11_I2M_WRITE_DISABLE (1 << 28) + +#define GEN8_L3SQCREG4 _MMIO(0xb118) +#define GEN11_LQSC_CLEAN_EVICT_DISABLE (1 << 6) +#define GEN8_LQSC_RO_PERF_DIS (1 << 27) +#define GEN8_LQSC_FLUSH_COHERENT_LINES (1 << 21) +#define GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE REG_BIT(22) + +#define GEN9_SCRATCH1 _MMIO(0xb11c) +#define EVICTION_PERF_FIX_ENABLE REG_BIT(8) + +#define BDW_SCRATCH1 _MMIO(0xb11c) +#define GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE (1 << 2) + +#define GEN11_SCRATCH2 _MMIO(0xb140) +#define GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE (1 << 19) + +#define GEN11_L3SQCREG5 _MMIO(0xb158) +#define L3_PWM_TIMER_INIT_VAL_MASK REG_GENMASK(9, 0) + +#define MLTICTXCTL _MMIO(0xb170) +#define TDONRENDER REG_BIT(2) + +#define XEHP_L3SCQREG7 _MMIO(0xb188) +#define BLEND_FILL_CACHING_OPT_DIS REG_BIT(3) + +#define L3SQCREG1_CCS0 _MMIO(0xb200) +#define FLUSHALLNONCOH REG_BIT(5) + +#define GEN11_GLBLINVL _MMIO(0xb404) +#define GEN11_BANK_HASH_ADDR_EXCL_MASK (0x7f << 5) +#define GEN11_BANK_HASH_ADDR_EXCL_BIT0 (1 << 5) + +#define GEN11_LSN_UNSLCVC _MMIO(0xb43c) +#define GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC (1 << 9) +#define GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC (1 << 7) + +#define __GEN9_RCS0_MOCS0 0xc800 +#define GEN9_GFX_MOCS(i) _MMIO(__GEN9_RCS0_MOCS0 + (i) * 4) +#define __GEN9_VCS0_MOCS0 0xc900 +#define GEN9_MFX0_MOCS(i) _MMIO(__GEN9_VCS0_MOCS0 + (i) * 4) +#define __GEN9_VCS1_MOCS0 0xca00 +#define GEN9_MFX1_MOCS(i) _MMIO(__GEN9_VCS1_MOCS0 + (i) * 4) +#define __GEN9_VECS0_MOCS0 0xcb00 +#define GEN9_VEBOX_MOCS(i) _MMIO(__GEN9_VECS0_MOCS0 + (i) * 4) +#define __GEN9_BCS0_MOCS0 0xcc00 +#define GEN9_BLT_MOCS(i) _MMIO(__GEN9_BCS0_MOCS0 + (i) * 4) + +#define GEN12_FAULT_TLB_DATA0 _MMIO(0xceb8) +#define GEN12_FAULT_TLB_DATA1 _MMIO(0xcebc) +#define FAULT_VA_HIGH_BITS (0xf << 0) +#define FAULT_GTT_SEL (1 << 4) + +#define GEN12_RING_FAULT_REG _MMIO(0xcec4) +#define GEN8_RING_FAULT_ENGINE_ID(x) (((x) >> 12) & 0x7) +#define RING_FAULT_GTTSEL_MASK (1 << 11) +#define RING_FAULT_SRCID(x) (((x) >> 3) & 0xff) +#define RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3) +#define RING_FAULT_VALID (1 << 0) + +#define GEN12_GFX_TLB_INV_CR _MMIO(0xced8) +#define GEN12_VD_TLB_INV_CR _MMIO(0xcedc) +#define GEN12_VE_TLB_INV_CR _MMIO(0xcee0) +#define GEN12_BLT_TLB_INV_CR _MMIO(0xcee4) + +#define GEN12_MERT_MOD_CTRL _MMIO(0xcf28) +#define RENDER_MOD_CTRL _MMIO(0xcf2c) +#define COMP_MOD_CTRL _MMIO(0xcf30) +#define VDBX_MOD_CTRL _MMIO(0xcf34) +#define VEBX_MOD_CTRL _MMIO(0xcf38) +#define FORCE_MISS_FTLB REG_BIT(3) + +#define GEN12_GAMSTLB_CTRL _MMIO(0xcf4c) +#define CONTROL_BLOCK_CLKGATE_DIS REG_BIT(12) +#define EGRESS_BLOCK_CLKGATE_DIS REG_BIT(11) +#define TAG_BLOCK_CLKGATE_DIS REG_BIT(7) + +#define GEN12_GAMCNTRL_CTRL _MMIO(0xcf54) +#define INVALIDATION_BROADCAST_MODE_DIS REG_BIT(12) +#define GLOBAL_INVALIDATION_MODE REG_BIT(2) + +#define GEN12_GAM_DONE _MMIO(0xcf68) + +#define GEN7_HALF_SLICE_CHICKEN1 _MMIO(0xe100) /* IVB GT1 + VLV */ +#define GEN7_MAX_PS_THREAD_DEP (8 << 12) +#define GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE (1 << 10) +#define GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE (1 << 4) +#define GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE (1 << 3) + +#define GEN7_SAMPLER_INSTDONE _MMIO(0xe160) +#define GEN7_ROW_INSTDONE _MMIO(0xe164) + +#define HALF_SLICE_CHICKEN2 _MMIO(0xe180) +#define GEN8_ST_PO_DISABLE (1 << 13) + +#define HALF_SLICE_CHICKEN3 _MMIO(0xe184) +#define HSW_SAMPLE_C_PERFORMANCE (1 << 9) +#define GEN8_CENTROID_PIXEL_OPT_DIS (1 << 8) +#define GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC (1 << 5) +#define GEN8_SAMPLER_POWER_BYPASS_DIS (1 << 1) + +#define GEN9_HALF_SLICE_CHICKEN5 _MMIO(0xe188) +#define GEN9_DG_MIRROR_FIX_ENABLE (1 << 5) +#define GEN9_CCS_TLB_PREFETCH_ENABLE (1 << 3) + +#define GEN10_SAMPLER_MODE _MMIO(0xe18c) +#define ENABLE_SMALLPL REG_BIT(15) +#define GEN11_SAMPLER_ENABLE_HEADLESS_MSG REG_BIT(5) + +#define GEN9_HALF_SLICE_CHICKEN7 _MMIO(0xe194) +#define DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA REG_BIT(15) +#define GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR REG_BIT(8) +#define GEN9_ENABLE_YV12_BUGFIX REG_BIT(4) +#define GEN9_ENABLE_GPGPU_PREEMPTION REG_BIT(2) + +#define GEN10_CACHE_MODE_SS _MMIO(0xe420) +#define ENABLE_PREFETCH_INTO_IC REG_BIT(3) +#define FLOAT_BLEND_OPTIMIZATION_ENABLE REG_BIT(4) + +#define EU_PERF_CNTL0 _MMIO(0xe458) +#define EU_PERF_CNTL4 _MMIO(0xe45c) + +#define GEN9_ROW_CHICKEN4 _MMIO(0xe48c) +#define GEN12_DISABLE_GRF_CLEAR REG_BIT(13) +#define XEHP_DIS_BBL_SYSPIPE REG_BIT(11) +#define GEN12_DISABLE_TDL_PUSH REG_BIT(9) +#define GEN11_DIS_PICK_2ND_EU REG_BIT(7) +#define GEN12_DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX REG_BIT(4) + +#define HSW_ROW_CHICKEN3 _MMIO(0xe49c) +#define HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE (1 << 6) + +#define GEN8_ROW_CHICKEN _MMIO(0xe4f0) +#define FLOW_CONTROL_ENABLE REG_BIT(15) +#define UGM_BACKUP_MODE REG_BIT(13) +#define MDQ_ARBITRATION_MODE REG_BIT(12) +#define SYSTOLIC_DOP_CLOCK_GATING_DIS REG_BIT(10) +#define PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE REG_BIT(8) +#define STALL_DOP_GATING_DISABLE REG_BIT(5) +#define THROTTLE_12_5 REG_GENMASK(4, 2) +#define DISABLE_EARLY_EOT REG_BIT(1) + +#define GEN7_ROW_CHICKEN2 _MMIO(0xe4f4) +#define GEN12_DISABLE_READ_SUPPRESSION REG_BIT(15) +#define GEN12_DISABLE_EARLY_READ REG_BIT(14) +#define GEN12_ENABLE_LARGE_GRF_MODE REG_BIT(12) +#define GEN12_PUSH_CONST_DEREF_HOLD_DIS REG_BIT(8) + +#define RT_CTRL _MMIO(0xe530) +#define DIS_NULL_QUERY REG_BIT(10) + +#define EU_PERF_CNTL1 _MMIO(0xe558) +#define EU_PERF_CNTL5 _MMIO(0xe55c) + +#define GEN12_HDC_CHICKEN0 _MMIO(0xe5f0) +#define LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK REG_GENMASK(13, 11) +#define ICL_HDC_MODE _MMIO(0xe5f4) + +#define EU_PERF_CNTL2 _MMIO(0xe658) +#define EU_PERF_CNTL6 _MMIO(0xe65c) +#define EU_PERF_CNTL3 _MMIO(0xe758) + +#define LSC_CHICKEN_BIT_0 _MMIO(0xe7c8) +#define FORCE_1_SUB_MESSAGE_PER_FRAGMENT REG_BIT(15) +#define LSC_CHICKEN_BIT_0_UDW _MMIO(0xe7c8 + 4) +#define DIS_CHAIN_2XSIMD8 REG_BIT(55 - 32) +#define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32) +#define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32) +#define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32) +#define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32) + +#define SARB_CHICKEN1 _MMIO(0xe90c) +#define COMP_CKN_IN REG_GENMASK(30, 29) + +#define GEN7_HALF_SLICE_CHICKEN1_GT2 _MMIO(0xf100) + +#define GEN7_ROW_CHICKEN2_GT2 _MMIO(0xf4f4) +#define DOP_CLOCK_GATING_DISABLE (1 << 0) +#define PUSH_CONSTANT_DEREF_DISABLE (1 << 8) +#define GEN11_TDL_CLOCK_GATING_FIX_DISABLE (1 << 1) + +#define __GEN11_VCS2_MOCS0 0x10000 +#define GEN11_MFX2_MOCS(i) _MMIO(__GEN11_VCS2_MOCS0 + (i) * 4) + +#define CRSTANDVID _MMIO(0x11100) +#define PXVFREQ(fstart) _MMIO(0x11110 + (fstart) * 4) /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */ +#define PXVFREQ_PX_MASK 0x7f000000 +#define PXVFREQ_PX_SHIFT 24 +#define VIDFREQ_BASE _MMIO(0x11110) +#define VIDFREQ1 _MMIO(0x11110) /* VIDFREQ1-4 (0x1111c) (Cantiga) */ +#define VIDFREQ2 _MMIO(0x11114) +#define VIDFREQ3 _MMIO(0x11118) +#define VIDFREQ4 _MMIO(0x1111c) +#define VIDFREQ_P0_MASK 0x1f000000 +#define VIDFREQ_P0_SHIFT 24 +#define VIDFREQ_P0_CSCLK_MASK 0x00f00000 +#define VIDFREQ_P0_CSCLK_SHIFT 20 +#define VIDFREQ_P0_CRCLK_MASK 0x000f0000 +#define VIDFREQ_P0_CRCLK_SHIFT 16 +#define VIDFREQ_P1_MASK 0x00001f00 +#define VIDFREQ_P1_SHIFT 8 +#define VIDFREQ_P1_CSCLK_MASK 0x000000f0 +#define VIDFREQ_P1_CSCLK_SHIFT 4 +#define VIDFREQ_P1_CRCLK_MASK 0x0000000f +#define INTTOEXT_BASE _MMIO(0x11120) /* INTTOEXT1-8 (0x1113c) */ +#define INTTOEXT_MAP3_SHIFT 24 +#define INTTOEXT_MAP3_MASK (0x1f << INTTOEXT_MAP3_SHIFT) +#define INTTOEXT_MAP2_SHIFT 16 +#define INTTOEXT_MAP2_MASK (0x1f << INTTOEXT_MAP2_SHIFT) +#define INTTOEXT_MAP1_SHIFT 8 +#define INTTOEXT_MAP1_MASK (0x1f << INTTOEXT_MAP1_SHIFT) +#define INTTOEXT_MAP0_SHIFT 0 +#define INTTOEXT_MAP0_MASK (0x1f << INTTOEXT_MAP0_SHIFT) +#define MEMSWCTL _MMIO(0x11170) /* Ironlake only */ +#define MEMCTL_CMD_MASK 0xe000 +#define MEMCTL_CMD_SHIFT 13 +#define MEMCTL_CMD_RCLK_OFF 0 +#define MEMCTL_CMD_RCLK_ON 1 +#define MEMCTL_CMD_CHFREQ 2 +#define MEMCTL_CMD_CHVID 3 +#define MEMCTL_CMD_VMMOFF 4 +#define MEMCTL_CMD_VMMON 5 +#define MEMCTL_CMD_STS (1 << 12) /* write 1 triggers command, clears + when command complete */ +#define MEMCTL_FREQ_MASK 0x0f00 /* jitter, from 0-15 */ +#define MEMCTL_FREQ_SHIFT 8 +#define MEMCTL_SFCAVM (1 << 7) +#define MEMCTL_TGT_VID_MASK 0x007f +#define MEMIHYST _MMIO(0x1117c) +#define MEMINTREN _MMIO(0x11180) /* 16 bits */ +#define MEMINT_RSEXIT_EN (1 << 8) +#define MEMINT_CX_SUPR_EN (1 << 7) +#define MEMINT_CONT_BUSY_EN (1 << 6) +#define MEMINT_AVG_BUSY_EN (1 << 5) +#define MEMINT_EVAL_CHG_EN (1 << 4) +#define MEMINT_MON_IDLE_EN (1 << 3) +#define MEMINT_UP_EVAL_EN (1 << 2) +#define MEMINT_DOWN_EVAL_EN (1 << 1) +#define MEMINT_SW_CMD_EN (1 << 0) +#define MEMINTRSTR _MMIO(0x11182) /* 16 bits */ +#define MEM_RSEXIT_MASK 0xc000 +#define MEM_RSEXIT_SHIFT 14 +#define MEM_CONT_BUSY_MASK 0x3000 +#define MEM_CONT_BUSY_SHIFT 12 +#define MEM_AVG_BUSY_MASK 0x0c00 +#define MEM_AVG_BUSY_SHIFT 10 +#define MEM_EVAL_CHG_MASK 0x0300 +#define MEM_EVAL_BUSY_SHIFT 8 +#define MEM_MON_IDLE_MASK 0x00c0 +#define MEM_MON_IDLE_SHIFT 6 +#define MEM_UP_EVAL_MASK 0x0030 +#define MEM_UP_EVAL_SHIFT 4 +#define MEM_DOWN_EVAL_MASK 0x000c +#define MEM_DOWN_EVAL_SHIFT 2 +#define MEM_SW_CMD_MASK 0x0003 +#define MEM_INT_STEER_GFX 0 +#define MEM_INT_STEER_CMR 1 +#define MEM_INT_STEER_SMI 2 +#define MEM_INT_STEER_SCI 3 +#define MEMINTRSTS _MMIO(0x11184) +#define MEMINT_RSEXIT (1 << 7) +#define MEMINT_CONT_BUSY (1 << 6) +#define MEMINT_AVG_BUSY (1 << 5) +#define MEMINT_EVAL_CHG (1 << 4) +#define MEMINT_MON_IDLE (1 << 3) +#define MEMINT_UP_EVAL (1 << 2) +#define MEMINT_DOWN_EVAL (1 << 1) +#define MEMINT_SW_CMD (1 << 0) +#define MEMMODECTL _MMIO(0x11190) +#define MEMMODE_BOOST_EN (1 << 31) +#define MEMMODE_BOOST_FREQ_MASK 0x0f000000 /* jitter for boost, 0-15 */ +#define MEMMODE_BOOST_FREQ_SHIFT 24 +#define MEMMODE_IDLE_MODE_MASK 0x00030000 +#define MEMMODE_IDLE_MODE_SHIFT 16 +#define MEMMODE_IDLE_MODE_EVAL 0 +#define MEMMODE_IDLE_MODE_CONT 1 +#define MEMMODE_HWIDLE_EN (1 << 15) +#define MEMMODE_SWMODE_EN (1 << 14) +#define MEMMODE_RCLK_GATE (1 << 13) +#define MEMMODE_HW_UPDATE (1 << 12) +#define MEMMODE_FSTART_MASK 0x00000f00 /* starting jitter, 0-15 */ +#define MEMMODE_FSTART_SHIFT 8 +#define MEMMODE_FMAX_MASK 0x000000f0 /* max jitter, 0-15 */ +#define MEMMODE_FMAX_SHIFT 4 +#define MEMMODE_FMIN_MASK 0x0000000f /* min jitter, 0-15 */ +#define RCBMAXAVG _MMIO(0x1119c) +#define MEMSWCTL2 _MMIO(0x1119e) /* Cantiga only */ +#define SWMEMCMD_RENDER_OFF (0 << 13) +#define SWMEMCMD_RENDER_ON (1 << 13) +#define SWMEMCMD_SWFREQ (2 << 13) +#define SWMEMCMD_TARVID (3 << 13) +#define SWMEMCMD_VRM_OFF (4 << 13) +#define SWMEMCMD_VRM_ON (5 << 13) +#define CMDSTS (1 << 12) +#define SFCAVM (1 << 11) +#define SWFREQ_MASK 0x0380 /* P0-7 */ +#define SWFREQ_SHIFT 7 +#define TARVID_MASK 0x001f +#define MEMSTAT_CTG _MMIO(0x111a0) +#define RCBMINAVG _MMIO(0x111a0) +#define RCUPEI _MMIO(0x111b0) +#define RCDNEI _MMIO(0x111b4) +#define RSTDBYCTL _MMIO(0x111b8) +#define RS1EN (1 << 31) +#define RS2EN (1 << 30) +#define RS3EN (1 << 29) +#define D3RS3EN (1 << 28) /* Display D3 imlies RS3 */ +#define SWPROMORSX (1 << 27) /* RSx promotion timers ignored */ +#define RCWAKERW (1 << 26) /* Resetwarn from PCH causes wakeup */ +#define DPRSLPVREN (1 << 25) /* Fast voltage ramp enable */ +#define GFXTGHYST (1 << 24) /* Hysteresis to allow trunk gating */ +#define RCX_SW_EXIT (1 << 23) /* Leave RSx and prevent re-entry */ +#define RSX_STATUS_MASK (7 << 20) +#define RSX_STATUS_ON (0 << 20) +#define RSX_STATUS_RC1 (1 << 20) +#define RSX_STATUS_RC1E (2 << 20) +#define RSX_STATUS_RS1 (3 << 20) +#define RSX_STATUS_RS2 (4 << 20) /* aka rc6 */ +#define RSX_STATUS_RSVD (5 << 20) /* deep rc6 unsupported on ilk */ +#define RSX_STATUS_RS3 (6 << 20) /* rs3 unsupported on ilk */ +#define RSX_STATUS_RSVD2 (7 << 20) +#define UWRCRSXE (1 << 19) /* wake counter limit prevents rsx */ +#define RSCRP (1 << 18) /* rs requests control on rs1/2 reqs */ +#define JRSC (1 << 17) /* rsx coupled to cpu c-state */ +#define RS2INC0 (1 << 16) /* allow rs2 in cpu c0 */ +#define RS1CONTSAV_MASK (3 << 14) +#define RS1CONTSAV_NO_RS1 (0 << 14) /* rs1 doesn't save/restore context */ +#define RS1CONTSAV_RSVD (1 << 14) +#define RS1CONTSAV_SAVE_RS1 (2 << 14) /* rs1 saves context */ +#define RS1CONTSAV_FULL_RS1 (3 << 14) /* rs1 saves and restores context */ +#define NORMSLEXLAT_MASK (3 << 12) +#define SLOW_RS123 (0 << 12) +#define SLOW_RS23 (1 << 12) +#define SLOW_RS3 (2 << 12) +#define NORMAL_RS123 (3 << 12) +#define RCMODE_TIMEOUT (1 << 11) /* 0 is eval interval method */ +#define IMPROMOEN (1 << 10) /* promo is immediate or delayed until next idle interval (only for timeout method above) */ +#define RCENTSYNC (1 << 9) /* rs coupled to cpu c-state (3/6/7) */ +#define STATELOCK (1 << 7) /* locked to rs_cstate if 0 */ +#define RS_CSTATE_MASK (3 << 4) +#define RS_CSTATE_C367_RS1 (0 << 4) +#define RS_CSTATE_C36_RS1_C7_RS2 (1 << 4) +#define RS_CSTATE_RSVD (2 << 4) +#define RS_CSTATE_C367_RS2 (3 << 4) +#define REDSAVES (1 << 3) /* no context save if was idle during rs0 */ +#define REDRESTORES (1 << 2) /* no restore if was idle during rs0 */ +#define VIDCTL _MMIO(0x111c0) +#define VIDSTS _MMIO(0x111c8) +#define VIDSTART _MMIO(0x111cc) /* 8 bits */ +#define MEMSTAT_ILK _MMIO(0x111f8) +#define MEMSTAT_VID_MASK 0x7f00 +#define MEMSTAT_VID_SHIFT 8 +#define MEMSTAT_PSTATE_MASK 0x00f8 +#define MEMSTAT_PSTATE_SHIFT 3 +#define MEMSTAT_MON_ACTV (1 << 2) +#define MEMSTAT_SRC_CTL_MASK 0x0003 +#define MEMSTAT_SRC_CTL_CORE 0 +#define MEMSTAT_SRC_CTL_TRB 1 +#define MEMSTAT_SRC_CTL_THM 2 +#define MEMSTAT_SRC_CTL_STDBY 3 +#define PMMISC _MMIO(0x11214) +#define MCPPCE_EN (1 << 0) /* enable PM_MSG from PCH->MPC */ +#define SDEW _MMIO(0x1124c) +#define CSIEW0 _MMIO(0x11250) +#define CSIEW1 _MMIO(0x11254) +#define CSIEW2 _MMIO(0x11258) +#define PEW(i) _MMIO(0x1125c + (i) * 4) /* 5 registers */ +#define DEW(i) _MMIO(0x11270 + (i) * 4) /* 3 registers */ +#define MCHAFE _MMIO(0x112c0) +#define CSIEC _MMIO(0x112e0) +#define DMIEC _MMIO(0x112e4) +#define DDREC _MMIO(0x112e8) +#define PEG0EC _MMIO(0x112ec) +#define PEG1EC _MMIO(0x112f0) +#define GFXEC _MMIO(0x112f4) +#define INTTOEXT_BASE_ILK _MMIO(0x11300) +#define RPPREVBSYTUPAVG _MMIO(0x113b8) +#define RCPREVBSYTUPAVG _MMIO(0x113b8) +#define RCPREVBSYTDNAVG _MMIO(0x113bc) +#define RPPREVBSYTDNAVG _MMIO(0x113bc) +#define ECR _MMIO(0x11600) +#define ECR_GPFE (1 << 31) +#define ECR_IMONE (1 << 30) +#define ECR_CAP_MASK 0x0000001f /* Event range, 0-31 */ +#define OGW0 _MMIO(0x11608) +#define OGW1 _MMIO(0x1160c) +#define EG0 _MMIO(0x11610) +#define EG1 _MMIO(0x11614) +#define EG2 _MMIO(0x11618) +#define EG3 _MMIO(0x1161c) +#define EG4 _MMIO(0x11620) +#define EG5 _MMIO(0x11624) +#define EG6 _MMIO(0x11628) +#define EG7 _MMIO(0x1162c) +#define PXW(i) _MMIO(0x11664 + (i) * 4) /* 4 registers */ +#define PXWL(i) _MMIO(0x11680 + (i) * 8) /* 8 registers */ +#define LCFUSE02 _MMIO(0x116c0) +#define LCFUSE_HIV_MASK 0x000000ff + +#define GAC_ECO_BITS _MMIO(0x14090) +#define ECOBITS_SNB_BIT (1 << 13) +#define ECOBITS_PPGTT_CACHE64B (3 << 8) +#define ECOBITS_PPGTT_CACHE4B (0 << 8) + +#define GEN12_RCU_MODE _MMIO(0x14800) +#define GEN12_RCU_MODE_CCS_ENABLE REG_BIT(0) + +#define CHV_FUSE_GT _MMIO(VLV_DISPLAY_BASE + 0x2168) +#define CHV_FGT_DISABLE_SS0 (1 << 10) +#define CHV_FGT_DISABLE_SS1 (1 << 11) +#define CHV_FGT_EU_DIS_SS0_R0_SHIFT 16 +#define CHV_FGT_EU_DIS_SS0_R0_MASK (0xf << CHV_FGT_EU_DIS_SS0_R0_SHIFT) +#define CHV_FGT_EU_DIS_SS0_R1_SHIFT 20 +#define CHV_FGT_EU_DIS_SS0_R1_MASK (0xf << CHV_FGT_EU_DIS_SS0_R1_SHIFT) +#define CHV_FGT_EU_DIS_SS1_R0_SHIFT 24 +#define CHV_FGT_EU_DIS_SS1_R0_MASK (0xf << CHV_FGT_EU_DIS_SS1_R0_SHIFT) +#define CHV_FGT_EU_DIS_SS1_R1_SHIFT 28 +#define CHV_FGT_EU_DIS_SS1_R1_MASK (0xf << CHV_FGT_EU_DIS_SS1_R1_SHIFT) + +#define BCS_SWCTRL _MMIO(0x22200) +#define BCS_SRC_Y REG_BIT(0) +#define BCS_DST_Y REG_BIT(1) + +#define GAB_CTL _MMIO(0x24000) +#define GAB_CTL_CONT_AFTER_PAGEFAULT (1 << 8) + +#define GEN6_PMISR _MMIO(0x44020) +#define GEN6_PMIMR _MMIO(0x44024) /* rps_lock */ +#define GEN6_PMIIR _MMIO(0x44028) +#define GEN6_PMIER _MMIO(0x4402c) +#define GEN6_PM_MBOX_EVENT (1 << 25) +#define GEN6_PM_THERMAL_EVENT (1 << 24) +/* + * For Gen11 these are in the upper word of the GPM_WGBOXPERF + * registers. Shifting is handled on accessing the imr and ier. + */ +#define GEN6_PM_RP_DOWN_TIMEOUT (1 << 6) +#define GEN6_PM_RP_UP_THRESHOLD (1 << 5) +#define GEN6_PM_RP_DOWN_THRESHOLD (1 << 4) +#define GEN6_PM_RP_UP_EI_EXPIRED (1 << 2) +#define GEN6_PM_RP_DOWN_EI_EXPIRED (1 << 1) +#define GEN6_PM_RPS_EVENTS (GEN6_PM_RP_UP_EI_EXPIRED | \ + GEN6_PM_RP_UP_THRESHOLD | \ + GEN6_PM_RP_DOWN_EI_EXPIRED | \ + GEN6_PM_RP_DOWN_THRESHOLD | \ + GEN6_PM_RP_DOWN_TIMEOUT) + +#define GEN7_GT_SCRATCH(i) _MMIO(0x4f100 + (i) * 4) +#define GEN7_GT_SCRATCH_REG_NUM 8 + +#define GFX_FLSH_CNTL_GEN6 _MMIO(0x101008) +#define GFX_FLSH_CNTL_EN (1 << 0) + +#define GTFIFODBG _MMIO(0x120000) +#define GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV (0x1f << 20) +#define GT_FIFO_FREE_ENTRIES_CHV (0x7f << 13) +#define GT_FIFO_SBDROPERR (1 << 6) +#define GT_FIFO_BLOBDROPERR (1 << 5) +#define GT_FIFO_SB_READ_ABORTERR (1 << 4) +#define GT_FIFO_DROPERR (1 << 3) +#define GT_FIFO_OVFERR (1 << 2) +#define GT_FIFO_IAWRERR (1 << 1) +#define GT_FIFO_IARDERR (1 << 0) + +#define GTFIFOCTL _MMIO(0x120008) +#define GT_FIFO_FREE_ENTRIES_MASK 0x7f +#define GT_FIFO_NUM_RESERVED_ENTRIES 20 +#define GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL (1 << 12) +#define GT_FIFO_CTL_RC6_POLICY_STALL (1 << 11) + +#define FORCEWAKE_MT_ACK _MMIO(0x130040) +#define FORCEWAKE_ACK_HSW _MMIO(0x130044) +#define FORCEWAKE_ACK_GT_GEN9 _MMIO(0x130044) +#define FORCEWAKE_KERNEL BIT(0) +#define FORCEWAKE_USER BIT(1) +#define FORCEWAKE_KERNEL_FALLBACK BIT(15) +#define FORCEWAKE_ACK _MMIO(0x130090) +#define VLV_GTLC_WAKE_CTRL _MMIO(0x130090) +#define VLV_GTLC_RENDER_CTX_EXISTS (1 << 25) +#define VLV_GTLC_MEDIA_CTX_EXISTS (1 << 24) +#define VLV_GTLC_ALLOWWAKEREQ (1 << 0) +#define VLV_GTLC_PW_STATUS _MMIO(0x130094) +#define VLV_GTLC_ALLOWWAKEACK (1 << 0) +#define VLV_GTLC_ALLOWWAKEERR (1 << 1) +#define VLV_GTLC_PW_MEDIA_STATUS_MASK (1 << 5) +#define VLV_GTLC_PW_RENDER_STATUS_MASK (1 << 7) +#define VLV_GTLC_SURVIVABILITY_REG _MMIO(0x130098) +#define VLV_GFX_CLK_STATUS_BIT (1 << 3) +#define VLV_GFX_CLK_FORCE_ON_BIT (1 << 2) +#define FORCEWAKE_VLV _MMIO(0x1300b0) +#define FORCEWAKE_ACK_VLV _MMIO(0x1300b4) +#define FORCEWAKE_MEDIA_VLV _MMIO(0x1300b8) +#define FORCEWAKE_ACK_MEDIA_VLV _MMIO(0x1300bc) + +#define GEN6_GT_THREAD_STATUS_REG _MMIO(0x13805c) +#define GEN6_GT_THREAD_STATUS_CORE_MASK 0x7 + +#define GEN6_GT_CORE_STATUS _MMIO(0x138060) +#define GEN6_CORE_CPD_STATE_MASK (7 << 4) +#define GEN6_RCn_MASK 7 +#define GEN6_RC0 0 +#define GEN6_RC3 2 +#define GEN6_RC6 3 +#define GEN6_RC7 4 + +#define GEN8_GT_SLICE_INFO _MMIO(0x138064) +#define GEN8_LSLICESTAT_MASK 0x7 + +#define GEN6_GT_GFX_RC6_LOCKED _MMIO(0x138104) +#define VLV_COUNTER_CONTROL _MMIO(0x138104) +#define VLV_COUNT_RANGE_HIGH (1 << 15) +#define VLV_MEDIA_RC0_COUNT_EN (1 << 5) +#define VLV_RENDER_RC0_COUNT_EN (1 << 4) +#define VLV_MEDIA_RC6_COUNT_EN (1 << 1) +#define VLV_RENDER_RC6_COUNT_EN (1 << 0) +#define GEN6_GT_GFX_RC6 _MMIO(0x138108) +#define VLV_GT_RENDER_RC6 _MMIO(0x138108) +#define VLV_GT_MEDIA_RC6 _MMIO(0x13810c) + +#define GEN6_GT_GFX_RC6p _MMIO(0x13810c) +#define GEN6_GT_GFX_RC6pp _MMIO(0x138110) +#define VLV_RENDER_C0_COUNT _MMIO(0x138118) +#define VLV_MEDIA_C0_COUNT _MMIO(0x13811c) + +#define GEN11_GT_INTR_DW(x) _MMIO(0x190018 + ((x) * 4)) +#define GEN11_CSME (31) +#define GEN11_GUNIT (28) +#define GEN11_GUC (25) +#define GEN11_WDPERF (20) +#define GEN11_KCR (19) +#define GEN11_GTPM (16) +#define GEN11_BCS (15) +#define GEN12_CCS3 (7) +#define GEN12_CCS2 (6) +#define GEN12_CCS1 (5) +#define GEN12_CCS0 (4) +#define GEN11_RCS0 (0) +#define GEN11_VECS(x) (31 - (x)) +#define GEN11_VCS(x) (x) + +#define GEN11_RENDER_COPY_INTR_ENABLE _MMIO(0x190030) +#define GEN11_VCS_VECS_INTR_ENABLE _MMIO(0x190034) +#define GEN11_GUC_SG_INTR_ENABLE _MMIO(0x190038) +#define ENGINE1_MASK REG_GENMASK(31, 16) +#define ENGINE0_MASK REG_GENMASK(15, 0) +#define GEN11_GPM_WGBOXPERF_INTR_ENABLE _MMIO(0x19003c) +#define GEN11_CRYPTO_RSVD_INTR_ENABLE _MMIO(0x190040) +#define GEN11_GUNIT_CSME_INTR_ENABLE _MMIO(0x190044) +#define GEN12_CCS_RSVD_INTR_ENABLE _MMIO(0x190048) + +#define GEN11_INTR_IDENTITY_REG(x) _MMIO(0x190060 + ((x) * 4)) +#define GEN11_INTR_DATA_VALID (1 << 31) +#define GEN11_INTR_ENGINE_CLASS(x) (((x) & GENMASK(18, 16)) >> 16) +#define GEN11_INTR_ENGINE_INSTANCE(x) (((x) & GENMASK(25, 20)) >> 20) +#define GEN11_INTR_ENGINE_INTR(x) ((x) & 0xffff) +/* irq instances for OTHER_CLASS */ +#define OTHER_GUC_INSTANCE 0 +#define OTHER_GTPM_INSTANCE 1 +#define OTHER_KCR_INSTANCE 4 + +#define GEN11_IIR_REG_SELECTOR(x) _MMIO(0x190070 + ((x) * 4)) + +#define GEN11_RCS0_RSVD_INTR_MASK _MMIO(0x190090) +#define GEN11_BCS_RSVD_INTR_MASK _MMIO(0x1900a0) +#define GEN11_VCS0_VCS1_INTR_MASK _MMIO(0x1900a8) +#define GEN11_VCS2_VCS3_INTR_MASK _MMIO(0x1900ac) +#define GEN12_VCS4_VCS5_INTR_MASK _MMIO(0x1900b0) +#define GEN12_VCS6_VCS7_INTR_MASK _MMIO(0x1900b4) +#define GEN11_VECS0_VECS1_INTR_MASK _MMIO(0x1900d0) +#define GEN12_VECS2_VECS3_INTR_MASK _MMIO(0x1900d4) +#define GEN11_GUC_SG_INTR_MASK _MMIO(0x1900e8) +#define GEN11_GPM_WGBOXPERF_INTR_MASK _MMIO(0x1900ec) +#define GEN11_CRYPTO_RSVD_INTR_MASK _MMIO(0x1900f0) +#define GEN11_GUNIT_CSME_INTR_MASK _MMIO(0x1900f4) +#define GEN12_CCS0_CCS1_INTR_MASK _MMIO(0x190100) +#define GEN12_CCS2_CCS3_INTR_MASK _MMIO(0x190104) + +#define GEN12_SFC_DONE(n) _MMIO(0x1cc000 + (n) * 0x1000) + +#endif /* __INTEL_GT_REGS__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index a94be0306464..a5f5b2dda332 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -10,9 +10,11 @@ #include <drm/drm_cache.h> +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_lmem.h" #include "i915_trace.h" #include "intel_gt.h" +#include "intel_gt_regs.h" #include "intel_gtt.h" struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz) @@ -105,14 +107,19 @@ void __i915_vm_close(struct i915_address_space *vm) list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { struct drm_i915_gem_object *obj = vma->obj; - /* Keep the obj (and hence the vma) alive as _we_ destroy it */ - if (!kref_get_unless_zero(&obj->base.refcount)) + if (!kref_get_unless_zero(&obj->base.refcount)) { + /* + * Unbind the dying vma to ensure the bound_list + * is completely drained. We leave the destruction to + * the object destructor. + */ + atomic_and(~I915_VMA_PIN_MASK, &vma->flags); + WARN_ON(__i915_vma_unbind(vma)); continue; + } - atomic_and(~I915_VMA_PIN_MASK, &vma->flags); - WARN_ON(__i915_vma_unbind(vma)); - __i915_vma_put(vma); - + /* Keep the obj (and hence the vma) alive as _we_ destroy it */ + i915_vma_destroy_locked(vma); i915_gem_object_put(obj); } GEM_BUG_ON(!list_empty(&vm->bound_list)); @@ -161,6 +168,9 @@ static void __i915_vm_release(struct work_struct *work) struct i915_address_space *vm = container_of(work, struct i915_address_space, release_work); + /* Synchronize async unbinds. */ + i915_vma_resource_bind_dep_sync_all(vm); + vm->cleanup(vm); i915_address_space_fini(vm); @@ -189,6 +199,7 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass) if (!kref_read(&vm->resv_ref)) kref_init(&vm->resv_ref); + vm->pending_unbind = RB_ROOT_CACHED; INIT_WORK(&vm->release_work, __i915_vm_release); atomic_set(&vm->open, 1); @@ -219,6 +230,19 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass) GEM_BUG_ON(!vm->total); drm_mm_init(&vm->mm, 0, vm->total); + + memset64(vm->min_alignment, I915_GTT_MIN_ALIGNMENT, + ARRAY_SIZE(vm->min_alignment)); + + if (HAS_64K_PAGES(vm->i915) && NEEDS_COMPACT_PT(vm->i915) && + subclass == VM_CLASS_PPGTT) { + vm->min_alignment[INTEL_MEMORY_LOCAL] = I915_GTT_PAGE_SIZE_2M; + vm->min_alignment[INTEL_MEMORY_STOLEN_LOCAL] = I915_GTT_PAGE_SIZE_2M; + } else if (HAS_64K_PAGES(vm->i915)) { + vm->min_alignment[INTEL_MEMORY_LOCAL] = I915_GTT_PAGE_SIZE_64K; + vm->min_alignment[INTEL_MEMORY_STOLEN_LOCAL] = I915_GTT_PAGE_SIZE_64K; + } + vm->mm.head_node.color = I915_COLOR_UNEVICTABLE; INIT_LIST_HEAD(&vm->bound_list); diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index 177b42b935a1..9d83c2d3959c 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -27,7 +27,10 @@ #include "gt/intel_reset.h" #include "i915_selftest.h" +#include "i915_vma_resource.h" #include "i915_vma_types.h" +#include "i915_params.h" +#include "intel_memory_region.h" #define I915_GFP_ALLOW_FAIL (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN) @@ -89,6 +92,8 @@ typedef u64 gen8_pte_t; #define GEN12_GGTT_PTE_LM BIT_ULL(1) +#define GEN12_PDE_64K BIT(6) + /* * Cacheability Control is a 4-bit value. The low three bits are stored in bits * 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. @@ -157,6 +162,7 @@ struct i915_page_table { atomic_t used; struct i915_page_table *stash; }; + bool is_compact; }; struct i915_page_directory { @@ -194,13 +200,21 @@ void *__px_vaddr(struct drm_i915_gem_object *p); struct i915_vm_pt_stash { /* preallocated chains of page tables/directories */ struct i915_page_table *pt[2]; + /* + * Optionally override the alignment/size of the physical page that + * contains each PT. If not set defaults back to the usual + * I915_GTT_PAGE_SIZE_4K. This does not influence the other paging + * structures. MUST be a power-of-two. ONLY applicable on discrete + * platforms. + */ + int pt_sz; }; struct i915_vma_ops { /* Map an object into an address space with the given cache flags. */ void (*bind_vma)(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags); /* @@ -208,7 +222,8 @@ struct i915_vma_ops { * setting the valid PTE entries to a reserved scratch page. */ void (*unbind_vma)(struct i915_address_space *vm, - struct i915_vma *vma); + struct i915_vma_resource *vma_res); + }; struct i915_address_space { @@ -221,6 +236,7 @@ struct i915_address_space { struct device *dma; u64 total; /* size addr space maps (ex. 2GB for ggtt) */ u64 reserved; /* size addr space reserved */ + u64 min_alignment[INTEL_MEMORY_STOLEN_LOCAL + 1]; unsigned int bind_async_flags; @@ -263,6 +279,9 @@ struct i915_address_space { /* Flags used when creating page-table objects for this vm */ unsigned long lmem_pt_obj_flags; + /* Interval tree for pending unbind vma resources */ + struct rb_root_cached pending_unbind; + struct drm_i915_gem_object * (*alloc_pt_dma)(struct i915_address_space *vm, int sz); struct drm_i915_gem_object * @@ -285,7 +304,7 @@ struct i915_address_space { enum i915_cache_level cache_level, u32 flags); void (*insert_entries)(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags); void (*cleanup)(struct i915_address_space *vm); @@ -379,6 +398,25 @@ i915_vm_has_scratch_64K(struct i915_address_space *vm) return vm->scratch_order == get_order(I915_GTT_PAGE_SIZE_64K); } +static inline u64 i915_vm_min_alignment(struct i915_address_space *vm, + enum intel_memory_type type) +{ + /* avoid INTEL_MEMORY_MOCK overflow */ + if ((int)type >= ARRAY_SIZE(vm->min_alignment)) + type = INTEL_MEMORY_SYSTEM; + + return vm->min_alignment[type]; +} + +static inline u64 i915_vm_obj_min_alignment(struct i915_address_space *vm, + struct drm_i915_gem_object *obj) +{ + struct intel_memory_region *mr = READ_ONCE(obj->mm.region); + enum intel_memory_type type = mr ? mr->type : INTEL_MEMORY_SYSTEM; + + return i915_vm_min_alignment(vm, type); +} + static inline bool i915_vm_has_cache_coloring(struct i915_address_space *vm) { @@ -565,7 +603,7 @@ void free_scratch(struct i915_address_space *vm); struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz); struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz); -struct i915_page_table *alloc_pt(struct i915_address_space *vm); +struct i915_page_table *alloc_pt(struct i915_address_space *vm, int sz); struct i915_page_directory *alloc_pd(struct i915_address_space *vm); struct i915_page_directory *__alloc_pd(int npde); @@ -600,11 +638,11 @@ void gen6_ggtt_invalidate(struct i915_ggtt *ggtt); void ppgtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags); void ppgtt_unbind_vma(struct i915_address_space *vm, - struct i915_vma *vma); + struct i915_vma_resource *vma_res); void gtt_write_workarounds(struct intel_gt *gt); @@ -627,8 +665,8 @@ __vm_create_scratch_for_read_pinned(struct i915_address_space *vm, unsigned long static inline struct sgt_dma { struct scatterlist *sg; dma_addr_t dma, max; -} sgt_dma(struct i915_vma *vma) { - struct scatterlist *sg = vma->pages->sgl; +} sgt_dma(struct i915_vma_resource *vma_res) { + struct scatterlist *sg = vma_res->bi.pages->sgl; dma_addr_t addr = sg_dma_address(sg); return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) }; diff --git a/drivers/gpu/drm/i915/gt/intel_llc.c b/drivers/gpu/drm/i915/gt/intel_llc.c index 08d7d5ae263a..40e2e28ee6c7 100644 --- a/drivers/gpu/drm/i915/gt/intel_llc.c +++ b/drivers/gpu/drm/i915/gt/intel_llc.c @@ -7,8 +7,10 @@ #include <linux/cpufreq.h> #include "i915_drv.h" +#include "i915_reg.h" #include "intel_gt.h" #include "intel_llc.h" +#include "intel_mchbar_regs.h" #include "intel_pcode.h" struct ia_constants { @@ -140,11 +142,10 @@ static void gen6_update_ring_freq(struct intel_llc *llc) unsigned int ia_freq, ring_freq; calc_ia_freq(llc, gpu_freq, &consts, &ia_freq, &ring_freq); - sandybridge_pcode_write(i915, - GEN6_PCODE_WRITE_MIN_FREQ_TABLE, - ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT | - ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT | - gpu_freq); + snb_pcode_write(i915, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, + ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT | + ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT | + gpu_freq); } } diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index b3489599e4de..07bef7128fdb 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -8,9 +8,13 @@ #include "gen8_engine_cs.h" #include "i915_drv.h" #include "i915_perf.h" +#include "i915_reg.h" +#include "intel_context.h" #include "intel_engine.h" +#include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt.h" +#include "intel_gt_regs.h" #include "intel_lrc.h" #include "intel_lrc_reg.h" #include "intel_ring.h" @@ -619,7 +623,7 @@ static const u8 *reg_offsets(const struct intel_engine_cs *engine) GEM_BUG_ON(GRAPHICS_VER(engine->i915) >= 12 && !intel_engine_has_relative_mmio(engine)); - if (engine->class == RENDER_CLASS) { + if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) { if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) return dg2_rcs_offsets; else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) @@ -1065,6 +1069,10 @@ lrc_pin(struct intel_context *ce, void lrc_unpin(struct intel_context *ce) { + if (unlikely(ce->parallel.last_rq)) { + i915_request_put(ce->parallel.last_rq); + ce->parallel.last_rq = NULL; + } check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, ce->engine); } @@ -1160,6 +1168,29 @@ gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs) return cs; } +/* + * On DG2 during context restore of a preempted context in GPGPU mode, + * RCS restore hang is detected. This is extremely timing dependent. + * To address this below sw wabb is implemented for DG2 A steppings. + */ +static u32 * +dg2_emit_rcs_hang_wabb(const struct intel_context *ce, u32 *cs) +{ + *cs++ = MI_LOAD_REGISTER_IMM(1); + *cs++ = i915_mmio_reg_offset(GEN12_STATE_ACK_DEBUG); + *cs++ = 0x21; + + *cs++ = MI_LOAD_REGISTER_REG; + *cs++ = i915_mmio_reg_offset(RING_NOPID(ce->engine->mmio_base)); + *cs++ = i915_mmio_reg_offset(GEN12_CULLBIT1); + + *cs++ = MI_LOAD_REGISTER_REG; + *cs++ = i915_mmio_reg_offset(RING_NOPID(ce->engine->mmio_base)); + *cs++ = i915_mmio_reg_offset(GEN12_CULLBIT2); + + return cs; +} + static u32 * gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) { @@ -1167,6 +1198,11 @@ gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) cs = gen12_emit_cmd_buf_wa(ce, cs); cs = gen12_emit_restore_scratch(ce, cs); + /* Wa_22011450934:dg2 */ + if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_A0, STEP_B0) || + IS_DG2_GRAPHICS_STEP(ce->engine->i915, G11, STEP_A0, STEP_B0)) + cs = dg2_emit_rcs_hang_wabb(ce, cs); + /* Wa_16013000631:dg2 */ if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_B0, STEP_C0) || IS_DG2_G11(ce->engine->i915)) @@ -1181,6 +1217,14 @@ gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) cs = gen12_emit_timestamp_wa(ce, cs); cs = gen12_emit_restore_scratch(ce, cs); + /* Wa_16013000631:dg2 */ + if (IS_DG2_GRAPHICS_STEP(ce->engine->i915, G10, STEP_B0, STEP_C0) || + IS_DG2_G11(ce->engine->i915)) + if (ce->engine->class == COMPUTE_CLASS) + cs = gen8_emit_pipe_control(cs, + PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, + 0); + return cs; } @@ -1583,7 +1627,7 @@ void lrc_init_wa_ctx(struct intel_engine_cs *engine) unsigned int i; int err; - if (engine->class != RENDER_CLASS) + if (!(engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) return; switch (GRAPHICS_VER(engine->i915)) { @@ -1686,6 +1730,17 @@ static void st_update_runtime_underflow(struct intel_context *ce, s32 dt) #endif } +static u32 lrc_get_runtime(const struct intel_context *ce) +{ + /* + * We can use either ppHWSP[16] which is recorded before the context + * switch (and so excludes the cost of context switches) or use the + * value from the context image itself, which is saved/restored earlier + * and so includes the cost of the save. + */ + return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]); +} + void lrc_update_runtime(struct intel_context *ce) { u32 old; diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.h b/drivers/gpu/drm/i915/gt/intel_lrc.h index 7f697845c4cf..6e4f9f58fca5 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc.h @@ -6,14 +6,17 @@ #ifndef __INTEL_LRC_H__ #define __INTEL_LRC_H__ -#include <linux/types.h> +#include "i915_priolist_types.h" -#include "intel_context.h" -#include "intel_lrc_reg.h" +#include <linux/bitfield.h> +#include <linux/types.h> struct drm_i915_gem_object; +struct i915_gem_ww_ctx; +struct intel_context; struct intel_engine_cs; struct intel_ring; +struct kref; /* At the start of the context image is its per-process HWS page */ #define LRC_PPHWSP_PN (0) @@ -68,15 +71,53 @@ void lrc_check_regs(const struct intel_context *ce, const char *when); void lrc_update_runtime(struct intel_context *ce); -static inline u32 lrc_get_runtime(const struct intel_context *ce) + +enum { + INTEL_ADVANCED_CONTEXT = 0, + INTEL_LEGACY_32B_CONTEXT, + INTEL_ADVANCED_AD_CONTEXT, + INTEL_LEGACY_64B_CONTEXT +}; + +enum { + FAULT_AND_HANG = 0, + FAULT_AND_HALT, /* Debug only */ + FAULT_AND_STREAM, + FAULT_AND_CONTINUE /* Unsupported */ +}; + +#define CTX_GTT_ADDRESS_MASK GENMASK(31, 12) +#define GEN8_CTX_VALID (1 << 0) +#define GEN8_CTX_FORCE_PD_RESTORE (1 << 1) +#define GEN8_CTX_FORCE_RESTORE (1 << 2) +#define GEN8_CTX_L3LLC_COHERENT (1 << 5) +#define GEN8_CTX_PRIVILEGE (1 << 8) +#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 +#define GEN12_CTX_PRIORITY_MASK GENMASK(10, 9) +#define GEN12_CTX_PRIORITY_HIGH FIELD_PREP(GEN12_CTX_PRIORITY_MASK, 2) +#define GEN12_CTX_PRIORITY_NORMAL FIELD_PREP(GEN12_CTX_PRIORITY_MASK, 1) +#define GEN12_CTX_PRIORITY_LOW FIELD_PREP(GEN12_CTX_PRIORITY_MASK, 0) +#define GEN8_CTX_ID_SHIFT 32 +#define GEN8_CTX_ID_WIDTH 21 +#define GEN11_SW_CTX_ID_SHIFT 37 +#define GEN11_SW_CTX_ID_WIDTH 11 +#define GEN11_ENGINE_CLASS_SHIFT 61 +#define GEN11_ENGINE_CLASS_WIDTH 3 +#define GEN11_ENGINE_INSTANCE_SHIFT 48 +#define GEN11_ENGINE_INSTANCE_WIDTH 6 +#define XEHP_SW_CTX_ID_SHIFT 39 +#define XEHP_SW_CTX_ID_WIDTH 16 +#define XEHP_SW_COUNTER_SHIFT 58 +#define XEHP_SW_COUNTER_WIDTH 6 + +static inline u32 lrc_desc_priority(int prio) { - /* - * We can use either ppHWSP[16] which is recorded before the context - * switch (and so excludes the cost of context switches) or use the - * value from the context image itself, which is saved/restored earlier - * and so includes the cost of the save. - */ - return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]); + if (prio > I915_PRIORITY_NORMAL) + return GEN12_CTX_PRIORITY_HIGH; + else if (prio < I915_PRIORITY_NORMAL) + return GEN12_CTX_PRIORITY_LOW; + else + return GEN12_CTX_PRIORITY_NORMAL; } #endif /* __INTEL_LRC_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h index f785d0ed238f..304000c7e345 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h @@ -53,21 +53,6 @@ #define GEN8_EXECLISTS_STATUS_BUF 0x370 #define GEN11_EXECLISTS_STATUS_BUF2 0x3c0 -/* Execlists regs */ -#define RING_ELSP(base) _MMIO((base) + 0x230) -#define RING_EXECLIST_STATUS_LO(base) _MMIO((base) + 0x234) -#define RING_EXECLIST_STATUS_HI(base) _MMIO((base) + 0x234 + 4) -#define RING_CONTEXT_CONTROL(base) _MMIO((base) + 0x244) -#define CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT REG_BIT(0) -#define CTX_CTRL_RS_CTX_ENABLE REG_BIT(1) -#define CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT REG_BIT(2) -#define CTX_CTRL_INHIBIT_SYN_CTX_SWITCH REG_BIT(3) -#define GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE REG_BIT(8) -#define RING_CONTEXT_STATUS_PTR(base) _MMIO((base) + 0x3a0) -#define RING_EXECLIST_SQ_CONTENTS(base) _MMIO((base) + 0x510) -#define RING_EXECLIST_CONTROL(base) _MMIO((base) + 0x550) -#define EL_CTRL_LOAD REG_BIT(0) - /* * The docs specify that the write pointer wraps around after 5h, "After status * is written out to the last available status QW at offset 5h, this pointer diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c index 18b44af56969..20444d6ceb3c 100644 --- a/drivers/gpu/drm/i915/gt/intel_migrate.c +++ b/drivers/gpu/drm/i915/gt/intel_migrate.c @@ -32,6 +32,38 @@ static bool engine_supports_migration(struct intel_engine_cs *engine) return true; } +static void xehpsdv_toggle_pdes(struct i915_address_space *vm, + struct i915_page_table *pt, + void *data) +{ + struct insert_pte_data *d = data; + + /* + * Insert a dummy PTE into every PT that will map to LMEM to ensure + * we have a correctly setup PDE structure for later use. + */ + vm->insert_page(vm, 0, d->offset, I915_CACHE_NONE, PTE_LM); + GEM_BUG_ON(!pt->is_compact); + d->offset += SZ_2M; +} + +static void xehpsdv_insert_pte(struct i915_address_space *vm, + struct i915_page_table *pt, + void *data) +{ + struct insert_pte_data *d = data; + + /* + * We are playing tricks here, since the actual pt, from the hw + * pov, is only 256bytes with 32 entries, or 4096bytes with 512 + * entries, but we are still guaranteed that the physical + * alignment is 64K underneath for the pt, and we are careful + * not to access the space in the void. + */ + vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, PTE_LM); + d->offset += SZ_64K; +} + static void insert_pte(struct i915_address_space *vm, struct i915_page_table *pt, void *data) @@ -74,7 +106,32 @@ static struct i915_address_space *migrate_vm(struct intel_gt *gt) * i.e. within the same non-preemptible window so that we do not switch * to another migration context that overwrites the PTE. * - * TODO: Add support for huge LMEM PTEs + * This changes quite a bit on platforms with HAS_64K_PAGES support, + * where we instead have three windows, each CHUNK_SIZE in size. The + * first is reserved for mapping system-memory, and that just uses the + * 512 entry layout using 4K GTT pages. The other two windows just map + * lmem pages and must use the new compact 32 entry layout using 64K GTT + * pages, which ensures we can address any lmem object that the user + * throws at us. We then also use the xehpsdv_toggle_pdes as a way of + * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the + * compact layout for each of these page-tables, that fall within the + * [CHUNK_SIZE, 3 * CHUNK_SIZE) range. + * + * We lay the ppGTT out as: + * + * [0, CHUNK_SZ) -> first window/object, maps smem + * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src + * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst + * + * For the PTE window it's also quite different, since each PTE must + * point to some 64K page, one for each PT(since it's in lmem), and yet + * each is only <= 4096bytes, but since the unused space within that PTE + * range is never touched, this should be fine. + * + * So basically each PT now needs 64K of virtual memory, instead of 4K, + * which looks like: + * + * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE */ vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY); @@ -86,6 +143,9 @@ static struct i915_address_space *migrate_vm(struct intel_gt *gt) goto err_vm; } + if (HAS_64K_PAGES(gt->i915)) + stash.pt_sz = I915_GTT_PAGE_SIZE_64K; + /* * Each engine instance is assigned its own chunk in the VM, so * that we can run multiple instances concurrently @@ -105,14 +165,20 @@ static struct i915_address_space *migrate_vm(struct intel_gt *gt) * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need * 4x2 page directories for source/destination. */ - sz = 2 * CHUNK_SZ; + if (HAS_64K_PAGES(gt->i915)) + sz = 3 * CHUNK_SZ; + else + sz = 2 * CHUNK_SZ; d.offset = base + sz; /* * We need another page directory setup so that we can write * the 8x512 PTE in each chunk. */ - sz += (sz >> 12) * sizeof(u64); + if (HAS_64K_PAGES(gt->i915)) + sz += (sz / SZ_2M) * SZ_64K; + else + sz += (sz >> 12) * sizeof(u64); err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz); if (err) @@ -133,7 +199,18 @@ static struct i915_address_space *migrate_vm(struct intel_gt *gt) goto err_vm; /* Now allow the GPU to rewrite the PTE via its own ppGTT */ - vm->vm.foreach(&vm->vm, base, d.offset - base, insert_pte, &d); + if (HAS_64K_PAGES(gt->i915)) { + vm->vm.foreach(&vm->vm, base, d.offset - base, + xehpsdv_insert_pte, &d); + d.offset = base + CHUNK_SZ; + vm->vm.foreach(&vm->vm, + d.offset, + 2 * CHUNK_SZ, + xehpsdv_toggle_pdes, &d); + } else { + vm->vm.foreach(&vm->vm, base, d.offset - base, + insert_pte, &d); + } } return &vm->vm; @@ -269,19 +346,38 @@ static int emit_pte(struct i915_request *rq, u64 offset, int length) { + bool has_64K_pages = HAS_64K_PAGES(rq->engine->i915); const u64 encode = rq->context->vm->pte_encode(0, cache_level, is_lmem ? PTE_LM : 0); struct intel_ring *ring = rq->ring; - int total = 0; + int pkt, dword_length; + u32 total = 0; + u32 page_size; u32 *hdr, *cs; - int pkt; GEM_BUG_ON(GRAPHICS_VER(rq->engine->i915) < 8); + page_size = I915_GTT_PAGE_SIZE; + dword_length = 0x400; + /* Compute the page directory offset for the target address range */ - offset >>= 12; - offset *= sizeof(u64); - offset += 2 * CHUNK_SZ; + if (has_64K_pages) { + GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M)); + + offset /= SZ_2M; + offset *= SZ_64K; + offset += 3 * CHUNK_SZ; + + if (is_lmem) { + page_size = I915_GTT_PAGE_SIZE_64K; + dword_length = 0x40; + } + } else { + offset >>= 12; + offset *= sizeof(u64); + offset += 2 * CHUNK_SZ; + } + offset += (u64)rq->engine->instance << 32; cs = intel_ring_begin(rq, 6); @@ -289,7 +385,7 @@ static int emit_pte(struct i915_request *rq, return PTR_ERR(cs); /* Pack as many PTE updates as possible into a single MI command */ - pkt = min_t(int, 0x400, ring->space / sizeof(u32) + 5); + pkt = min_t(int, dword_length, ring->space / sizeof(u32) + 5); pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5); hdr = cs; @@ -299,6 +395,8 @@ static int emit_pte(struct i915_request *rq, do { if (cs - hdr >= pkt) { + int dword_rem; + *hdr += cs - hdr - 2; *cs++ = MI_NOOP; @@ -310,7 +408,18 @@ static int emit_pte(struct i915_request *rq, if (IS_ERR(cs)) return PTR_ERR(cs); - pkt = min_t(int, 0x400, ring->space / sizeof(u32) + 5); + dword_rem = dword_length; + if (has_64K_pages) { + if (IS_ALIGNED(total, SZ_2M)) { + offset = round_up(offset, SZ_64K); + } else { + dword_rem = SZ_2M - (total & (SZ_2M - 1)); + dword_rem /= page_size; + dword_rem *= 2; + } + } + + pkt = min_t(int, dword_rem, ring->space / sizeof(u32) + 5); pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5); hdr = cs; @@ -319,13 +428,15 @@ static int emit_pte(struct i915_request *rq, *cs++ = upper_32_bits(offset); } + GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size)); + *cs++ = lower_32_bits(encode | it->dma); *cs++ = upper_32_bits(encode | it->dma); offset += 8; - total += I915_GTT_PAGE_SIZE; + total += page_size; - it->dma += I915_GTT_PAGE_SIZE; + it->dma += page_size; if (it->dma >= it->max) { it->sg = __sg_next(it->sg); if (!it->sg || sg_dma_len(it->sg) == 0) @@ -356,7 +467,8 @@ static bool wa_1209644611_applies(int ver, u32 size) return height % 4 == 3 && height <= 8; } -static int emit_copy(struct i915_request *rq, int size) +static int emit_copy(struct i915_request *rq, + u32 dst_offset, u32 src_offset, int size) { const int ver = GRAPHICS_VER(rq->engine->i915); u32 instance = rq->engine->instance; @@ -371,31 +483,31 @@ static int emit_copy(struct i915_request *rq, int size) *cs++ = BLT_DEPTH_32 | PAGE_SIZE; *cs++ = 0; *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; - *cs++ = CHUNK_SZ; /* dst offset */ + *cs++ = dst_offset; *cs++ = instance; *cs++ = 0; *cs++ = PAGE_SIZE; - *cs++ = 0; /* src offset */ + *cs++ = src_offset; *cs++ = instance; } else if (ver >= 8) { *cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2); *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; *cs++ = 0; *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; - *cs++ = CHUNK_SZ; /* dst offset */ + *cs++ = dst_offset; *cs++ = instance; *cs++ = 0; *cs++ = PAGE_SIZE; - *cs++ = 0; /* src offset */ + *cs++ = src_offset; *cs++ = instance; } else { GEM_BUG_ON(instance); *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE; - *cs++ = CHUNK_SZ; /* dst offset */ + *cs++ = dst_offset; *cs++ = PAGE_SIZE; - *cs++ = 0; /* src offset */ + *cs++ = src_offset; } intel_ring_advance(rq, cs); @@ -423,6 +535,7 @@ intel_context_migrate_copy(struct intel_context *ce, GEM_BUG_ON(ce->ring->size < SZ_64K); do { + u32 src_offset, dst_offset; int len; rq = i915_request_create(ce); @@ -450,15 +563,28 @@ intel_context_migrate_copy(struct intel_context *ce, if (err) goto out_rq; - len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem, 0, - CHUNK_SZ); + src_offset = 0; + dst_offset = CHUNK_SZ; + if (HAS_64K_PAGES(ce->engine->i915)) { + GEM_BUG_ON(!src_is_lmem && !dst_is_lmem); + + src_offset = 0; + dst_offset = 0; + if (src_is_lmem) + src_offset = CHUNK_SZ; + if (dst_is_lmem) + dst_offset = 2 * CHUNK_SZ; + } + + len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem, + src_offset, CHUNK_SZ); if (len <= 0) { err = len; goto out_rq; } err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem, - CHUNK_SZ, len); + dst_offset, len); if (err < 0) goto out_rq; if (err < len) { @@ -470,7 +596,7 @@ intel_context_migrate_copy(struct intel_context *ce, if (err) goto out_rq; - err = emit_copy(rq, len); + err = emit_copy(rq, dst_offset, src_offset, len); /* Arbitration is re-enabled between requests. */ out_rq: @@ -488,14 +614,15 @@ out_ce: return err; } -static int emit_clear(struct i915_request *rq, int size, u32 value) +static int emit_clear(struct i915_request *rq, u64 offset, int size, u32 value) { const int ver = GRAPHICS_VER(rq->engine->i915); - u32 instance = rq->engine->instance; u32 *cs; GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX); + offset += (u64)rq->engine->instance << 32; + cs = intel_ring_begin(rq, ver >= 8 ? 8 : 6); if (IS_ERR(cs)) return PTR_ERR(cs); @@ -505,17 +632,17 @@ static int emit_clear(struct i915_request *rq, int size, u32 value) *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; *cs++ = 0; *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; - *cs++ = 0; /* offset */ - *cs++ = instance; + *cs++ = lower_32_bits(offset); + *cs++ = upper_32_bits(offset); *cs++ = value; *cs++ = MI_NOOP; } else { - GEM_BUG_ON(instance); + GEM_BUG_ON(upper_32_bits(offset)); *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; *cs++ = 0; *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; - *cs++ = 0; + *cs++ = lower_32_bits(offset); *cs++ = value; } @@ -542,6 +669,7 @@ intel_context_migrate_clear(struct intel_context *ce, GEM_BUG_ON(ce->ring->size < SZ_64K); do { + u32 offset; int len; rq = i915_request_create(ce); @@ -569,7 +697,11 @@ intel_context_migrate_clear(struct intel_context *ce, if (err) goto out_rq; - len = emit_pte(rq, &it, cache_level, is_lmem, 0, CHUNK_SZ); + offset = 0; + if (HAS_64K_PAGES(ce->engine->i915) && is_lmem) + offset = CHUNK_SZ; + + len = emit_pte(rq, &it, cache_level, is_lmem, offset, CHUNK_SZ); if (len <= 0) { err = len; goto out_rq; @@ -579,7 +711,7 @@ intel_context_migrate_clear(struct intel_context *ce, if (err) goto out_rq; - err = emit_clear(rq, len, value); + err = emit_clear(rq, offset, len, value); /* Arbitration is re-enabled between requests. */ out_rq: diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.c b/drivers/gpu/drm/i915/gt/intel_mocs.c index 9c253ba593c6..c4c37585ae8c 100644 --- a/drivers/gpu/drm/i915/gt/intel_mocs.c +++ b/drivers/gpu/drm/i915/gt/intel_mocs.c @@ -7,7 +7,7 @@ #include "intel_engine.h" #include "intel_gt.h" -#include "intel_lrc_reg.h" +#include "intel_gt_regs.h" #include "intel_mocs.h" #include "intel_ring.h" diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c index 083b3090c69c..d91e2beb7517 100644 --- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c @@ -12,7 +12,7 @@ #include "gen6_ppgtt.h" #include "gen8_ppgtt.h" -struct i915_page_table *alloc_pt(struct i915_address_space *vm) +struct i915_page_table *alloc_pt(struct i915_address_space *vm, int sz) { struct i915_page_table *pt; @@ -20,12 +20,13 @@ struct i915_page_table *alloc_pt(struct i915_address_space *vm) if (unlikely(!pt)) return ERR_PTR(-ENOMEM); - pt->base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); + pt->base = vm->alloc_pt_dma(vm, sz); if (IS_ERR(pt->base)) { kfree(pt); return ERR_PTR(-ENOMEM); } + pt->is_compact = false; atomic_set(&pt->used, 0); return pt; } @@ -179,32 +180,34 @@ struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt, void ppgtt_bind_vma(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags) { u32 pte_flags; - if (!test_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) { - vm->allocate_va_range(vm, stash, vma->node.start, vma->size); - set_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma)); + if (!vma_res->allocated) { + vm->allocate_va_range(vm, stash, vma_res->start, + vma_res->vma_size); + vma_res->allocated = true; } /* Applicable to VLV, and gen8+ */ pte_flags = 0; - if (i915_gem_object_is_readonly(vma->obj)) + if (vma_res->bi.readonly) pte_flags |= PTE_READ_ONLY; - if (i915_gem_object_is_lmem(vma->obj)) + if (vma_res->bi.lmem) pte_flags |= PTE_LM; - vm->insert_entries(vm, vma, cache_level, pte_flags); + vm->insert_entries(vm, vma_res, cache_level, pte_flags); wmb(); } -void ppgtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) +void ppgtt_unbind_vma(struct i915_address_space *vm, + struct i915_vma_resource *vma_res) { - if (test_and_clear_bit(I915_VMA_ALLOC_BIT, __i915_vma_flags(vma))) - vm->clear_range(vm, vma->node.start, vma->size); + if (vma_res->allocated) + vm->clear_range(vm, vma_res->start, vma_res->vma_size); } static unsigned long pd_count(u64 size, int shift) @@ -218,17 +221,25 @@ int i915_vm_alloc_pt_stash(struct i915_address_space *vm, u64 size) { unsigned long count; - int shift, n; + int shift, n, pt_sz; shift = vm->pd_shift; if (!shift) return 0; + pt_sz = stash->pt_sz; + if (!pt_sz) + pt_sz = I915_GTT_PAGE_SIZE_4K; + else + GEM_BUG_ON(!IS_DGFX(vm->i915)); + + GEM_BUG_ON(!is_power_of_2(pt_sz)); + count = pd_count(size, shift); while (count--) { struct i915_page_table *pt; - pt = alloc_pt(vm); + pt = alloc_pt(vm, pt_sz); if (IS_ERR(pt)) { i915_vm_free_pt_stash(vm, stash); return PTR_ERR(pt); diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c index c3155ee58689..6df359c534fe 100644 --- a/drivers/gpu/drm/i915/gt/intel_rc6.c +++ b/drivers/gpu/drm/i915/gt/intel_rc6.c @@ -6,9 +6,12 @@ #include <linux/pm_runtime.h> #include "i915_drv.h" +#include "i915_reg.h" #include "i915_vgpu.h" +#include "intel_engine_regs.h" #include "intel_gt.h" #include "intel_gt_pm.h" +#include "intel_gt_regs.h" #include "intel_pcode.h" #include "intel_rc6.h" @@ -267,8 +270,7 @@ static void gen6_rc6_enable(struct intel_rc6 *rc6) GEN6_RC_CTL_HW_ENABLE; rc6vids = 0; - ret = sandybridge_pcode_read(i915, GEN6_PCODE_READ_RC6VIDS, - &rc6vids, NULL); + ret = snb_pcode_read(i915, GEN6_PCODE_READ_RC6VIDS, &rc6vids, NULL); if (GRAPHICS_VER(i915) == 6 && ret) { drm_dbg(&i915->drm, "Couldn't check for BIOS workaround\n"); } else if (GRAPHICS_VER(i915) == 6 && @@ -278,7 +280,7 @@ static void gen6_rc6_enable(struct intel_rc6 *rc6) GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); rc6vids &= 0xffff00; rc6vids |= GEN6_ENCODE_RC6_VID(450); - ret = sandybridge_pcode_write(i915, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); + ret = snb_pcode_write(i915, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); if (ret) drm_err(&i915->drm, "Couldn't fix incorrect rc6 voltage\n"); @@ -449,10 +451,10 @@ static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6) enable_rc6 = false; } - if (!((intel_uncore_read(uncore, PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1 && - (intel_uncore_read(uncore, PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1 && - (intel_uncore_read(uncore, PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1 && - (intel_uncore_read(uncore, PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1)) { + if (!((intel_uncore_read(uncore, PWRCTX_MAXCNT(RENDER_RING_BASE)) & IDLE_TIME_MASK) > 1 && + (intel_uncore_read(uncore, PWRCTX_MAXCNT(GEN6_BSD_RING_BASE)) & IDLE_TIME_MASK) > 1 && + (intel_uncore_read(uncore, PWRCTX_MAXCNT(BLT_RING_BASE)) & IDLE_TIME_MASK) > 1 && + (intel_uncore_read(uncore, PWRCTX_MAXCNT(VEBOX_RING_BASE)) & IDLE_TIME_MASK) > 1)) { drm_dbg(&i915->drm, "Engine Idle wait time not set properly.\n"); enable_rc6 = false; diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.h b/drivers/gpu/drm/i915/gt/intel_rc6.h index e119ec4a0bcc..b6fea71afc22 100644 --- a/drivers/gpu/drm/i915/gt/intel_rc6.h +++ b/drivers/gpu/drm/i915/gt/intel_rc6.h @@ -6,7 +6,7 @@ #ifndef INTEL_RC6_H #define INTEL_RC6_H -#include "i915_reg.h" +#include "i915_reg_defs.h" struct intel_engine_cs; struct intel_rc6; diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.c b/drivers/gpu/drm/i915/gt/intel_region_lmem.c index fde2dcb59809..6cecfdae07ad 100644 --- a/drivers/gpu/drm/i915/gt/intel_region_lmem.c +++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.c @@ -4,6 +4,7 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "intel_memory_region.h" #include "intel_region_lmem.h" #include "intel_region_ttm.h" @@ -11,60 +12,7 @@ #include "gem/i915_gem_region.h" #include "gem/i915_gem_ttm.h" #include "gt/intel_gt.h" - -static int init_fake_lmem_bar(struct intel_memory_region *mem) -{ - struct drm_i915_private *i915 = mem->i915; - struct i915_ggtt *ggtt = &i915->ggtt; - unsigned long n; - int ret; - - /* We want to 1:1 map the mappable aperture to our reserved region */ - - mem->fake_mappable.start = 0; - mem->fake_mappable.size = resource_size(&mem->region); - mem->fake_mappable.color = I915_COLOR_UNEVICTABLE; - - ret = drm_mm_reserve_node(&ggtt->vm.mm, &mem->fake_mappable); - if (ret) - return ret; - - mem->remap_addr = dma_map_resource(i915->drm.dev, - mem->region.start, - mem->fake_mappable.size, - DMA_BIDIRECTIONAL, - DMA_ATTR_FORCE_CONTIGUOUS); - if (dma_mapping_error(i915->drm.dev, mem->remap_addr)) { - drm_mm_remove_node(&mem->fake_mappable); - return -EINVAL; - } - - for (n = 0; n < mem->fake_mappable.size >> PAGE_SHIFT; ++n) { - ggtt->vm.insert_page(&ggtt->vm, - mem->remap_addr + (n << PAGE_SHIFT), - n << PAGE_SHIFT, - I915_CACHE_NONE, 0); - } - - mem->region = (struct resource)DEFINE_RES_MEM(mem->remap_addr, - mem->fake_mappable.size); - - return 0; -} - -static void release_fake_lmem_bar(struct intel_memory_region *mem) -{ - if (!drm_mm_node_allocated(&mem->fake_mappable)) - return; - - drm_mm_remove_node(&mem->fake_mappable); - - dma_unmap_resource(mem->i915->drm.dev, - mem->remap_addr, - mem->fake_mappable.size, - DMA_BIDIRECTIONAL, - DMA_ATTR_FORCE_CONTIGUOUS); -} +#include "gt/intel_gt_regs.h" static int region_lmem_release(struct intel_memory_region *mem) @@ -73,7 +21,6 @@ region_lmem_release(struct intel_memory_region *mem) ret = intel_region_ttm_fini(mem); io_mapping_fini(&mem->iomap); - release_fake_lmem_bar(mem); return ret; } @@ -83,17 +30,10 @@ region_lmem_init(struct intel_memory_region *mem) { int ret; - if (mem->i915->params.fake_lmem_start) { - ret = init_fake_lmem_bar(mem); - GEM_BUG_ON(ret); - } - if (!io_mapping_init_wc(&mem->iomap, mem->io_start, - resource_size(&mem->region))) { - ret = -EIO; - goto out_no_io; - } + mem->io_size)) + return -EIO; ret = intel_region_ttm_init(mem); if (ret) @@ -103,8 +43,6 @@ region_lmem_init(struct intel_memory_region *mem) out_no_buddy: io_mapping_fini(&mem->iomap); -out_no_io: - release_fake_lmem_bar(mem); return ret; } @@ -115,50 +53,6 @@ static const struct intel_memory_region_ops intel_region_lmem_ops = { .init_object = __i915_gem_ttm_object_init, }; -struct intel_memory_region * -intel_gt_setup_fake_lmem(struct intel_gt *gt) -{ - struct drm_i915_private *i915 = gt->i915; - struct pci_dev *pdev = to_pci_dev(i915->drm.dev); - struct intel_memory_region *mem; - resource_size_t mappable_end; - resource_size_t io_start; - resource_size_t start; - - if (!HAS_LMEM(i915)) - return ERR_PTR(-ENODEV); - - if (!i915->params.fake_lmem_start) - return ERR_PTR(-ENODEV); - - GEM_BUG_ON(i915_ggtt_has_aperture(&i915->ggtt)); - - /* Your mappable aperture belongs to me now! */ - mappable_end = pci_resource_len(pdev, 2); - io_start = pci_resource_start(pdev, 2); - start = i915->params.fake_lmem_start; - - mem = intel_memory_region_create(i915, - start, - mappable_end, - PAGE_SIZE, - io_start, - INTEL_MEMORY_LOCAL, - 0, - &intel_region_lmem_ops); - if (!IS_ERR(mem)) { - drm_info(&i915->drm, "Intel graphics fake LMEM: %pR\n", - &mem->region); - drm_info(&i915->drm, - "Intel graphics fake LMEM IO start: %llx\n", - (u64)mem->io_start); - drm_info(&i915->drm, "Intel graphics fake LMEM size: %llx\n", - (u64)resource_size(&mem->region)); - } - - return mem; -} - static bool get_legacy_lowmem_region(struct intel_uncore *uncore, u64 *start, u32 *size) { @@ -205,8 +99,29 @@ static struct intel_memory_region *setup_lmem(struct intel_gt *gt) if (!IS_DGFX(i915)) return ERR_PTR(-ENODEV); - /* Stolen starts from GSMBASE on DG1 */ - lmem_size = intel_uncore_read64(uncore, GEN12_GSMBASE); + if (HAS_FLAT_CCS(i915)) { + u64 tile_stolen, flat_ccs_base; + + lmem_size = pci_resource_len(pdev, 2); + flat_ccs_base = intel_gt_read_register(gt, XEHPSDV_FLAT_CCS_BASE_ADDR); + flat_ccs_base = (flat_ccs_base >> XEHPSDV_CCS_BASE_SHIFT) * SZ_64K; + + if (GEM_WARN_ON(lmem_size < flat_ccs_base)) + return ERR_PTR(-ENODEV); + + tile_stolen = lmem_size - flat_ccs_base; + + /* If the FLAT_CCS_BASE_ADDR register is not populated, flag an error */ + if (tile_stolen == lmem_size) + drm_err(&i915->drm, + "CCS_BASE_ADDR register did not have expected value\n"); + + lmem_size -= tile_stolen; + } else { + /* Stolen starts from GSMBASE without CCS */ + lmem_size = intel_uncore_read64(&i915->uncore, GEN12_GSMBASE); + } + io_start = pci_resource_start(pdev, 2); if (GEM_WARN_ON(lmem_size > pci_resource_len(pdev, 2))) @@ -219,6 +134,7 @@ static struct intel_memory_region *setup_lmem(struct intel_gt *gt) lmem_size, min_page_size, io_start, + lmem_size, INTEL_MEMORY_LOCAL, 0, &intel_region_lmem_ops); @@ -232,6 +148,8 @@ static struct intel_memory_region *setup_lmem(struct intel_gt *gt) drm_dbg(&i915->drm, "Local memory: %pR\n", &mem->region); drm_dbg(&i915->drm, "Local memory IO start: %pa\n", &mem->io_start); + drm_info(&i915->drm, "Local memory IO size: %pa\n", + &mem->io_size); drm_info(&i915->drm, "Local memory available: %pa\n", &lmem_size); diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.h b/drivers/gpu/drm/i915/gt/intel_region_lmem.h index 062d0542ae34..1438576b527a 100644 --- a/drivers/gpu/drm/i915/gt/intel_region_lmem.h +++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.h @@ -10,7 +10,4 @@ struct intel_gt; struct intel_memory_region *intel_gt_setup_lmem(struct intel_gt *gt); -struct intel_memory_region * -intel_gt_setup_fake_lmem(struct intel_gt *gt); - #endif /* !__INTEL_REGION_LMEM_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_renderstate.c b/drivers/gpu/drm/i915/gt/intel_renderstate.c index b575cd6e0b7a..5121e6dc2fa5 100644 --- a/drivers/gpu/drm/i915/gt/intel_renderstate.c +++ b/drivers/gpu/drm/i915/gt/intel_renderstate.c @@ -3,6 +3,8 @@ * Copyright © 2014 Intel Corporation */ +#include "gem/i915_gem_internal.h" + #include "i915_drv.h" #include "intel_renderstate.h" #include "intel_context.h" diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index 7be0002d9d70..82713264b96c 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -11,14 +11,20 @@ #include "gem/i915_gem_context.h" +#include "gt/intel_gt_regs.h" + #include "i915_drv.h" +#include "i915_file_private.h" #include "i915_gpu_error.h" #include "i915_irq.h" #include "intel_breadcrumbs.h" #include "intel_engine_pm.h" +#include "intel_engine_regs.h" #include "intel_gt.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" +#include "intel_mchbar_regs.h" +#include "intel_pci_config.h" #include "intel_reset.h" #include "uc/intel_guc.h" @@ -343,25 +349,25 @@ static void get_sfc_forced_lock_data(struct intel_engine_cs *engine, MISSING_CASE(engine->class); fallthrough; case VIDEO_DECODE_CLASS: - sfc_lock->lock_reg = GEN11_VCS_SFC_FORCED_LOCK(engine); + sfc_lock->lock_reg = GEN11_VCS_SFC_FORCED_LOCK(engine->mmio_base); sfc_lock->lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT; - sfc_lock->ack_reg = GEN11_VCS_SFC_LOCK_STATUS(engine); + sfc_lock->ack_reg = GEN11_VCS_SFC_LOCK_STATUS(engine->mmio_base); sfc_lock->ack_bit = GEN11_VCS_SFC_LOCK_ACK_BIT; - sfc_lock->usage_reg = GEN11_VCS_SFC_LOCK_STATUS(engine); + sfc_lock->usage_reg = GEN11_VCS_SFC_LOCK_STATUS(engine->mmio_base); sfc_lock->usage_bit = GEN11_VCS_SFC_USAGE_BIT; sfc_lock->reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance); break; case VIDEO_ENHANCEMENT_CLASS: - sfc_lock->lock_reg = GEN11_VECS_SFC_FORCED_LOCK(engine); + sfc_lock->lock_reg = GEN11_VECS_SFC_FORCED_LOCK(engine->mmio_base); sfc_lock->lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT; - sfc_lock->ack_reg = GEN11_VECS_SFC_LOCK_ACK(engine); + sfc_lock->ack_reg = GEN11_VECS_SFC_LOCK_ACK(engine->mmio_base); sfc_lock->ack_bit = GEN11_VECS_SFC_LOCK_ACK_BIT; - sfc_lock->usage_reg = GEN11_VECS_SFC_USAGE(engine); + sfc_lock->usage_reg = GEN11_VECS_SFC_USAGE(engine->mmio_base); sfc_lock->usage_bit = GEN11_VECS_SFC_USAGE_BIT; sfc_lock->reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance); @@ -408,7 +414,7 @@ static int gen11_lock_sfc(struct intel_engine_cs *engine, * forced lock on the VE engine that shares the same SFC. */ if (!(intel_uncore_read_fw(uncore, - GEN12_HCP_SFC_LOCK_STATUS(engine)) & + GEN12_HCP_SFC_LOCK_STATUS(engine->mmio_base)) & GEN12_HCP_SFC_USAGE_BIT)) return 0; @@ -598,6 +604,15 @@ static int gen8_reset_engines(struct intel_gt *gt, */ } + /* + * Wa_22011100796:dg2, whenever Full soft reset is required, + * reset all individual engines firstly, and then do a full soft reset. + * + * This is best effort, so ignore any error from the initial reset. + */ + if (IS_DG2(gt->i915) && engine_mask == ALL_ENGINES) + gen11_reset_engines(gt, gt->info.engine_mask, 0); + if (GRAPHICS_VER(gt->i915) >= 11) ret = gen11_reset_engines(gt, engine_mask, retry); else diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c index 2fdd52b62092..40ffcb94e379 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.c +++ b/drivers/gpu/drm/i915/gt/intel_ring.c @@ -3,12 +3,14 @@ * Copyright © 2019 Intel Corporation */ +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_object.h" #include "i915_drv.h" #include "i915_vma.h" #include "intel_engine.h" +#include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_ring.h" #include "intel_timeline.h" diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 3e6fac0340ef..6d7ec3bf1f32 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -3,6 +3,10 @@ * Copyright © 2008-2021 Intel Corporation */ +#include <drm/drm_cache.h> + +#include "gem/i915_gem_internal.h" + #include "gen2_engine_cs.h" #include "gen6_engine_cs.h" #include "gen6_ppgtt.h" @@ -11,8 +15,10 @@ #include "i915_mitigations.h" #include "intel_breadcrumbs.h" #include "intel_context.h" +#include "intel_engine_regs.h" #include "intel_gt.h" #include "intel_gt_irq.h" +#include "intel_gt_regs.h" #include "intel_reset.h" #include "intel_ring.h" #include "shmem_utils.h" @@ -1002,15 +1008,15 @@ static void gen6_bsd_submit_request(struct i915_request *request) /* Disable notification that the ring is IDLE. The GT * will then assume that it is busy and bring it out of rc6. */ - intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, - _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); + intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE), + _MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE)); /* Clear the context id. Here be magic! */ intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0); /* Wait for the ring not to be idle, i.e. for it to wake up. */ if (__intel_wait_for_register_fw(uncore, - GEN6_BSD_SLEEP_PSMI_CONTROL, + RING_PSMI_CTL(GEN6_BSD_RING_BASE), GEN6_BSD_SLEEP_INDICATOR, 0, 1000, 0, NULL)) @@ -1023,8 +1029,8 @@ static void gen6_bsd_submit_request(struct i915_request *request) /* Let the ring send IDLE messages to the GT again, * and so let it sleep to conserve power when idle. */ - intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, - _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); + intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE), + _MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE)); intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); } diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 54e7df788dbf..c8124101aada 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -6,11 +6,14 @@ #include <drm/i915_drm.h> #include "i915_drv.h" +#include "i915_irq.h" #include "intel_breadcrumbs.h" #include "intel_gt.h" #include "intel_gt_clock_utils.h" #include "intel_gt_irq.h" #include "intel_gt_pm_irq.h" +#include "intel_gt_regs.h" +#include "intel_mchbar_regs.h" #include "intel_pcode.h" #include "intel_rps.h" #include "vlv_sideband.h" @@ -1090,9 +1093,8 @@ static void gen6_rps_init(struct intel_rps *rps) IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) { u32 ddcc_status = 0; - if (sandybridge_pcode_read(i915, - HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, - &ddcc_status, NULL) == 0) + if (snb_pcode_read(i915, HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, + &ddcc_status, NULL) == 0) rps->efficient_freq = clamp_t(u8, (ddcc_status >> 8) & 0xff, @@ -1484,7 +1486,7 @@ void intel_rps_enable(struct intel_rps *rps) if (has_busy_stats(rps)) intel_rps_set_timer(rps); - else if (GRAPHICS_VER(i915) >= 6) + else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11) intel_rps_set_interrupts(rps); else /* Ironlake currently uses intel_ips.ko */ {} @@ -1940,8 +1942,7 @@ void intel_rps_init(struct intel_rps *rps) if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { u32 params = 0; - sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS, - ¶ms, NULL); + snb_pcode_read(i915, GEN6_READ_OC_PARAMS, ¶ms, NULL); if (params & BIT(31)) { /* OC supported */ drm_dbg(&i915->drm, "Overclocking supported, max: %dMHz, overclock: %dMHz\n", diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.c b/drivers/gpu/drm/i915/gt/intel_sseu.c index bdf09051b8a0..4ac0bbaf0c31 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu.c +++ b/drivers/gpu/drm/i915/gt/intel_sseu.c @@ -4,7 +4,8 @@ */ #include "i915_drv.h" -#include "intel_lrc_reg.h" +#include "intel_engine_regs.h" +#include "intel_gt_regs.h" #include "intel_sseu.h" void intel_sseu_set_info(struct sseu_dev_info *sseu, u8 max_slices, @@ -31,7 +32,9 @@ intel_sseu_subslice_total(const struct sseu_dev_info *sseu) return total; } -u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice) +static u32 +_intel_sseu_get_subslices(const struct sseu_dev_info *sseu, + const u8 *subslice_mask, u8 slice) { int i, offset = slice * sseu->ss_stride; u32 mask = 0; @@ -39,12 +42,21 @@ u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice) GEM_BUG_ON(slice >= sseu->max_slices); for (i = 0; i < sseu->ss_stride; i++) - mask |= (u32)sseu->subslice_mask[offset + i] << - i * BITS_PER_BYTE; + mask |= (u32)subslice_mask[offset + i] << i * BITS_PER_BYTE; return mask; } +u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice) +{ + return _intel_sseu_get_subslices(sseu, sseu->subslice_mask, slice); +} + +u32 intel_sseu_get_compute_subslices(const struct sseu_dev_info *sseu) +{ + return _intel_sseu_get_subslices(sseu, sseu->compute_subslice_mask, 0); +} + void intel_sseu_set_subslices(struct sseu_dev_info *sseu, int slice, u8 *subslice_mask, u32 ss_mask) { diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.h b/drivers/gpu/drm/i915/gt/intel_sseu.h index 60882a74741e..8a79cd8eaab4 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu.h +++ b/drivers/gpu/drm/i915/gt/intel_sseu.h @@ -103,7 +103,9 @@ intel_sseu_subslice_total(const struct sseu_dev_info *sseu); unsigned int intel_sseu_subslices_per_slice(const struct sseu_dev_info *sseu, u8 slice); -u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice); +u32 intel_sseu_get_subslices(const struct sseu_dev_info *sseu, u8 slice); + +u32 intel_sseu_get_compute_subslices(const struct sseu_dev_info *sseu); void intel_sseu_set_subslices(struct sseu_dev_info *sseu, int slice, u8 *subslice_mask, u32 ss_mask); diff --git a/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c b/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c index 8bb3a91dad82..903626f106ea 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c +++ b/drivers/gpu/drm/i915/gt/intel_sseu_debugfs.c @@ -6,6 +6,7 @@ #include "i915_drv.h" #include "intel_gt_debugfs.h" +#include "intel_gt_regs.h" #include "intel_sseu_debugfs.h" static void sseu_copy_subslices(const struct sseu_dev_info *sseu, diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c index 438bbc7b8147..b9640212d659 100644 --- a/drivers/gpu/drm/i915/gt/intel_timeline.c +++ b/drivers/gpu/drm/i915/gt/intel_timeline.c @@ -3,9 +3,12 @@ * Copyright © 2016-2018 Intel Corporation */ -#include "i915_drv.h" +#include <drm/drm_cache.h> + +#include "gem/i915_gem_internal.h" #include "i915_active.h" +#include "i915_drv.h" #include "i915_syncmap.h" #include "intel_gt.h" #include "intel_ring.h" diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index ab3277a3d593..c014b40d2e9f 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -6,8 +6,10 @@ #include "i915_drv.h" #include "intel_context.h" #include "intel_engine_pm.h" +#include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt.h" +#include "intel_gt_regs.h" #include "intel_ring.h" #include "intel_workarounds.h" @@ -235,7 +237,7 @@ static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine, wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); /* WaDisableAsyncFlipPerfMode:bdw,chv */ - wa_masked_en(wal, MI_MODE, ASYNC_FLIP_PERF_DISABLE); + wa_masked_en(wal, RING_MI_MODE(RENDER_RING_BASE), ASYNC_FLIP_PERF_DISABLE); /* WaDisablePartialInstShootdown:bdw,chv */ wa_masked_en(wal, GEN8_ROW_CHICKEN, @@ -682,11 +684,10 @@ static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine, wa_masked_en(wal, SLICE_COMMON_ECO_CHICKEN1, MSC_MSAA_REODER_BUF_BYPASS_DISABLE); - /* Wa_22012532006:dg2 */ - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0) || - IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) - wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, - DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA); + /* Wa_14014947963:dg2 */ + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_FOREVER) || + IS_DG2_G11(engine->i915) || IS_DG2_G12(engine->i915)) + wa_masked_field_set(wal, VF_PREEMPTION, PREEMPTION_VERTEX_COUNT, 0x4000); } static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine, @@ -1342,12 +1343,6 @@ xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) /* Wa_1409757795:xehpsdv */ wa_write_or(wal, SCCGCTL94DC, CG3DDISURB); - /* Wa_18011725039:xehpsdv */ - if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) { - wa_masked_dis(wal, MLTICTXCTL, TDONRENDER); - wa_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH); - } - /* Wa_16011155590:xehpsdv */ if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, @@ -1384,19 +1379,12 @@ xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) GAMTLBVEBOX0_CLKGATE_DIS); } - /* Wa_14012362059:xehpsdv */ - wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); - /* Wa_16012725990:xehpsdv */ if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_FOREVER)) wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, VFUNIT_CLKGATE_DIS); /* Wa_14011060649:xehpsdv */ wa_14011060649(gt, wal); - - /* Wa_14014368820:xehpsdv */ - wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | - GLOBAL_INVALIDATION_MODE); } static void @@ -1438,10 +1426,6 @@ dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) } if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0)) { - /* Wa_14010680813:dg2_g10 */ - wa_write_or(wal, GEN12_GAMSTLB_CTRL, CONTROL_BLOCK_CLKGATE_DIS | - EGRESS_BLOCK_CLKGATE_DIS | TAG_BLOCK_CLKGATE_DIS); - /* Wa_14010948348:dg2_g10 */ wa_write_or(wal, UNSLCGCTL9430, MSQDUNIT_CLKGATE_DIS); @@ -1488,16 +1472,6 @@ dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) wa_write_or(wal, SSMCGCTL9530, RTFUNIT_CLKGATE_DIS); } - if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0) || - IS_DG2_GRAPHICS_STEP(gt->i915, G11, STEP_A0, STEP_B0)) { - /* Wa_14012362059:dg2 */ - wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); - } - - /* Wa_1509235366:dg2 */ - wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | - GLOBAL_INVALIDATION_MODE); - /* Wa_14014830051:dg2 */ wa_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN); @@ -1506,7 +1480,6 @@ dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) * recommended tuning settings documented in the bspec's * performance guide section. */ - wa_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS); wa_write_or(wal, GEN12_SQCM, EN_32B_ACCESS); } @@ -1935,6 +1908,11 @@ static void dg2_whitelist_build(struct intel_engine_cs *engine) RING_FORCE_TO_NONPRIV_RANGE_4); break; + case COMPUTE_CLASS: + /* Wa_16011157294:dg2_g10 */ + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) + whitelist_reg(w, GEN9_CTX_PREEMPT_REG); + break; default: break; } @@ -2038,7 +2016,29 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { struct drm_i915_private *i915 = engine->i915; - if (IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) { + if (IS_DG2(i915)) { + /* Wa_14015227452:dg2 */ + wa_masked_en(wal, GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE); + + /* Wa_1509235366:dg2 */ + wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | + GLOBAL_INVALIDATION_MODE); + + /* + * The following are not actually "workarounds" but rather + * recommended tuning settings documented in the bspec's + * performance guide section. + */ + wa_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS); + + /* Wa_18018781329:dg2 */ + wa_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB); + wa_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB); + wa_write_or(wal, VDBX_MOD_CTRL, FORCE_MISS_FTLB); + wa_write_or(wal, VEBX_MOD_CTRL, FORCE_MISS_FTLB); + } + + if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) { /* Wa_14013392000:dg2_g11 */ wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_ENABLE_LARGE_GRF_MODE); @@ -2046,15 +2046,15 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8); } - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0) || - IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) { + if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0) || + IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) { /* Wa_14012419201:dg2 */ wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX); } - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_C0) || - IS_DG2_G11(engine->i915)) { + if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) || + IS_DG2_G11(i915)) { /* * Wa_22012826095:dg2 * Wa_22013059131:dg2 @@ -2069,14 +2069,14 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) } /* Wa_1308578152:dg2_g10 when first gslice is fused off */ - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_C0) && + if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) && needs_wa_1308578152(engine)) { wa_masked_dis(wal, GEN12_CS_DEBUG_MODE1_CCCSUNIT_BE_COMMON, GEN12_REPLAY_MODE_GRANULARITY); } - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_FOREVER) || - IS_DG2_G11(engine->i915)) { + if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) || + IS_DG2_G11(i915) || IS_DG2_G12(i915)) { /* Wa_22013037850:dg2 */ wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW, DISABLE_128B_EVICTION_COMMAND_UDW); @@ -2093,7 +2093,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK); } - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) { + if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) { /* * Wa_1608949956:dg2_g10 * Wa_14010198302:dg2_g10 @@ -2112,7 +2112,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) 0, false); } - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) { + if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) { /* Wa_22010430635:dg2 */ wa_masked_en(wal, GEN9_ROW_CHICKEN4, @@ -2122,8 +2122,8 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) wa_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE); } - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0) || - IS_DG2_G11(engine->i915)) { + if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_C0) || + IS_DG2_G11(i915)) { /* Wa_22012654132:dg2 */ wa_add(wal, GEN10_CACHE_MODE_SS, 0, _MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC), @@ -2132,10 +2132,28 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) } /* Wa_14013202645:dg2 */ - if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_C0) || - IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) + if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) || + IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) wa_write_or(wal, RT_CTRL, DIS_NULL_QUERY); + /* Wa_22012532006:dg2 */ + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0) || + IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) + wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, + DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA); + + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) { + /* Wa_14010680813:dg2_g10 */ + wa_write_or(wal, GEN12_GAMSTLB_CTRL, CONTROL_BLOCK_CLKGATE_DIS | + EGRESS_BLOCK_CLKGATE_DIS | TAG_BLOCK_CLKGATE_DIS); + } + + if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0) || + IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) { + /* Wa_14012362059:dg2 */ + wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); + } + if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) || IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) { /* @@ -2208,7 +2226,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) * For DG1 this only applies to A0. */ wa_masked_en(wal, - GEN6_RC_SLEEP_PSMI_CONTROL, + RING_PSMI_CTL(RENDER_RING_BASE), GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE | GEN8_RC_SEMA_IDLE_MSG_DISABLE); } @@ -2423,7 +2441,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) if (GRAPHICS_VER(i915) == 7) { /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ wa_masked_en(wal, - GFX_MODE_GEN7, + RING_MODE_GEN7(RENDER_RING_BASE), GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE); /* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */ @@ -2461,7 +2479,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv */ wa_masked_en(wal, - MI_MODE, + RING_MI_MODE(RENDER_RING_BASE), ASYNC_FLIP_PERF_DISABLE); if (GRAPHICS_VER(i915) == 6) { @@ -2520,7 +2538,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) if (IS_GRAPHICS_VER(i915, 4, 6)) /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ - wa_add(wal, MI_MODE, + wa_add(wal, RING_MI_MODE(RENDER_RING_BASE), 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH), /* XXX bit doesn't stick on Broadwater */ IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true); @@ -2536,7 +2554,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) * they are already accustomed to from before contexts were * enabled. */ - wa_add(wal, ECOSKPD, + wa_add(wal, ECOSKPD(RENDER_RING_BASE), 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE), 0 /* XXX bit doesn't stick on Broadwater */, true); @@ -2555,6 +2573,53 @@ xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) } } +/* + * The workarounds in this function apply to shared registers in + * the general render reset domain that aren't tied to a + * specific engine. Since all render+compute engines get reset + * together, and the contents of these registers are lost during + * the shared render domain reset, we'll define such workarounds + * here and then add them to just a single RCS or CCS engine's + * workaround list (whichever engine has the XXXX flag). + */ +static void +general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) +{ + struct drm_i915_private *i915 = engine->i915; + + if (IS_XEHPSDV(i915)) { + /* Wa_1409954639 */ + wa_masked_en(wal, + GEN8_ROW_CHICKEN, + SYSTOLIC_DOP_CLOCK_GATING_DIS); + + /* Wa_1607196519 */ + wa_masked_en(wal, + GEN9_ROW_CHICKEN4, + GEN12_DISABLE_GRF_CLEAR); + + /* Wa_14010670810:xehpsdv */ + wa_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE); + + /* Wa_14010449647:xehpsdv */ + wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, + GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE); + + /* Wa_18011725039:xehpsdv */ + if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) { + wa_masked_dis(wal, MLTICTXCTL, TDONRENDER); + wa_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH); + } + + /* Wa_14012362059:xehpsdv */ + wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB); + + /* Wa_14014368820:xehpsdv */ + wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS | + GLOBAL_INVALIDATION_MODE); + } +} + static void engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) { @@ -2563,6 +2628,14 @@ engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal engine_fake_wa_init(engine, wal); + /* + * These are common workarounds that just need to applied + * to a single RCS/CCS engine's workaround list since + * they're reset as part of the general render domain reset. + */ + if (engine->class == RENDER_CLASS) + general_render_compute_wa_init(engine, wal); + if (engine->class == RENDER_CLASS) rcs_engine_wa_init(engine, wal); else diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds_types.h b/drivers/gpu/drm/i915/gt/intel_workarounds_types.h index 1e873681795d..8a4b6de4e754 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds_types.h +++ b/drivers/gpu/drm/i915/gt/intel_workarounds_types.h @@ -8,7 +8,7 @@ #include <linux/types.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" struct i915_wa { i915_reg_t reg; diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c index 8af261831470..0dcb3ed44a73 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c @@ -6,6 +6,7 @@ #include <linux/sort.h> #include "i915_selftest.h" +#include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt_clock_utils.h" #include "selftest_engine.h" diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c index e10da897e07a..72d5faab8f9a 100644 --- a/drivers/gpu/drm/i915/gt/selftest_execlists.c +++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c @@ -5,6 +5,7 @@ #include <linux/prime_numbers.h> +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gt/intel_engine_heartbeat.h" #include "gt/intel_reset.h" diff --git a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c index 8bf62a5826cc..be94f863bdef 100644 --- a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c @@ -5,6 +5,7 @@ #include <linux/sort.h> +#include "intel_engine_regs.h" #include "intel_gt_clock_utils.h" #include "selftest_llc.h" diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c index 15d63435ec4d..83ff4c2e57c5 100644 --- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c +++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c @@ -6,7 +6,9 @@ #include <linux/kthread.h> #include "gem/i915_gem_context.h" +#include "gem/i915_gem_internal.h" +#include "i915_gem_evict.h" #include "intel_gt.h" #include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" @@ -1382,7 +1384,7 @@ static int evict_vma(void *data) complete(&arg->completion); mutex_lock(&vm->mutex); - err = i915_gem_evict_for_node(vm, &evict, 0); + err = i915_gem_evict_for_node(vm, NULL, &evict, 0); mutex_unlock(&vm->mutex); return err; diff --git a/drivers/gpu/drm/i915/gt/selftest_llc.c b/drivers/gpu/drm/i915/gt/selftest_llc.c index 459b775f163a..2cd184ab32b1 100644 --- a/drivers/gpu/drm/i915/gt/selftest_llc.c +++ b/drivers/gpu/drm/i915/gt/selftest_llc.c @@ -31,9 +31,8 @@ static int gen6_verify_ring_freq(struct intel_llc *llc) calc_ia_freq(llc, gpu_freq, &consts, &ia_freq, &ring_freq); val = gpu_freq; - if (sandybridge_pcode_read(i915, - GEN6_PCODE_READ_MIN_FREQ_TABLE, - &val, NULL)) { + if (snb_pcode_read(i915, GEN6_PCODE_READ_MIN_FREQ_TABLE, + &val, NULL)) { pr_err("Failed to read freq table[%d], range [%d, %d]\n", gpu_freq, consts.min_gpu_freq, consts.max_gpu_freq); err = -ENXIO; diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 618c905daa19..21c29d315cc0 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -5,6 +5,8 @@ #include <linux/prime_numbers.h> +#include "gem/i915_gem_internal.h" + #include "i915_selftest.h" #include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" diff --git a/drivers/gpu/drm/i915/gt/selftest_migrate.c b/drivers/gpu/drm/i915/gt/selftest_migrate.c index fa4293d2944f..c9c4f391c5cc 100644 --- a/drivers/gpu/drm/i915/gt/selftest_migrate.c +++ b/drivers/gpu/drm/i915/gt/selftest_migrate.c @@ -5,6 +5,8 @@ #include <linux/sort.h> +#include "gem/i915_gem_internal.h" + #include "selftests/i915_random.h" static const unsigned int sizes[] = { diff --git a/drivers/gpu/drm/i915/gt/selftest_reset.c b/drivers/gpu/drm/i915/gt/selftest_reset.c index 8a873f6bda7f..37c38bdd5f47 100644 --- a/drivers/gpu/drm/i915/gt/selftest_reset.c +++ b/drivers/gpu/drm/i915/gt/selftest_reset.c @@ -19,7 +19,7 @@ __igt_reset_stolen(struct intel_gt *gt, intel_engine_mask_t mask, const char *msg) { - struct i915_ggtt *ggtt = >->i915->ggtt; + struct i915_ggtt *ggtt = gt->ggtt; const struct resource *dsm = >->i915->dsm; resource_size_t num_pages, page; struct intel_engine_cs *engine; diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c index 7ee2513e15f9..6a69ac0184ad 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rps.c +++ b/drivers/gpu/drm/i915/gt/selftest_rps.c @@ -6,8 +6,11 @@ #include <linux/pm_qos.h> #include <linux/sort.h> +#include "gem/i915_gem_internal.h" + #include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" +#include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt_clock_utils.h" #include "intel_gt_pm.h" @@ -518,9 +521,8 @@ static void show_pcu_config(struct intel_rps *rps) for (gpu_freq = min_gpu_freq; gpu_freq <= max_gpu_freq; gpu_freq++) { int ia_freq = gpu_freq; - sandybridge_pcode_read(i915, - GEN6_PCODE_READ_MIN_FREQ_TABLE, - &ia_freq, NULL); + snb_pcode_read(i915, GEN6_PCODE_READ_MIN_FREQ_TABLE, + &ia_freq, NULL); pr_info("%5d %5d %5d\n", gpu_freq * 50, diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c index e2eb686a9763..0410c402f2a3 100644 --- a/drivers/gpu/drm/i915/gt/selftest_timeline.c +++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c @@ -8,6 +8,7 @@ #include "intel_context.h" #include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" +#include "intel_engine_regs.h" #include "intel_gpu_commands.h" #include "intel_gt.h" #include "intel_gt_requests.h" diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c index 0287c2573c51..67a9aab801dd 100644 --- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c +++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c @@ -3,6 +3,7 @@ * Copyright © 2018 Intel Corporation */ +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gt/intel_engine_user.h" #include "gt/intel_gt.h" diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c index 0683b27a3890..402f085f3a02 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.c +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -3,6 +3,7 @@ * Copyright © 2020 Intel Corporation */ +#include <linux/iosys-map.h> #include <linux/mm.h> #include <linux/pagemap.h> #include <linux/shmem_fs.h> @@ -123,6 +124,37 @@ static int __shmem_rw(struct file *file, loff_t off, return 0; } +int shmem_read_to_iosys_map(struct file *file, loff_t off, + struct iosys_map *map, size_t map_off, size_t len) +{ + unsigned long pfn; + + for (pfn = off >> PAGE_SHIFT; len; pfn++) { + unsigned int this = + min_t(size_t, PAGE_SIZE - offset_in_page(off), len); + struct page *page; + void *vaddr; + + page = shmem_read_mapping_page_gfp(file->f_mapping, pfn, + GFP_KERNEL); + if (IS_ERR(page)) + return PTR_ERR(page); + + vaddr = kmap(page); + iosys_map_memcpy_to(map, map_off, vaddr + offset_in_page(off), + this); + mark_page_accessed(page); + kunmap(page); + put_page(page); + + len -= this; + map_off += this; + off = 0; + } + + return 0; +} + int shmem_read(struct file *file, loff_t off, void *dst, size_t len) { return __shmem_rw(file, off, dst, len, false); diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.h b/drivers/gpu/drm/i915/gt/shmem_utils.h index c1669170c351..b2b04d88c6e5 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.h +++ b/drivers/gpu/drm/i915/gt/shmem_utils.h @@ -8,6 +8,7 @@ #include <linux/types.h> +struct iosys_map; struct drm_i915_gem_object; struct file; @@ -17,6 +18,8 @@ struct file *shmem_create_from_object(struct drm_i915_gem_object *obj); void *shmem_pin_map(struct file *file); void shmem_unpin_map(struct file *file, void *ptr); +int shmem_read_to_iosys_map(struct file *file, loff_t off, + struct iosys_map *map, size_t map_off, size_t len); int shmem_read(struct file *file, loff_t off, void *dst, size_t len); int shmem_write(struct file *file, loff_t off, void *src, size_t len); diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h index fe5d7d261797..7afdadc7656f 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h @@ -7,9 +7,9 @@ #define _ABI_GUC_ACTIONS_ABI_H /** - * DOC: HOST2GUC_REGISTER_CTB + * DOC: HOST2GUC_SELF_CFG * - * This message is used as part of the `CTB based communication`_ setup. + * This message is used by Host KMD to setup of the `GuC Self Config KLVs`_. * * This message must be sent as `MMIO HXG Message`_. * @@ -22,20 +22,18 @@ * | +-------+--------------------------------------------------------------+ * | | 27:16 | DATA0 = MBZ | * | +-------+--------------------------------------------------------------+ - * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_REGISTER_CTB` = 0x4505 | + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_SELF_CFG` = 0x0508 | * +---+-------+--------------------------------------------------------------+ - * | 1 | 31:12 | RESERVED = MBZ | + * | 1 | 31:16 | **KLV_KEY** - KLV key, see `GuC Self Config KLVs`_ | * | +-------+--------------------------------------------------------------+ - * | | 11:8 | **TYPE** - type for the `CT Buffer`_ | + * | | 15:0 | **KLV_LEN** - KLV length | * | | | | - * | | | - _`GUC_CTB_TYPE_HOST2GUC` = 0 | - * | | | - _`GUC_CTB_TYPE_GUC2HOST` = 1 | - * | +-------+--------------------------------------------------------------+ - * | | 7:0 | **SIZE** - size of the `CT Buffer`_ in 4K units minus 1 | + * | | | - 32 bit KLV = 1 | + * | | | - 64 bit KLV = 2 | * +---+-------+--------------------------------------------------------------+ - * | 2 | 31:0 | **DESC_ADDR** - GGTT address of the `CTB Descriptor`_ | + * | 2 | 31:0 | **VALUE32** - Bits 31-0 of the KLV value | * +---+-------+--------------------------------------------------------------+ - * | 3 | 31:0 | **BUFF_ADDF** - GGTT address of the `CT Buffer`_ | + * | 3 | 31:0 | **VALUE64** - Bits 63-32 of the KLV value (**KLV_LEN** = 2) | * +---+-------+--------------------------------------------------------------+ * * +---+-------+--------------------------------------------------------------+ @@ -45,28 +43,25 @@ * | +-------+--------------------------------------------------------------+ * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | * | +-------+--------------------------------------------------------------+ - * | | 27:0 | DATA0 = MBZ | + * | | 27:0 | DATA0 = **NUM** - 1 if KLV was parsed, 0 if not recognized | * +---+-------+--------------------------------------------------------------+ */ -#define GUC_ACTION_HOST2GUC_REGISTER_CTB 0x4505 +#define GUC_ACTION_HOST2GUC_SELF_CFG 0x0508 -#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 3u) -#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 -#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_MBZ (0xfffff << 12) -#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_TYPE (0xf << 8) -#define GUC_CTB_TYPE_HOST2GUC 0u -#define GUC_CTB_TYPE_GUC2HOST 1u -#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_SIZE (0xff << 0) -#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_2_DESC_ADDR GUC_HXG_REQUEST_MSG_n_DATAn -#define HOST2GUC_REGISTER_CTB_REQUEST_MSG_3_BUFF_ADDR GUC_HXG_REQUEST_MSG_n_DATAn +#define HOST2GUC_SELF_CFG_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 3u) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY (0xffff << 16) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN (0xffff << 0) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32 GUC_HXG_REQUEST_MSG_n_DATAn +#define HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64 GUC_HXG_REQUEST_MSG_n_DATAn -#define HOST2GUC_REGISTER_CTB_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN -#define HOST2GUC_REGISTER_CTB_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 +#define HOST2GUC_SELF_CFG_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define HOST2GUC_SELF_CFG_RESPONSE_MSG_0_NUM GUC_HXG_RESPONSE_MSG_0_DATA0 /** - * DOC: HOST2GUC_DEREGISTER_CTB + * DOC: HOST2GUC_CONTROL_CTB * - * This message is used as part of the `CTB based communication`_ teardown. + * This H2G action allows Vf Host to enable or disable H2G and G2H `CT Buffer`_. * * This message must be sent as `MMIO HXG Message`_. * @@ -79,15 +74,12 @@ * | +-------+--------------------------------------------------------------+ * | | 27:16 | DATA0 = MBZ | * | +-------+--------------------------------------------------------------+ - * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_DEREGISTER_CTB` = 0x4506 | + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_CONTROL_CTB` = 0x4509 | * +---+-------+--------------------------------------------------------------+ - * | 1 | 31:12 | RESERVED = MBZ | - * | +-------+--------------------------------------------------------------+ - * | | 11:8 | **TYPE** - type of the `CT Buffer`_ | + * | 1 | 31:0 | **CONTROL** - control `CTB based communication`_ | * | | | | - * | | | see `GUC_ACTION_HOST2GUC_REGISTER_CTB`_ | - * | +-------+--------------------------------------------------------------+ - * | | 7:0 | RESERVED = MBZ | + * | | | - _`GUC_CTB_CONTROL_DISABLE` = 0 | + * | | | - _`GUC_CTB_CONTROL_ENABLE` = 1 | * +---+-------+--------------------------------------------------------------+ * * +---+-------+--------------------------------------------------------------+ @@ -100,16 +92,16 @@ * | | 27:0 | DATA0 = MBZ | * +---+-------+--------------------------------------------------------------+ */ -#define GUC_ACTION_HOST2GUC_DEREGISTER_CTB 0x4506 +#define GUC_ACTION_HOST2GUC_CONTROL_CTB 0x4509 -#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) -#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 -#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_MBZ (0xfffff << 12) -#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_TYPE (0xf << 8) -#define HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_MBZ2 (0xff << 0) +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL GUC_HXG_REQUEST_MSG_n_DATAn +#define GUC_CTB_CONTROL_DISABLE 0u +#define GUC_CTB_CONTROL_ENABLE 1u -#define HOST2GUC_DEREGISTER_CTB_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN -#define HOST2GUC_DEREGISTER_CTB_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 +#define HOST2GUC_CONTROL_CTB_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define HOST2GUC_CONTROL_CTB_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 /* legacy definitions */ @@ -143,8 +135,12 @@ enum intel_guc_action { INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER = 0x4506, INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600, INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601, - INTEL_GUC_ACTION_RESET_CLIENT = 0x5507, + INTEL_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507, INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A, + INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002, + INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003, + INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004, + INTEL_GUC_ACTION_NOTIFY_EXCEPTION = 0x8005, INTEL_GUC_ACTION_LIMIT }; diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h index 7a8d4bfc5f6a..62cb4254a77a 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_slpc_abi.h @@ -7,7 +7,6 @@ #define _GUC_ACTIONS_SLPC_ABI_H_ #include <linux/types.h> -#include "i915_reg.h" /** * DOC: SLPC SHARED DATA STRUCTURE diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h index 488b6061ee89..c20658ee85a5 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_errors_abi.h @@ -11,4 +11,27 @@ enum intel_guc_response_status { INTEL_GUC_RESPONSE_STATUS_GENERIC_FAIL = 0xF000, }; +enum intel_guc_load_status { + INTEL_GUC_LOAD_STATUS_DEFAULT = 0x00, + INTEL_GUC_LOAD_STATUS_START = 0x01, + INTEL_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH = 0x02, + INTEL_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH = 0x03, + INTEL_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE = 0x04, + INTEL_GUC_LOAD_STATUS_GDT_DONE = 0x10, + INTEL_GUC_LOAD_STATUS_IDT_DONE = 0x20, + INTEL_GUC_LOAD_STATUS_LAPIC_DONE = 0x30, + INTEL_GUC_LOAD_STATUS_GUCINT_DONE = 0x40, + INTEL_GUC_LOAD_STATUS_DPC_READY = 0x50, + INTEL_GUC_LOAD_STATUS_DPC_ERROR = 0x60, + INTEL_GUC_LOAD_STATUS_EXCEPTION = 0x70, + INTEL_GUC_LOAD_STATUS_INIT_DATA_INVALID = 0x71, + INTEL_GUC_LOAD_STATUS_PXP_TEARDOWN_CTRL_ENABLED = 0x72, + INTEL_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_START, + INTEL_GUC_LOAD_STATUS_MPU_DATA_INVALID = 0x73, + INTEL_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID = 0x74, + INTEL_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_END, + + INTEL_GUC_LOAD_STATUS_READY = 0xF0, +}; + #endif /* _ABI_GUC_ERRORS_ABI_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_klvs_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_klvs_abi.h new file mode 100644 index 000000000000..f0814a57c191 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_klvs_abi.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _ABI_GUC_KLVS_ABI_H +#define _ABI_GUC_KLVS_ABI_H + +/** + * DOC: GuC KLV + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | **KEY** - KLV key identifier | + * | | | - `GuC Self Config KLVs`_ | + * | | | | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **LEN** - length of VALUE (in 32bit dwords) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **VALUE** - actual value of the KLV (format depends on KEY) | + * +---+-------+ | + * |...| | | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_KLV_LEN_MIN 1u +#define GUC_KLV_0_KEY (0xffff << 16) +#define GUC_KLV_0_LEN (0xffff << 0) +#define GUC_KLV_n_VALUE (0xffffffff << 0) + +/** + * DOC: GuC Self Config KLVs + * + * `GuC KLV`_ keys available for use with HOST2GUC_SELF_CFG_. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_ADDR` : 0x0902 + * Refers to 64 bit Global Gfx address of H2G `CT Buffer`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR` : 0x0903 + * Refers to 64 bit Global Gfx address of H2G `CTB Descriptor`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_SIZE` : 0x0904 + * Refers to size of H2G `CT Buffer`_ in bytes. + * Should be a multiple of 4K. + * + * _`GUC_KLV_SELF_CFG_G2H_CTB_ADDR` : 0x0905 + * Refers to 64 bit Global Gfx address of G2H `CT Buffer`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _`GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR` : 0x0906 + * Refers to 64 bit Global Gfx address of G2H `CTB Descriptor`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _`GUC_KLV_SELF_CFG_G2H_CTB_SIZE` : 0x0907 + * Refers to size of G2H `CT Buffer`_ in bytes. + * Should be a multiple of 4K. + */ + +#define GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY 0x0902 +#define GUC_KLV_SELF_CFG_H2G_CTB_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY 0x0903 +#define GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY 0x0904 +#define GUC_KLV_SELF_CFG_H2G_CTB_SIZE_LEN 1u + +#define GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY 0x0905 +#define GUC_KLV_SELF_CFG_G2H_CTB_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY 0x0906 +#define GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY 0x0907 +#define GUC_KLV_SELF_CFG_G2H_CTB_SIZE_LEN 1u + +#endif /* _ABI_GUC_KLVS_ABI_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index 6e228343e8cb..447a976c9f25 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -7,11 +7,13 @@ #include "gt/intel_gt.h" #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm_irq.h" +#include "gt/intel_gt_regs.h" #include "intel_guc.h" #include "intel_guc_slpc.h" #include "intel_guc_ads.h" #include "intel_guc_submission.h" #include "i915_drv.h" +#include "i915_irq.h" /** * DOC: GuC @@ -182,6 +184,9 @@ void intel_guc_init_early(struct intel_guc *guc) guc->send_regs.count = GUC_MAX_MMIO_MSG_LEN; BUILD_BUG_ON(GUC_MAX_MMIO_MSG_LEN > SOFT_SCRATCH_COUNT); } + + intel_guc_enable_msg(guc, INTEL_GUC_RECV_MSG_EXCEPTION | + INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED); } void intel_guc_init_late(struct intel_guc *guc) @@ -222,32 +227,48 @@ static u32 guc_ctl_log_params_flags(struct intel_guc *guc) u32 flags; #if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0) - #define UNIT SZ_1M - #define FLAG GUC_LOG_ALLOC_IN_MEGABYTE + #define LOG_UNIT SZ_1M + #define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS #else - #define UNIT SZ_4K - #define FLAG 0 + #define LOG_UNIT SZ_4K + #define LOG_FLAG 0 + #endif + + #if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0) + #define CAPTURE_UNIT SZ_1M + #define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS + #else + #define CAPTURE_UNIT SZ_4K + #define CAPTURE_FLAG 0 #endif BUILD_BUG_ON(!CRASH_BUFFER_SIZE); - BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, UNIT)); + BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT)); BUILD_BUG_ON(!DEBUG_BUFFER_SIZE); - BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, UNIT)); + BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT)); + BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT)); - BUILD_BUG_ON((CRASH_BUFFER_SIZE / UNIT - 1) > + BUILD_BUG_ON((CRASH_BUFFER_SIZE / LOG_UNIT - 1) > (GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT)); - BUILD_BUG_ON((DEBUG_BUFFER_SIZE / UNIT - 1) > + BUILD_BUG_ON((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) > (GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT)); + BUILD_BUG_ON((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) > + (GUC_LOG_CAPTURE_MASK >> GUC_LOG_CAPTURE_SHIFT)); flags = GUC_LOG_VALID | GUC_LOG_NOTIFY_ON_HALF_FULL | - FLAG | - ((CRASH_BUFFER_SIZE / UNIT - 1) << GUC_LOG_CRASH_SHIFT) | - ((DEBUG_BUFFER_SIZE / UNIT - 1) << GUC_LOG_DEBUG_SHIFT) | + CAPTURE_FLAG | + LOG_FLAG | + ((CRASH_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_CRASH_SHIFT) | + ((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_DEBUG_SHIFT) | + ((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) << GUC_LOG_CAPTURE_SHIFT) | (offset << GUC_LOG_BUF_ADDR_SHIFT); - #undef UNIT - #undef FLAG + #undef LOG_UNIT + #undef LOG_FLAG + #undef CAPTURE_UNIT + #undef CAPTURE_FLAG return flags; } @@ -260,6 +281,26 @@ static u32 guc_ctl_ads_flags(struct intel_guc *guc) return flags; } +static u32 guc_ctl_wa_flags(struct intel_guc *guc) +{ + struct intel_gt *gt = guc_to_gt(guc); + u32 flags = 0; + + /* Wa_22012773006:gen11,gen12 < XeHP */ + if (GRAPHICS_VER(gt->i915) >= 11 && + GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 50)) + flags |= GUC_WA_POLLCS; + + return flags; +} + +static u32 guc_ctl_devid(struct intel_guc *guc) +{ + struct drm_i915_private *i915 = guc_to_gt(guc)->i915; + + return (INTEL_DEVID(i915) << 16) | INTEL_REVID(i915); +} + /* * Initialise the GuC parameter block before starting the firmware * transfer. These parameters are read by the firmware on startup @@ -276,6 +317,8 @@ static void guc_init_params(struct intel_guc *guc) params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc); params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc); + params[GUC_CTL_WA] = guc_ctl_wa_flags(guc); + params[GUC_CTL_DEVID] = guc_ctl_devid(guc); for (i = 0; i < GUC_CTL_MAX_DWORDS; i++) DRM_DEBUG_DRIVER("param[%2d] = %#x\n", i, params[i]); @@ -513,9 +556,10 @@ int intel_guc_to_host_process_recv_msg(struct intel_guc *guc, /* Make sure to handle only enabled messages */ msg = payload[0] & guc->msg_enabled_mask; - if (msg & (INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER | - INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED)) - intel_guc_log_handle_flush_event(&guc->log); + if (msg & INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED) + drm_err(&guc_to_gt(guc)->i915->drm, "Received early GuC crash dump notification!\n"); + if (msg & INTEL_GUC_RECV_MSG_EXCEPTION) + drm_err(&guc_to_gt(guc)->i915->drm, "Received early GuC exception notification!\n"); return 0; } @@ -549,7 +593,7 @@ int intel_guc_suspend(struct intel_guc *guc) { int ret; u32 action[] = { - INTEL_GUC_ACTION_RESET_CLIENT, + INTEL_GUC_ACTION_CLIENT_SOFT_RESET, }; if (!intel_guc_is_ready(guc)) @@ -713,6 +757,56 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, return 0; } +static int __guc_action_self_cfg(struct intel_guc *guc, u16 key, u16 len, u64 value) +{ + u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_SELF_CFG), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) | + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32, lower_32_bits(value)), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64, upper_32_bits(value)), + }; + int ret; + + GEM_BUG_ON(len > 2); + GEM_BUG_ON(len == 1 && upper_32_bits(value)); + + /* Self config must go over MMIO */ + ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); + + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret > 1)) + return -EPROTO; + if (unlikely(!ret)) + return -ENOKEY; + + return 0; +} + +static int __guc_self_cfg(struct intel_guc *guc, u16 key, u16 len, u64 value) +{ + struct drm_i915_private *i915 = guc_to_gt(guc)->i915; + int err = __guc_action_self_cfg(guc, key, len, value); + + if (unlikely(err)) + i915_probe_error(i915, "Unsuccessful self-config (%pe) key %#hx value %#llx\n", + ERR_PTR(err), key, value); + return err; +} + +int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value) +{ + return __guc_self_cfg(guc, key, 1, value); +} + +int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value) +{ + return __guc_self_cfg(guc, key, 2, value); +} + /** * intel_guc_load_status - dump information about GuC load status * @guc: the GuC diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 3aabe164c329..bf7079480d47 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -6,8 +6,9 @@ #ifndef _INTEL_GUC_H_ #define _INTEL_GUC_H_ -#include <linux/xarray.h> #include <linux/delay.h> +#include <linux/iosys-map.h> +#include <linux/xarray.h> #include "intel_uncore.h" #include "intel_guc_fw.h" @@ -119,6 +120,15 @@ struct intel_guc { * function as it might be in an atomic context (no sleeping) */ struct work_struct destroyed_worker; + /** + * @reset_fail_worker: worker to trigger a GT reset after an + * engine reset fails + */ + struct work_struct reset_fail_worker; + /** + * @reset_fail_mask: mask of engines that failed to reset + */ + intel_engine_mask_t reset_fail_mask; } submission_state; /** @@ -137,10 +147,17 @@ struct intel_guc { /** @ads_vma: object allocated to hold the GuC ADS */ struct i915_vma *ads_vma; - /** @ads_blob: contents of the GuC ADS */ - struct __guc_ads_blob *ads_blob; + /** @ads_map: contents of the GuC ADS */ + struct iosys_map ads_map; /** @ads_regset_size: size of the save/restore regsets in the ADS */ u32 ads_regset_size; + /** + * @ads_regset_count: number of save/restore registers in the ADS for + * each engine + */ + u32 ads_regset_count[I915_NUM_ENGINES]; + /** @ads_regset: save/restore regsets in the ADS */ + struct guc_mmio_reg *ads_regset; /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */ u32 ads_golden_ctxt_size; /** @ads_engine_usage_size: size of engine usage in the ADS */ @@ -333,6 +350,8 @@ int intel_guc_resume(struct intel_guc *guc); struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size); int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, struct i915_vma **out_vma, void **out_vaddr); +int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value); +int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value); static inline bool intel_guc_is_supported(struct intel_guc *guc) { @@ -409,6 +428,8 @@ int intel_guc_context_reset_process_msg(struct intel_guc *guc, const u32 *msg, u32 len); int intel_guc_engine_failure_process_msg(struct intel_guc *guc, const u32 *msg, u32 len); +int intel_guc_error_capture_process_msg(struct intel_guc *guc, + const u32 *msg, u32 len); void intel_guc_find_hung_context(struct intel_engine_cs *engine); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c index 3acee0060e23..92cb88248391 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c @@ -5,7 +5,9 @@ #include <linux/bsearch.h> +#include "gt/intel_engine_regs.h" #include "gt/intel_gt.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_lrc.h" #include "gt/shmem_utils.h" #include "intel_guc_ads.h" @@ -40,6 +42,10 @@ * +---------------------------------------+ * | padding | * +---------------------------------------+ <== 4K aligned + * | capture lists | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned * | private data | * +---------------------------------------+ * | padding | @@ -54,6 +60,19 @@ struct __guc_ads_blob { struct guc_mmio_reg regset[]; } __packed; +#define ads_blob_read(guc_, field_) \ + iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_) + +#define ads_blob_write(guc_, field_, val_) \ + iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \ + field_, val_) + +#define info_map_write(map_, field_, val_) \ + iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_) + +#define info_map_read(map_, field_) \ + iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_) + static u32 guc_ads_regset_size(struct intel_guc *guc) { GEM_BUG_ON(!guc->ads_regset_size); @@ -65,6 +84,12 @@ static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc) return PAGE_ALIGN(guc->ads_golden_ctxt_size); } +static u32 guc_ads_capture_size(struct intel_guc *guc) +{ + /* FIXME: Allocate a proper capture list */ + return PAGE_ALIGN(PAGE_SIZE); +} + static u32 guc_ads_private_data_size(struct intel_guc *guc) { return PAGE_ALIGN(guc->fw.private_data_size); @@ -85,7 +110,7 @@ static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc) return PAGE_ALIGN(offset); } -static u32 guc_ads_private_data_offset(struct intel_guc *guc) +static u32 guc_ads_capture_offset(struct intel_guc *guc) { u32 offset; @@ -95,39 +120,53 @@ static u32 guc_ads_private_data_offset(struct intel_guc *guc) return PAGE_ALIGN(offset); } +static u32 guc_ads_private_data_offset(struct intel_guc *guc) +{ + u32 offset; + + offset = guc_ads_capture_offset(guc) + + guc_ads_capture_size(guc); + + return PAGE_ALIGN(offset); +} + static u32 guc_ads_blob_size(struct intel_guc *guc) { return guc_ads_private_data_offset(guc) + guc_ads_private_data_size(guc); } -static void guc_policies_init(struct intel_guc *guc, struct guc_policies *policies) +static void guc_policies_init(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); struct drm_i915_private *i915 = gt->i915; + u32 global_flags = 0; - policies->dpc_promote_time = GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US; - policies->max_num_work_items = GLOBAL_POLICY_MAX_NUM_WI; + ads_blob_write(guc, policies.dpc_promote_time, + GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); + ads_blob_write(guc, policies.max_num_work_items, + GLOBAL_POLICY_MAX_NUM_WI); - policies->global_flags = 0; if (i915->params.reset < 2) - policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; + global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; - policies->is_valid = 1; + ads_blob_write(guc, policies.global_flags, global_flags); + ads_blob_write(guc, policies.is_valid, 1); } void intel_guc_ads_print_policy_info(struct intel_guc *guc, struct drm_printer *dp) { - struct __guc_ads_blob *blob = guc->ads_blob; - - if (unlikely(!blob)) + if (unlikely(iosys_map_is_null(&guc->ads_map))) return; drm_printf(dp, "Global scheduling policies:\n"); - drm_printf(dp, " DPC promote time = %u\n", blob->policies.dpc_promote_time); - drm_printf(dp, " Max num work items = %u\n", blob->policies.max_num_work_items); - drm_printf(dp, " Flags = %u\n", blob->policies.global_flags); + drm_printf(dp, " DPC promote time = %u\n", + ads_blob_read(guc, policies.dpc_promote_time)); + drm_printf(dp, " Max num work items = %u\n", + ads_blob_read(guc, policies.max_num_work_items)); + drm_printf(dp, " Flags = %u\n", + ads_blob_read(guc, policies.global_flags)); } static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) @@ -142,29 +181,30 @@ static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) int intel_guc_global_policies_update(struct intel_guc *guc) { - struct __guc_ads_blob *blob = guc->ads_blob; struct intel_gt *gt = guc_to_gt(guc); + u32 scheduler_policies; intel_wakeref_t wakeref; int ret; - if (!blob) + if (iosys_map_is_null(&guc->ads_map)) return -EOPNOTSUPP; - GEM_BUG_ON(!blob->ads.scheduler_policies); + scheduler_policies = ads_blob_read(guc, ads.scheduler_policies); + GEM_BUG_ON(!scheduler_policies); - guc_policies_init(guc, &blob->policies); + guc_policies_init(guc); if (!intel_guc_is_ready(guc)) return 0; with_intel_runtime_pm(>->i915->runtime_pm, wakeref) - ret = guc_action_policies_update(guc, blob->ads.scheduler_policies); + ret = guc_action_policies_update(guc, scheduler_policies); return ret; } static void guc_mapping_table_init(struct intel_gt *gt, - struct guc_gt_system_info *system_info) + struct iosys_map *info_map) { unsigned int i, j; struct intel_engine_cs *engine; @@ -173,27 +213,31 @@ static void guc_mapping_table_init(struct intel_gt *gt, /* Table must be set to invalid values for entries not used */ for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) - system_info->mapping_table[i][j] = - GUC_MAX_INSTANCES_PER_CLASS; + info_map_write(info_map, mapping_table[i][j], + GUC_MAX_INSTANCES_PER_CLASS); for_each_engine(engine, gt, id) { u8 guc_class = engine_class_to_guc_class(engine->class); - system_info->mapping_table[guc_class][ilog2(engine->logical_mask)] = - engine->instance; + info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)], + engine->instance); } } /* * The save/restore register list must be pre-calculated to a temporary - * buffer of driver defined size before it can be generated in place - * inside the ADS. + * buffer before it can be copied inside the ADS. */ -#define MAX_MMIO_REGS 128 /* Arbitrary size, increase as needed */ struct temp_regset { + /* + * ptr to the section of the storage for the engine currently being + * worked on + */ struct guc_mmio_reg *registers; - u32 used; - u32 size; + /* ptr to the base of the allocated storage for all engines */ + struct guc_mmio_reg *storage; + u32 storage_used; + u32 storage_max; }; static int guc_mmio_reg_cmp(const void *a, const void *b) @@ -204,18 +248,44 @@ static int guc_mmio_reg_cmp(const void *a, const void *b) return (int)ra->offset - (int)rb->offset; } -static void guc_mmio_reg_add(struct temp_regset *regset, - u32 offset, u32 flags) +static struct guc_mmio_reg * __must_check +__mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg) +{ + u32 pos = regset->storage_used; + struct guc_mmio_reg *slot; + + if (pos >= regset->storage_max) { + size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE); + struct guc_mmio_reg *r = krealloc(regset->storage, + size, GFP_KERNEL); + if (!r) { + WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n", + -ENOMEM); + return ERR_PTR(-ENOMEM); + } + + regset->registers = r + (regset->registers - regset->storage); + regset->storage = r; + regset->storage_max = size / sizeof(*slot); + } + + slot = ®set->storage[pos]; + regset->storage_used++; + *slot = *reg; + + return slot; +} + +static long __must_check guc_mmio_reg_add(struct temp_regset *regset, + u32 offset, u32 flags) { - u32 count = regset->used; + u32 count = regset->storage_used - (regset->registers - regset->storage); struct guc_mmio_reg reg = { .offset = offset, .flags = flags, }; struct guc_mmio_reg *slot; - GEM_BUG_ON(count >= regset->size); - /* * The mmio list is built using separate lists within the driver. * It's possible that at some point we may attempt to add the same @@ -224,11 +294,11 @@ static void guc_mmio_reg_add(struct temp_regset *regset, */ if (bsearch(®, regset->registers, count, sizeof(reg), guc_mmio_reg_cmp)) - return; + return 0; - slot = ®set->registers[count]; - regset->used++; - *slot = reg; + slot = __mmio_reg_add(regset, ®); + if (IS_ERR(slot)) + return PTR_ERR(slot); while (slot-- > regset->registers) { GEM_BUG_ON(slot[0].offset == slot[1].offset); @@ -237,6 +307,8 @@ static void guc_mmio_reg_add(struct temp_regset *regset, swap(slot[1], slot[0]); } + + return 0; } #define GUC_MMIO_REG_ADD(regset, reg, masked) \ @@ -244,124 +316,140 @@ static void guc_mmio_reg_add(struct temp_regset *regset, i915_mmio_reg_offset((reg)), \ (masked) ? GUC_REGSET_MASKED : 0) -static void guc_mmio_regset_init(struct temp_regset *regset, - struct intel_engine_cs *engine) +static int guc_mmio_regset_init(struct temp_regset *regset, + struct intel_engine_cs *engine) { const u32 base = engine->mmio_base; struct i915_wa_list *wal = &engine->wa_list; struct i915_wa *wa; unsigned int i; + int ret = 0; - regset->used = 0; + /* + * Each engine's registers point to a new start relative to + * storage + */ + regset->registers = regset->storage + regset->storage_used; - GUC_MMIO_REG_ADD(regset, RING_MODE_GEN7(base), true); - GUC_MMIO_REG_ADD(regset, RING_HWS_PGA(base), false); - GUC_MMIO_REG_ADD(regset, RING_IMR(base), false); + ret |= GUC_MMIO_REG_ADD(regset, RING_MODE_GEN7(base), true); + ret |= GUC_MMIO_REG_ADD(regset, RING_HWS_PGA(base), false); + ret |= GUC_MMIO_REG_ADD(regset, RING_IMR(base), false); + + if (engine->class == RENDER_CLASS && + CCS_MASK(engine->gt)) + ret |= GUC_MMIO_REG_ADD(regset, GEN12_RCU_MODE, true); for (i = 0, wa = wal->list; i < wal->count; i++, wa++) - GUC_MMIO_REG_ADD(regset, wa->reg, wa->masked_reg); + ret |= GUC_MMIO_REG_ADD(regset, wa->reg, wa->masked_reg); /* Be extra paranoid and include all whitelist registers. */ for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) - GUC_MMIO_REG_ADD(regset, - RING_FORCE_TO_NONPRIV(base, i), - false); + ret |= GUC_MMIO_REG_ADD(regset, + RING_FORCE_TO_NONPRIV(base, i), + false); /* add in local MOCS registers */ for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++) - GUC_MMIO_REG_ADD(regset, GEN9_LNCFCMOCS(i), false); + ret |= GUC_MMIO_REG_ADD(regset, GEN9_LNCFCMOCS(i), false); + + return ret ? -1 : 0; } -static int guc_mmio_reg_state_query(struct intel_guc *guc) +static long guc_mmio_reg_state_create(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); struct intel_engine_cs *engine; enum intel_engine_id id; - struct temp_regset temp_set; - u32 total; + struct temp_regset temp_set = {}; + long total = 0; + long ret; - /* - * Need to actually build the list in order to filter out - * duplicates and other such data dependent constructions. - */ - temp_set.size = MAX_MMIO_REGS; - temp_set.registers = kmalloc_array(temp_set.size, - sizeof(*temp_set.registers), - GFP_KERNEL); - if (!temp_set.registers) - return -ENOMEM; - - total = 0; for_each_engine(engine, gt, id) { - guc_mmio_regset_init(&temp_set, engine); - total += temp_set.used; + u32 used = temp_set.storage_used; + + ret = guc_mmio_regset_init(&temp_set, engine); + if (ret < 0) + goto fail_regset_init; + + guc->ads_regset_count[id] = temp_set.storage_used - used; + total += guc->ads_regset_count[id]; } - kfree(temp_set.registers); + guc->ads_regset = temp_set.storage; + + drm_dbg(&guc_to_gt(guc)->i915->drm, "Used %zu KB for temporary ADS regset\n", + (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10); return total * sizeof(struct guc_mmio_reg); + +fail_regset_init: + kfree(temp_set.storage); + return ret; } -static void guc_mmio_reg_state_init(struct intel_guc *guc, - struct __guc_ads_blob *blob) +static void guc_mmio_reg_state_init(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); struct intel_engine_cs *engine; enum intel_engine_id id; - struct temp_regset temp_set; - struct guc_mmio_reg_set *ads_reg_set; u32 addr_ggtt, offset; - u8 guc_class; offset = guc_ads_regset_offset(guc); addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; - temp_set.registers = (struct guc_mmio_reg *)(((u8 *)blob) + offset); - temp_set.size = guc->ads_regset_size / sizeof(temp_set.registers[0]); + + iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset, + guc->ads_regset_size); for_each_engine(engine, gt, id) { + u32 count = guc->ads_regset_count[id]; + u8 guc_class; + /* Class index is checked in class converter */ GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); guc_class = engine_class_to_guc_class(engine->class); - ads_reg_set = &blob->ads.reg_state_list[guc_class][engine->instance]; - guc_mmio_regset_init(&temp_set, engine); - if (!temp_set.used) { - ads_reg_set->address = 0; - ads_reg_set->count = 0; + if (!count) { + ads_blob_write(guc, + ads.reg_state_list[guc_class][engine->instance].address, + 0); + ads_blob_write(guc, + ads.reg_state_list[guc_class][engine->instance].count, + 0); continue; } - ads_reg_set->address = addr_ggtt; - ads_reg_set->count = temp_set.used; + ads_blob_write(guc, + ads.reg_state_list[guc_class][engine->instance].address, + addr_ggtt); + ads_blob_write(guc, + ads.reg_state_list[guc_class][engine->instance].count, + count); - temp_set.size -= temp_set.used; - temp_set.registers += temp_set.used; - addr_ggtt += temp_set.used * sizeof(struct guc_mmio_reg); + addr_ggtt += count * sizeof(struct guc_mmio_reg); } - - GEM_BUG_ON(temp_set.size); } static void fill_engine_enable_masks(struct intel_gt *gt, - struct guc_gt_system_info *info) + struct iosys_map *info_map) { - info->engine_enabled_masks[GUC_RENDER_CLASS] = 1; - info->engine_enabled_masks[GUC_BLITTER_CLASS] = 1; - info->engine_enabled_masks[GUC_VIDEO_CLASS] = VDBOX_MASK(gt); - info->engine_enabled_masks[GUC_VIDEOENHANCE_CLASS] = VEBOX_MASK(gt); + info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], 1); + info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt)); + info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], 1); + info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt)); + info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt)); } #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) #define LRC_SKIP_SIZE (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE) -static int guc_prep_golden_context(struct intel_guc *guc, - struct __guc_ads_blob *blob) +static int guc_prep_golden_context(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); u32 addr_ggtt, offset; u32 total_size = 0, alloc_size, real_size; u8 engine_class, guc_class; - struct guc_gt_system_info *info, local_info; + struct guc_gt_system_info local_info; + struct iosys_map info_map; /* * Reserve the memory for the golden contexts and point GuC at it but @@ -375,14 +463,15 @@ static int guc_prep_golden_context(struct intel_guc *guc, * GuC will also validate that the LRC base + size fall within the * allowed GGTT range. */ - if (blob) { + if (!iosys_map_is_null(&guc->ads_map)) { offset = guc_ads_golden_ctxt_offset(guc); addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; - info = &blob->system_info; + info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, + offsetof(struct __guc_ads_blob, system_info)); } else { memset(&local_info, 0, sizeof(local_info)); - info = &local_info; - fill_engine_enable_masks(gt, info); + iosys_map_set_vaddr(&info_map, &local_info); + fill_engine_enable_masks(gt, &info_map); } for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { @@ -391,14 +480,14 @@ static int guc_prep_golden_context(struct intel_guc *guc, guc_class = engine_class_to_guc_class(engine_class); - if (!info->engine_enabled_masks[guc_class]) + if (!info_map_read(&info_map, engine_enabled_masks[guc_class])) continue; real_size = intel_engine_context_size(gt, engine_class); alloc_size = PAGE_ALIGN(real_size); total_size += alloc_size; - if (!blob) + if (iosys_map_is_null(&guc->ads_map)) continue; /* @@ -412,15 +501,18 @@ static int guc_prep_golden_context(struct intel_guc *guc, * what comes before it in the context image (which is identical * on all engines). */ - blob->ads.eng_state_size[guc_class] = real_size - LRC_SKIP_SIZE; - blob->ads.golden_context_lrca[guc_class] = addr_ggtt; + ads_blob_write(guc, ads.eng_state_size[guc_class], + real_size - LRC_SKIP_SIZE); + ads_blob_write(guc, ads.golden_context_lrca[guc_class], + addr_ggtt); + addr_ggtt += alloc_size; } - if (!blob) - return total_size; + /* Make sure current size matches what we calculated previously */ + if (guc->ads_golden_ctxt_size) + GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); - GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); return total_size; } @@ -444,18 +536,16 @@ static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_ static void guc_init_golden_context(struct intel_guc *guc) { - struct __guc_ads_blob *blob = guc->ads_blob; struct intel_engine_cs *engine; struct intel_gt *gt = guc_to_gt(guc); - u32 addr_ggtt, offset; - u32 total_size = 0, alloc_size, real_size; + unsigned long offset; + u32 addr_ggtt, total_size = 0, alloc_size, real_size; u8 engine_class, guc_class; - u8 *ptr; if (!intel_uc_uses_guc_submission(>->uc)) return; - GEM_BUG_ON(!blob); + GEM_BUG_ON(iosys_map_is_null(&guc->ads_map)); /* * Go back and fill in the golden context data now that it is @@ -463,15 +553,13 @@ static void guc_init_golden_context(struct intel_guc *guc) */ offset = guc_ads_golden_ctxt_offset(guc); addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; - ptr = ((u8 *)blob) + offset; for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { if (engine_class == OTHER_CLASS) continue; guc_class = engine_class_to_guc_class(engine_class); - - if (!blob->system_info.engine_enabled_masks[guc_class]) + if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class])) continue; real_size = intel_engine_context_size(gt, engine_class); @@ -482,65 +570,95 @@ static void guc_init_golden_context(struct intel_guc *guc) if (!engine) { drm_err(>->i915->drm, "No engine state recorded for class %d!\n", engine_class); - blob->ads.eng_state_size[guc_class] = 0; - blob->ads.golden_context_lrca[guc_class] = 0; + ads_blob_write(guc, ads.eng_state_size[guc_class], 0); + ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0); continue; } - GEM_BUG_ON(blob->ads.eng_state_size[guc_class] != + GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) != real_size - LRC_SKIP_SIZE); - GEM_BUG_ON(blob->ads.golden_context_lrca[guc_class] != addr_ggtt); + GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt); + addr_ggtt += alloc_size; - shmem_read(engine->default_state, 0, ptr, real_size); - ptr += alloc_size; + shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map, + offset, real_size); + offset += alloc_size; } GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); } +static void guc_capture_list_init(struct intel_guc *guc) +{ + int i, j; + u32 addr_ggtt, offset; + + offset = guc_ads_capture_offset(guc); + addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; + + /* FIXME: Populate a proper capture list */ + + for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { + for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) { + ads_blob_write(guc, ads.capture_instance[i][j], addr_ggtt); + ads_blob_write(guc, ads.capture_class[i][j], addr_ggtt); + } + + ads_blob_write(guc, ads.capture_global[i], addr_ggtt); + } +} + static void __guc_ads_init(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); struct drm_i915_private *i915 = gt->i915; - struct __guc_ads_blob *blob = guc->ads_blob; + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, + offsetof(struct __guc_ads_blob, system_info)); u32 base; /* GuC scheduling policies */ - guc_policies_init(guc, &blob->policies); + guc_policies_init(guc); /* System info */ - fill_engine_enable_masks(gt, &blob->system_info); + fill_engine_enable_masks(gt, &info_map); - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = - hweight8(gt->info.sseu.slice_mask); - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] = - gt->info.vdbox_sfc_access; + ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED], + hweight8(gt->info.sseu.slice_mask)); + ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK], + gt->info.vdbox_sfc_access); if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) { u32 distdbreg = intel_uncore_read(gt->uncore, GEN12_DIST_DBS_POPULATED); - blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] = - ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) & - GEN12_DOORBELLS_PER_SQIDI) + 1; + ads_blob_write(guc, + system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], + ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) + & GEN12_DOORBELLS_PER_SQIDI) + 1); } /* Golden contexts for re-initialising after a watchdog reset */ - guc_prep_golden_context(guc, blob); + guc_prep_golden_context(guc); - guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); + guc_mapping_table_init(guc_to_gt(guc), &info_map); base = intel_guc_ggtt_offset(guc, guc->ads_vma); + /* Capture list for hang debug */ + guc_capture_list_init(guc); + /* ADS */ - blob->ads.scheduler_policies = base + ptr_offset(blob, policies); - blob->ads.gt_system_info = base + ptr_offset(blob, system_info); + ads_blob_write(guc, ads.scheduler_policies, base + + offsetof(struct __guc_ads_blob, policies)); + ads_blob_write(guc, ads.gt_system_info, base + + offsetof(struct __guc_ads_blob, system_info)); /* MMIO save/restore list */ - guc_mmio_reg_state_init(guc, blob); + guc_mmio_reg_state_init(guc); /* Private Data */ - blob->ads.private_data = base + guc_ads_private_data_offset(guc); + ads_blob_write(guc, ads.private_data, base + + guc_ads_private_data_offset(guc)); i915_gem_object_flush_map(guc->ads_vma->obj); } @@ -554,19 +672,23 @@ static void __guc_ads_init(struct intel_guc *guc) */ int intel_guc_ads_create(struct intel_guc *guc) { + void *ads_blob; u32 size; int ret; GEM_BUG_ON(guc->ads_vma); - /* Need to calculate the reg state size dynamically: */ - ret = guc_mmio_reg_state_query(guc); + /* + * Create reg state size dynamically on system memory to be copied to + * the final ads blob on gt init/reset + */ + ret = guc_mmio_reg_state_create(guc); if (ret < 0) return ret; guc->ads_regset_size = ret; /* Likewise the golden contexts: */ - ret = guc_prep_golden_context(guc, NULL); + ret = guc_prep_golden_context(guc); if (ret < 0) return ret; guc->ads_golden_ctxt_size = ret; @@ -575,10 +697,15 @@ int intel_guc_ads_create(struct intel_guc *guc) size = guc_ads_blob_size(guc); ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, - (void **)&guc->ads_blob); + &ads_blob); if (ret) return ret; + if (i915_gem_object_is_lmem(guc->ads_vma->obj)) + iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob); + else + iosys_map_set_vaddr(&guc->ads_map, ads_blob); + __guc_ads_init(guc); return 0; @@ -599,7 +726,8 @@ void intel_guc_ads_init_late(struct intel_guc *guc) void intel_guc_ads_destroy(struct intel_guc *guc) { i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); - guc->ads_blob = NULL; + iosys_map_clear(&guc->ads_map); + kfree(guc->ads_regset); } static void guc_ads_private_data_reset(struct intel_guc *guc) @@ -610,8 +738,8 @@ static void guc_ads_private_data_reset(struct intel_guc *guc) if (!size) return; - memset((void *)guc->ads_blob + guc_ads_private_data_offset(guc), 0, - size); + iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc), + 0, size); } /** @@ -634,18 +762,16 @@ void intel_guc_ads_reset(struct intel_guc *guc) u32 intel_guc_engine_usage_offset(struct intel_guc *guc) { - struct __guc_ads_blob *blob = guc->ads_blob; - u32 base = intel_guc_ggtt_offset(guc, guc->ads_vma); - u32 offset = base + ptr_offset(blob, engine_usage); - - return offset; + return intel_guc_ggtt_offset(guc, guc->ads_vma) + + offsetof(struct __guc_ads_blob, engine_usage); } -struct guc_engine_usage_record *intel_guc_engine_usage(struct intel_engine_cs *engine) +struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine) { struct intel_guc *guc = &engine->gt->uc.guc; - struct __guc_ads_blob *blob = guc->ads_blob; u8 guc_class = engine_class_to_guc_class(engine->class); + size_t offset = offsetof(struct __guc_ads_blob, + engine_usage.engines[guc_class][ilog2(engine->logical_mask)]); - return &blob->engine_usage.engines[guc_class][ilog2(engine->logical_mask)]; + return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset); } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h index e74c110facff..1c64f4d6ea21 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.h @@ -7,6 +7,7 @@ #define _INTEL_GUC_ADS_H_ #include <linux/types.h> +#include <linux/iosys-map.h> struct intel_guc; struct drm_printer; @@ -18,7 +19,7 @@ void intel_guc_ads_init_late(struct intel_guc *guc); void intel_guc_ads_reset(struct intel_guc *guc); void intel_guc_ads_print_policy_info(struct intel_guc *guc, struct drm_printer *p); -struct guc_engine_usage_record *intel_guc_engine_usage(struct intel_engine_cs *engine); +struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine); u32 intel_guc_engine_usage_offset(struct intel_guc *guc); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c index aa6dd6415202..2f7fc87a78e1 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c @@ -112,18 +112,6 @@ void intel_guc_ct_init_early(struct intel_guc_ct *ct) init_waitqueue_head(&ct->wq); } -static inline const char *guc_ct_buffer_type_to_str(u32 type) -{ - switch (type) { - case GUC_CTB_TYPE_HOST2GUC: - return "SEND"; - case GUC_CTB_TYPE_GUC2HOST: - return "RECV"; - default: - return "<invalid>"; - } -} - static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc) { memset(desc, 0, sizeof(*desc)); @@ -156,71 +144,65 @@ static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb, guc_ct_buffer_reset(ctb); } -static int guc_action_register_ct_buffer(struct intel_guc *guc, u32 type, - u32 desc_addr, u32 buff_addr, u32 size) +static int guc_action_control_ctb(struct intel_guc *guc, u32 control) { - u32 request[HOST2GUC_REGISTER_CTB_REQUEST_MSG_LEN] = { + u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | - FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_REGISTER_CTB), - FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_SIZE, size / SZ_4K - 1) | - FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_TYPE, type), - FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_2_DESC_ADDR, desc_addr), - FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_3_BUFF_ADDR, buff_addr), + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB), + FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control), }; int ret; - GEM_BUG_ON(type != GUC_CTB_TYPE_HOST2GUC && type != GUC_CTB_TYPE_GUC2HOST); - GEM_BUG_ON(size % SZ_4K); + GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE); - /* CT registration must go over MMIO */ + /* CT control must go over MMIO */ ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); return ret > 0 ? -EPROTO : ret; } -static int ct_register_buffer(struct intel_guc_ct *ct, u32 type, - u32 desc_addr, u32 buff_addr, u32 size) +static int ct_control_enable(struct intel_guc_ct *ct, bool enable) { int err; - err = i915_inject_probe_error(guc_to_gt(ct_to_guc(ct))->i915, -ENXIO); + err = guc_action_control_ctb(ct_to_guc(ct), enable ? + GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE); if (unlikely(err)) - return err; + CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n", + enabledisable(enable), ERR_PTR(err)); - err = guc_action_register_ct_buffer(ct_to_guc(ct), type, - desc_addr, buff_addr, size); - if (unlikely(err)) - CT_ERROR(ct, "Failed to register %s buffer (%pe)\n", - guc_ct_buffer_type_to_str(type), ERR_PTR(err)); return err; } -static int guc_action_deregister_ct_buffer(struct intel_guc *guc, u32 type) +static int ct_register_buffer(struct intel_guc_ct *ct, bool send, + u32 desc_addr, u32 buff_addr, u32 size) { - u32 request[HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_LEN] = { - FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | - FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | - FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_DEREGISTER_CTB), - FIELD_PREP(HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_TYPE, type), - }; - int ret; - - GEM_BUG_ON(type != GUC_CTB_TYPE_HOST2GUC && type != GUC_CTB_TYPE_GUC2HOST); - - /* CT deregistration must go over MMIO */ - ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); + int err; - return ret > 0 ? -EPROTO : ret; -} + err = intel_guc_self_cfg64(ct_to_guc(ct), send ? + GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY : + GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, + desc_addr); + if (unlikely(err)) + goto failed; -static int ct_deregister_buffer(struct intel_guc_ct *ct, u32 type) -{ - int err = guc_action_deregister_ct_buffer(ct_to_guc(ct), type); + err = intel_guc_self_cfg64(ct_to_guc(ct), send ? + GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY : + GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, + buff_addr); + if (unlikely(err)) + goto failed; + err = intel_guc_self_cfg32(ct_to_guc(ct), send ? + GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY : + GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, + size); if (unlikely(err)) - CT_ERROR(ct, "Failed to deregister %s buffer (%pe)\n", - guc_ct_buffer_type_to_str(type), ERR_PTR(err)); +failed: + CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n", + send ? "SEND" : "RECV", ERR_PTR(err)); + return err; } @@ -308,7 +290,7 @@ void intel_guc_ct_fini(struct intel_guc_ct *ct) int intel_guc_ct_enable(struct intel_guc_ct *ct) { struct intel_guc *guc = ct_to_guc(ct); - u32 base, desc, cmds; + u32 base, desc, cmds, size; void *blob; int err; @@ -333,27 +315,27 @@ int intel_guc_ct_enable(struct intel_guc_ct *ct) */ desc = base + ptrdiff(ct->ctbs.recv.desc, blob); cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob); - err = ct_register_buffer(ct, GUC_CTB_TYPE_GUC2HOST, - desc, cmds, ct->ctbs.recv.size * 4); - + size = ct->ctbs.recv.size * 4; + err = ct_register_buffer(ct, false, desc, cmds, size); if (unlikely(err)) goto err_out; desc = base + ptrdiff(ct->ctbs.send.desc, blob); cmds = base + ptrdiff(ct->ctbs.send.cmds, blob); - err = ct_register_buffer(ct, GUC_CTB_TYPE_HOST2GUC, - desc, cmds, ct->ctbs.send.size * 4); + size = ct->ctbs.send.size * 4; + err = ct_register_buffer(ct, true, desc, cmds, size); + if (unlikely(err)) + goto err_out; + err = ct_control_enable(ct, true); if (unlikely(err)) - goto err_deregister; + goto err_out; ct->enabled = true; ct->stall_time = KTIME_MAX; return 0; -err_deregister: - ct_deregister_buffer(ct, GUC_CTB_TYPE_GUC2HOST); err_out: CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err)); return err; @@ -372,8 +354,7 @@ void intel_guc_ct_disable(struct intel_guc_ct *ct) ct->enabled = false; if (intel_guc_is_fw_running(guc)) { - ct_deregister_buffer(ct, GUC_CTB_TYPE_HOST2GUC); - ct_deregister_buffer(ct, GUC_CTB_TYPE_GUC2HOST); + ct_control_enable(ct, false); } } @@ -662,6 +643,7 @@ static int ct_send(struct intel_guc_ct *ct, struct ct_request request; unsigned long flags; unsigned int sleep_period_ms = 1; + bool send_again; u32 fence; int err; @@ -671,6 +653,9 @@ static int ct_send(struct intel_guc_ct *ct, GEM_BUG_ON(!response_buf && response_buf_size); might_sleep(); +resend: + send_again = false; + /* * We use a lazy spin wait loop here as we believe that if the CT * buffers are sized correctly the flow control condition should be @@ -725,6 +710,13 @@ retry: goto unlink; } + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { + CT_DEBUG(ct, "retrying request %#x (%u)\n", *action, + FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status)); + send_again = true; + goto unlink; + } + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { err = -EIO; goto unlink; @@ -747,6 +739,9 @@ unlink: list_del(&request.link); spin_unlock_irqrestore(&ct->requests.lock, flags); + if (unlikely(send_again)) + goto resend; + return err; } @@ -789,7 +784,7 @@ static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords) { struct ct_incoming_msg *msg; - msg = kmalloc(sizeof(*msg) + sizeof(u32) * num_dwords, GFP_ATOMIC); + msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC); if (msg) msg->size = num_dwords; return msg; @@ -918,6 +913,7 @@ static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *r GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN); GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC); GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS && + FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY && FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE); CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]); @@ -990,9 +986,27 @@ static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *r case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: ret = intel_guc_context_reset_process_msg(guc, payload, len); break; + case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: + ret = intel_guc_error_capture_process_msg(guc, payload, len); + if (unlikely(ret)) + CT_ERROR(ct, "error capture notification failed %x %*ph\n", + action, 4 * len, payload); + break; case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: ret = intel_guc_engine_failure_process_msg(guc, payload, len); break; + case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: + intel_guc_log_handle_flush_event(&guc->log); + ret = 0; + break; + case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: + CT_ERROR(ct, "Received GuC crash dump notification!\n"); + ret = 0; + break; + case INTEL_GUC_ACTION_NOTIFY_EXCEPTION: + CT_ERROR(ct, "Received GuC exception notification!\n"); + ret = 0; + break; default: ret = -EOPNOTSUPP; break; @@ -1098,6 +1112,7 @@ static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) break; case GUC_HXG_TYPE_RESPONSE_SUCCESS: case GUC_HXG_TYPE_RESPONSE_FAILURE: + case GUC_HXG_TYPE_NO_RESPONSE_RETRY: err = ct_handle_response(ct, msg); break; default: diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c index 31420ce1ce6b..a0372735cddb 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c @@ -10,18 +10,21 @@ */ #include "gt/intel_gt.h" +#include "gt/intel_gt_regs.h" #include "intel_guc_fw.h" #include "i915_drv.h" static void guc_prepare_xfer(struct intel_uncore *uncore) { - u32 shim_flags = GUC_DISABLE_SRAM_INIT_TO_ZEROES | - GUC_ENABLE_READ_CACHE_LOGIC | - GUC_ENABLE_MIA_CACHING | + u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC | GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA | GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA | GUC_ENABLE_MIA_CLOCK_GATING; + if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 50)) + shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES | + GUC_ENABLE_MIA_CACHING; + /* Must program this register before loading the ucode with DMA */ intel_uncore_write(uncore, GUC_SHIM_CONTROL, shim_flags); @@ -90,11 +93,10 @@ static int guc_xfer_rsa(struct intel_uc_fw *guc_fw, static inline bool guc_ready(struct intel_uncore *uncore, u32 *status) { u32 val = intel_uncore_read(uncore, GUC_STATUS); - u32 uk_val = val & GS_UKERNEL_MASK; + u32 uk_val = REG_FIELD_GET(GS_UKERNEL_MASK, val); *status = val; - return (uk_val == GS_UKERNEL_READY) || - ((val & GS_MIA_CORE_STATE) && (uk_val == GS_UKERNEL_LAPIC_DONE)); + return uk_val == INTEL_GUC_LOAD_STATUS_READY; } static int guc_wait_ucode(struct intel_uncore *uncore) @@ -105,17 +107,26 @@ static int guc_wait_ucode(struct intel_uncore *uncore) /* * Wait for the GuC to start up. * NB: Docs recommend not using the interrupt for completion. - * Measurements indicate this should take no more than 20ms, so a + * Measurements indicate this should take no more than 20ms + * (assuming the GT clock is at maximum frequency). So, a * timeout here indicates that the GuC has failed and is unusable. * (Higher levels of the driver may decide to reset the GuC and * attempt the ucode load again if this happens.) + * + * FIXME: There is a known (but exceedingly unlikely) race condition + * where the asynchronous frequency management code could reduce + * the GT clock while a GuC reload is in progress (during a full + * GT reset). A fix is in progress but there are complex locking + * issues to be resolved. In the meantime bump the timeout to + * 200ms. Even at slowest clock, this should be sufficient. And + * in the working case, a larger timeout makes no difference. */ - ret = wait_for(guc_ready(uncore, &status), 100); + ret = wait_for(guc_ready(uncore, &status), 200); if (ret) { struct drm_device *drm = &uncore->i915->drm; - drm_dbg(drm, "GuC load failed: status = 0x%08X\n", status); - drm_dbg(drm, "GuC load failed: status: Reset = %d, " + drm_info(drm, "GuC load failed: status = 0x%08X\n", status); + drm_info(drm, "GuC load failed: status: Reset = %d, " "BootROM = 0x%02X, UKernel = 0x%02X, " "MIA = 0x%02X, Auth = 0x%02X\n", REG_FIELD_GET(GS_MIA_IN_RESET, status), @@ -125,13 +136,13 @@ static int guc_wait_ucode(struct intel_uncore *uncore) REG_FIELD_GET(GS_AUTH_STATUS_MASK, status)); if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) { - drm_dbg(drm, "GuC firmware signature verification failed\n"); + drm_info(drm, "GuC firmware signature verification failed\n"); ret = -ENOEXEC; } - if ((status & GS_UKERNEL_MASK) == GS_UKERNEL_EXCEPTION) { - drm_dbg(drm, "GuC firmware exception. EIP: %#x\n", - intel_uncore_read(uncore, SOFT_SCRATCH(13))); + if (REG_FIELD_GET(GS_UKERNEL_MASK, status) == INTEL_GUC_LOAD_STATUS_EXCEPTION) { + drm_info(drm, "GuC firmware exception. EIP: %#x\n", + intel_uncore_read(uncore, SOFT_SCRATCH(13))); ret = -ENXIO; } } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h index 7072e30e99f4..4b300b6cc0f9 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h @@ -16,6 +16,7 @@ #include "abi/guc_errors_abi.h" #include "abi/guc_communication_mmio_abi.h" #include "abi/guc_communication_ctb_abi.h" +#include "abi/guc_klvs_abi.h" #include "abi/guc_messages_abi.h" /* Payload length only i.e. don't include G2H header length */ @@ -45,8 +46,8 @@ #define GUC_VIDEO_CLASS 1 #define GUC_VIDEOENHANCE_CLASS 2 #define GUC_BLITTER_CLASS 3 -#define GUC_RESERVED_CLASS 4 -#define GUC_LAST_ENGINE_CLASS GUC_RESERVED_CLASS +#define GUC_COMPUTE_CLASS 4 +#define GUC_LAST_ENGINE_CLASS GUC_COMPUTE_CLASS #define GUC_MAX_ENGINE_CLASSES 16 #define GUC_MAX_INSTANCES_PER_CLASS 32 @@ -84,19 +85,24 @@ #define GUC_STAGE_DESC_ATTR_TERMINATED BIT(7) #define GUC_CTL_LOG_PARAMS 0 -#define GUC_LOG_VALID (1 << 0) -#define GUC_LOG_NOTIFY_ON_HALF_FULL (1 << 1) -#define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) +#define GUC_LOG_VALID BIT(0) +#define GUC_LOG_NOTIFY_ON_HALF_FULL BIT(1) +#define GUC_LOG_CAPTURE_ALLOC_UNITS BIT(2) +#define GUC_LOG_LOG_ALLOC_UNITS BIT(3) #define GUC_LOG_CRASH_SHIFT 4 #define GUC_LOG_CRASH_MASK (0x3 << GUC_LOG_CRASH_SHIFT) #define GUC_LOG_DEBUG_SHIFT 6 #define GUC_LOG_DEBUG_MASK (0xF << GUC_LOG_DEBUG_SHIFT) +#define GUC_LOG_CAPTURE_SHIFT 10 +#define GUC_LOG_CAPTURE_MASK (0x3 << GUC_LOG_CAPTURE_SHIFT) #define GUC_LOG_BUF_ADDR_SHIFT 12 #define GUC_CTL_WA 1 +#define GUC_WA_POLLCS BIT(18) + #define GUC_CTL_FEATURE 2 -#define GUC_CTL_DISABLE_SCHEDULER (1 << 14) #define GUC_CTL_ENABLE_SLPC BIT(2) +#define GUC_CTL_DISABLE_SCHEDULER BIT(14) #define GUC_CTL_DEBUG 3 #define GUC_LOG_VERBOSITY_SHIFT 0 @@ -116,6 +122,8 @@ #define GUC_ADS_ADDR_SHIFT 1 #define GUC_ADS_ADDR_MASK (0xFFFFF << GUC_ADS_ADDR_SHIFT) +#define GUC_CTL_DEVID 5 + #define GUC_CTL_MAX_DWORDS (SOFT_SCRATCH_COUNT - 2) /* [1..14] */ /* Generic GT SysInfo data types */ @@ -148,23 +156,37 @@ FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID, id) | \ FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC, c) \ ) +/* the GuC arrays don't include OTHER_CLASS */ +static u8 engine_class_guc_class_map[] = { + [RENDER_CLASS] = GUC_RENDER_CLASS, + [COPY_ENGINE_CLASS] = GUC_BLITTER_CLASS, + [VIDEO_DECODE_CLASS] = GUC_VIDEO_CLASS, + [VIDEO_ENHANCEMENT_CLASS] = GUC_VIDEOENHANCE_CLASS, + [COMPUTE_CLASS] = GUC_COMPUTE_CLASS, +}; + +static u8 guc_class_engine_class_map[] = { + [GUC_RENDER_CLASS] = RENDER_CLASS, + [GUC_BLITTER_CLASS] = COPY_ENGINE_CLASS, + [GUC_VIDEO_CLASS] = VIDEO_DECODE_CLASS, + [GUC_VIDEOENHANCE_CLASS] = VIDEO_ENHANCEMENT_CLASS, + [GUC_COMPUTE_CLASS] = COMPUTE_CLASS, +}; + static inline u8 engine_class_to_guc_class(u8 class) { - BUILD_BUG_ON(GUC_RENDER_CLASS != RENDER_CLASS); - BUILD_BUG_ON(GUC_BLITTER_CLASS != COPY_ENGINE_CLASS); - BUILD_BUG_ON(GUC_VIDEO_CLASS != VIDEO_DECODE_CLASS); - BUILD_BUG_ON(GUC_VIDEOENHANCE_CLASS != VIDEO_ENHANCEMENT_CLASS); + BUILD_BUG_ON(ARRAY_SIZE(engine_class_guc_class_map) != MAX_ENGINE_CLASS + 1); GEM_BUG_ON(class > MAX_ENGINE_CLASS || class == OTHER_CLASS); - return class; + return engine_class_guc_class_map[class]; } static inline u8 guc_class_to_engine_class(u8 guc_class) { + BUILD_BUG_ON(ARRAY_SIZE(guc_class_engine_class_map) != GUC_LAST_ENGINE_CLASS + 1); GEM_BUG_ON(guc_class > GUC_LAST_ENGINE_CLASS); - GEM_BUG_ON(guc_class == GUC_RESERVED_CLASS); - return guc_class; + return guc_class_engine_class_map[guc_class]; } /* Work item for submitting workloads into work queue of GuC. */ @@ -263,7 +285,10 @@ struct guc_mmio_reg { u32 offset; u32 value; u32 flags; -#define GUC_REGSET_MASKED (1 << 0) + u32 mask; +#define GUC_REGSET_MASKED BIT(0) +#define GUC_REGSET_MASKED_WITH_VALUE BIT(2) +#define GUC_REGSET_RESTORE_ONLY BIT(3) } __packed; /* GuC register sets */ @@ -280,6 +305,12 @@ struct guc_gt_system_info { u32 generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_MAX]; } __packed; +enum { + GUC_CAPTURE_LIST_INDEX_PF = 0, + GUC_CAPTURE_LIST_INDEX_VF = 1, + GUC_CAPTURE_LIST_INDEX_MAX = 2, +}; + /* GuC Additional Data Struct */ struct guc_ads { struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; @@ -291,7 +322,11 @@ struct guc_ads { u32 golden_context_lrca[GUC_MAX_ENGINE_CLASSES]; u32 eng_state_size[GUC_MAX_ENGINE_CLASSES]; u32 private_data; - u32 reserved[15]; + u32 reserved2; + u32 capture_instance[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES]; + u32 capture_class[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES]; + u32 capture_global[GUC_CAPTURE_LIST_INDEX_MAX]; + u32 reserved[14]; } __packed; /* Engine usage stats */ @@ -312,6 +347,7 @@ struct guc_engine_usage { enum guc_log_buffer_type { GUC_DEBUG_LOG_BUFFER, GUC_CRASH_DUMP_LOG_BUFFER, + GUC_CAPTURE_LOG_BUFFER, GUC_MAX_LOG_BUFFER }; @@ -342,6 +378,7 @@ struct guc_log_buffer_state { u32 write_ptr; u32 size; u32 sampled_write_ptr; + u32 wrap_offset; union { struct { u32 flush_to_file:1; @@ -382,7 +419,7 @@ struct guc_shared_ctx_data { /* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */ enum intel_guc_recv_message { INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED = BIT(1), - INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER = BIT(3) + INTEL_GUC_RECV_MSG_EXCEPTION = BIT(30), }; #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c index ac0931f0374b..b53f61f3101f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c @@ -7,6 +7,7 @@ #include "gt/intel_gt.h" #include "i915_drv.h" +#include "i915_irq.h" #include "i915_memcpy.h" #include "intel_guc_log.h" @@ -55,20 +56,6 @@ static int guc_action_control_log(struct intel_guc *guc, bool enable, return intel_guc_send(guc, action, ARRAY_SIZE(action)); } -static void guc_log_enable_flush_events(struct intel_guc_log *log) -{ - intel_guc_enable_msg(log_to_guc(log), - INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER | - INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED); -} - -static void guc_log_disable_flush_events(struct intel_guc_log *log) -{ - intel_guc_disable_msg(log_to_guc(log), - INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER | - INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED); -} - /* * Sub buffer switch callback. Called whenever relay has to switch to a new * sub buffer, relay stays on the same sub buffer if 0 is returned. @@ -201,6 +188,8 @@ static unsigned int guc_get_log_buffer_size(enum guc_log_buffer_type type) return DEBUG_BUFFER_SIZE; case GUC_CRASH_DUMP_LOG_BUFFER: return CRASH_BUFFER_SIZE; + case GUC_CAPTURE_LOG_BUFFER: + return CAPTURE_BUFFER_SIZE; default: MISSING_CASE(type); } @@ -463,14 +452,19 @@ int intel_guc_log_create(struct intel_guc_log *log) * +-------------------------------+ 32B * | Debug state header | * +-------------------------------+ 64B + * | Capture state header | + * +-------------------------------+ 96B * | | * +===============================+ PAGE_SIZE (4KB) * | Crash Dump logs | * +===============================+ + CRASH_SIZE * | Debug logs | * +===============================+ + DEBUG_SIZE + * | Capture logs | + * +===============================+ + CAPTURE_SIZE */ - guc_log_size = PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE; + guc_log_size = PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE + + CAPTURE_BUFFER_SIZE; vma = intel_guc_allocate_vma(guc, guc_log_size); if (IS_ERR(vma)) { @@ -592,8 +586,6 @@ int intel_guc_log_relay_start(struct intel_guc_log *log) if (log->relay.started) return -EEXIST; - guc_log_enable_flush_events(log); - /* * When GuC is logging without us relaying to userspace, we're ignoring * the flush notification. This means that we need to unconditionally @@ -640,7 +632,6 @@ static void guc_log_relay_stop(struct intel_guc_log *log) if (!log->relay.started) return; - guc_log_disable_flush_events(log); intel_synchronize_irq(i915); flush_work(&log->relay.flush_work); @@ -661,7 +652,8 @@ void intel_guc_log_relay_close(struct intel_guc_log *log) void intel_guc_log_handle_flush_event(struct intel_guc_log *log) { - queue_work(system_highpri_wq, &log->relay.flush_work); + if (log->relay.started) + queue_work(system_highpri_wq, &log->relay.flush_work); } static const char * @@ -672,6 +664,8 @@ stringify_guc_log_type(enum guc_log_buffer_type type) return "DEBUG"; case GUC_CRASH_DUMP_LOG_BUFFER: return "CRASH"; + case GUC_CAPTURE_LOG_BUFFER: + return "CAPTURE"; default: MISSING_CASE(type); } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h index fe6ab7550a14..d7e1b6471fed 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h @@ -18,12 +18,15 @@ struct intel_guc; #if defined(CONFIG_DRM_I915_DEBUG_GUC) #define CRASH_BUFFER_SIZE SZ_2M #define DEBUG_BUFFER_SIZE SZ_16M +#define CAPTURE_BUFFER_SIZE SZ_4M #elif defined(CONFIG_DRM_I915_DEBUG_GEM) #define CRASH_BUFFER_SIZE SZ_1M #define DEBUG_BUFFER_SIZE SZ_2M +#define CAPTURE_BUFFER_SIZE SZ_1M #else #define CRASH_BUFFER_SIZE SZ_8K #define DEBUG_BUFFER_SIZE SZ_64K +#define CAPTURE_BUFFER_SIZE SZ_16K #endif /* diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h index b37fc2ffaef2..66027a42cda9 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h @@ -9,7 +9,7 @@ #include <linux/compiler.h> #include <linux/types.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" /* Definitions of GuC H/W registers, bits, etc */ @@ -22,10 +22,6 @@ #define GS_BOOTROM_JUMP_PASSED (0x76 << GS_BOOTROM_SHIFT) #define GS_UKERNEL_SHIFT 8 #define GS_UKERNEL_MASK (0xFF << GS_UKERNEL_SHIFT) -#define GS_UKERNEL_LAPIC_DONE (0x30 << GS_UKERNEL_SHIFT) -#define GS_UKERNEL_DPC_ERROR (0x60 << GS_UKERNEL_SHIFT) -#define GS_UKERNEL_EXCEPTION (0x70 << GS_UKERNEL_SHIFT) -#define GS_UKERNEL_READY (0xF0 << GS_UKERNEL_SHIFT) #define GS_MIA_SHIFT 16 #define GS_MIA_MASK (0x07 << GS_MIA_SHIFT) #define GS_MIA_CORE_STATE (0x01 << GS_MIA_SHIFT) @@ -98,6 +94,9 @@ #define GUC_ENABLE_MIA_CLOCK_GATING (1<<15) #define GUC_GEN10_SHIM_WC_ENABLE (1<<21) +#define GUC_SHIM_CONTROL2 _MMIO(0xc068) +#define GUC_IS_PRIVILEGED (1<<29) + #define GUC_SEND_INTERRUPT _MMIO(0xc4c8) #define GUC_SEND_TRIGGER (1<<0) #define GEN11_GUC_HOST_INTERRUPT _MMIO(0x1901f0) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c index ba21ace973da..ac749ab11035 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c @@ -3,9 +3,15 @@ * Copyright © 2021 Intel Corporation */ +#include <drm/drm_cache.h> + #include "i915_drv.h" +#include "i915_reg.h" #include "intel_guc_slpc.h" +#include "intel_mchbar_regs.h" #include "gt/intel_gt.h" +#include "gt/intel_gt_regs.h" +#include "gt/intel_rps.h" static inline struct intel_guc *slpc_to_guc(struct intel_guc_slpc *slpc) { @@ -574,10 +580,10 @@ static int slpc_use_fused_rp0(struct intel_guc_slpc *slpc) static void slpc_get_rp_values(struct intel_guc_slpc *slpc) { + struct intel_rps *rps = &slpc_to_gt(slpc)->rps; u32 rp_state_cap; - rp_state_cap = intel_uncore_read(slpc_to_gt(slpc)->uncore, - GEN6_RP_STATE_CAP); + rp_state_cap = intel_rps_read_state_cap(rps); slpc->rp0_freq = REG_FIELD_GET(RP0_CAP_MASK, rp_state_cap) * GT_FREQUENCY_MULTIPLIER; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 154ad726e266..1ce7e04aa837 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -9,13 +9,15 @@ #include "gt/gen8_engine_cs.h" #include "gt/intel_breadcrumbs.h" #include "gt/intel_context.h" -#include "gt/intel_engine_pm.h" #include "gt/intel_engine_heartbeat.h" +#include "gt/intel_engine_pm.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_gpu_commands.h" #include "gt/intel_gt.h" #include "gt/intel_gt_clock_utils.h" #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_gt_requests.h" #include "gt/intel_lrc.h" #include "gt/intel_lrc_reg.h" @@ -172,11 +174,8 @@ static inline void init_sched_state(struct intel_context *ce) __maybe_unused static bool sched_state_is_init(struct intel_context *ce) { - /* - * XXX: Kernel contexts can have SCHED_STATE_NO_LOCK_REGISTERED after - * suspend. - */ - return !(ce->guc_state.sched_state &= + /* Kernel contexts can have SCHED_STATE_REGISTERED after suspend. */ + return !(ce->guc_state.sched_state & ~(SCHED_STATE_BLOCKED_MASK | SCHED_STATE_REGISTERED)); } @@ -1137,6 +1136,9 @@ __extend_last_switch(struct intel_guc *guc, u64 *prev_start, u32 new_start) *prev_start = ((u64)gt_stamp_hi << 32) | new_start; } +#define record_read(map_, field_) \ + iosys_map_rd_field(map_, 0, struct guc_engine_usage_record, field_) + /* * GuC updates shared memory and KMD reads it. Since this is not synchronized, * we run into a race where the value read is inconsistent. Sometimes the @@ -1151,17 +1153,17 @@ __extend_last_switch(struct intel_guc *guc, u64 *prev_start, u32 new_start) static void __get_engine_usage_record(struct intel_engine_cs *engine, u32 *last_in, u32 *id, u32 *total) { - struct guc_engine_usage_record *rec = intel_guc_engine_usage(engine); + struct iosys_map rec_map = intel_guc_engine_usage_record_map(engine); int i = 0; do { - *last_in = READ_ONCE(rec->last_switch_in_stamp); - *id = READ_ONCE(rec->current_context_index); - *total = READ_ONCE(rec->total_runtime); + *last_in = record_read(&rec_map, last_switch_in_stamp); + *id = record_read(&rec_map, current_context_index); + *total = record_read(&rec_map, total_runtime); - if (READ_ONCE(rec->last_switch_in_stamp) == *last_in && - READ_ONCE(rec->current_context_index) == *id && - READ_ONCE(rec->total_runtime) == *total) + if (record_read(&rec_map, last_switch_in_stamp) == *last_in && + record_read(&rec_map, current_context_index) == *id && + record_read(&rec_map, total_runtime) == *total) break; } while (++i < 6); } @@ -1428,7 +1430,8 @@ submission_disabled(struct intel_guc *guc) struct i915_sched_engine * const sched_engine = guc->sched_engine; return unlikely(!sched_engine || - !__tasklet_is_enabled(&sched_engine->tasklet)); + !__tasklet_is_enabled(&sched_engine->tasklet) || + intel_gt_is_wedged(guc_to_gt(guc))); } static void disable_submission(struct intel_guc *guc) @@ -1473,8 +1476,6 @@ static void guc_flush_destroyed_contexts(struct intel_guc *guc); void intel_guc_submission_reset_prepare(struct intel_guc *guc) { - int i; - if (unlikely(!guc_submission_initialized(guc))) { /* Reset called during driver load? GuC not yet initialised! */ return; @@ -1491,21 +1492,7 @@ void intel_guc_submission_reset_prepare(struct intel_guc *guc) guc_flush_submissions(guc); guc_flush_destroyed_contexts(guc); - - /* - * Handle any outstanding G2Hs before reset. Call IRQ handler directly - * each pass as interrupt have been disabled. We always scrub for - * outstanding G2H as it is possible for outstanding_submission_g2h to - * be incremented after the context state update. - */ - for (i = 0; i < 4 && atomic_read(&guc->outstanding_submission_g2h); ++i) { - intel_guc_to_host_event_handler(guc); -#define wait_for_reset(guc, wait_var) \ - intel_guc_wait_for_pending_msg(guc, wait_var, false, (HZ / 20)) - do { - wait_for_reset(guc, &guc->outstanding_submission_g2h); - } while (!list_empty(&guc->ct.requests.incoming)); - } + flush_work(&guc->ct.requests.worker); scrub_guc_desc_for_outstanding_g2h(guc); } @@ -1610,7 +1597,6 @@ static void __guc_reset_context(struct intel_context *ce, bool stalled) unsigned long flags; u32 head; int i, number_children = ce->parallel.number_children; - bool skip = false; struct intel_context *parent = ce; GEM_BUG_ON(intel_context_is_child(ce)); @@ -1621,23 +1607,10 @@ static void __guc_reset_context(struct intel_context *ce, bool stalled) * GuC will implicitly mark the context as non-schedulable when it sends * the reset notification. Make sure our state reflects this change. The * context will be marked enabled on resubmission. - * - * XXX: If the context is reset as a result of the request cancellation - * this G2H is received after the schedule disable complete G2H which is - * wrong as this creates a race between the request cancellation code - * re-submitting the context and this G2H handler. This is a bug in the - * GuC but can be worked around in the meantime but converting this to a - * NOP if a pending enable is in flight as this indicates that a request - * cancellation has occurred. */ spin_lock_irqsave(&ce->guc_state.lock, flags); - if (likely(!context_pending_enable(ce))) - clr_context_enabled(ce); - else - skip = true; + clr_context_enabled(ce); spin_unlock_irqrestore(&ce->guc_state.lock, flags); - if (unlikely(skip)) - goto out_put; /* * For each context in the relationship find the hanging request @@ -1669,7 +1642,6 @@ next_context: } __unwind_incomplete_requests(parent); -out_put: intel_context_put(parent); } @@ -1804,7 +1776,7 @@ void intel_guc_submission_reset_finish(struct intel_guc *guc) { /* Reset called during driver load or during wedge? */ if (unlikely(!guc_submission_initialized(guc) || - test_bit(I915_WEDGED, &guc_to_gt(guc)->reset.flags))) { + intel_gt_is_wedged(guc_to_gt(guc)))) { return; } @@ -1823,6 +1795,7 @@ void intel_guc_submission_reset_finish(struct intel_guc *guc) } static void destroyed_worker_func(struct work_struct *w); +static void reset_fail_worker_func(struct work_struct *w); /* * Set up the memory resources to be shared with the GuC (via the GGTT) @@ -1845,22 +1818,11 @@ int intel_guc_submission_init(struct intel_guc *guc) */ GEM_BUG_ON(!guc->lrc_desc_pool); - xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ); - - spin_lock_init(&guc->submission_state.lock); - INIT_LIST_HEAD(&guc->submission_state.guc_id_list); - ida_init(&guc->submission_state.guc_ids); - INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts); - INIT_WORK(&guc->submission_state.destroyed_worker, - destroyed_worker_func); - guc->submission_state.guc_ids_bitmap = bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL); if (!guc->submission_state.guc_ids_bitmap) return -ENOMEM; - spin_lock_init(&guc->timestamp.lock); - INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping); guc->timestamp.ping_delay = (POLL_TIME_CLKS / gt->clock_frequency + 1) * HZ; guc->timestamp.shift = gpm_timestamp_shift(gt); @@ -2609,12 +2571,6 @@ static void guc_context_cancel_request(struct intel_context *ce, true); } - /* - * XXX: Racey if context is reset, see comment in - * __guc_reset_context(). - */ - flush_work(&ce_to_guc(ce)->ct.requests.worker); - guc_context_unblock(block_context); intel_context_put(ce); } @@ -3328,8 +3284,6 @@ static void guc_parent_context_unpin(struct intel_context *ce) GEM_BUG_ON(!intel_context_is_parent(ce)); GEM_BUG_ON(!intel_engine_is_virtual(ce->engine)); - if (ce->parallel.last_rq) - i915_request_put(ce->parallel.last_rq); unpin_guc_id(guc, ce); lrc_unpin(ce); } @@ -3641,6 +3595,9 @@ static int guc_resume(struct intel_engine_cs *engine) setup_hwsp(engine); start_engine(engine); + if (engine->class == RENDER_CLASS) + xehp_enable_ccs_engines(engine); + return 0; } @@ -3822,7 +3779,7 @@ int intel_guc_submission_setup(struct intel_engine_cs *engine) guc_default_irqs(engine); guc_init_breadcrumbs(engine); - if (engine->class == RENDER_CLASS) + if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE) rcs_submission_override(engine); lrc_init_wa_ctx(engine); @@ -3864,6 +3821,20 @@ static bool __guc_submission_selected(struct intel_guc *guc) void intel_guc_submission_init_early(struct intel_guc *guc) { + xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ); + + spin_lock_init(&guc->submission_state.lock); + INIT_LIST_HEAD(&guc->submission_state.guc_id_list); + ida_init(&guc->submission_state.guc_ids); + INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts); + INIT_WORK(&guc->submission_state.destroyed_worker, + destroyed_worker_func); + INIT_WORK(&guc->submission_state.reset_fail_worker, + reset_fail_worker_func); + + spin_lock_init(&guc->timestamp.lock); + INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping); + guc->submission_state.num_guc_ids = GUC_MAX_LRC_DESCRIPTORS; guc->submission_supported = __guc_submission_supported(guc); guc->submission_selected = __guc_submission_selected(guc); @@ -4051,14 +4022,13 @@ static void guc_handle_context_reset(struct intel_guc *guc, { trace_intel_context_reset(ce); - /* - * XXX: Racey if request cancellation has occurred, see comment in - * __guc_reset_context(). - */ - if (likely(!intel_context_is_banned(ce) && - !context_blocked(ce))) { + if (likely(!intel_context_is_banned(ce))) { capture_error_state(guc, ce); guc_context_replay(ce); + } else { + drm_info(&guc_to_gt(guc)->i915->drm, + "Ignoring context reset notification of banned context 0x%04X on %s", + ce->guc_id.id, ce->engine->name); } } @@ -4097,6 +4067,24 @@ int intel_guc_context_reset_process_msg(struct intel_guc *guc, return 0; } +int intel_guc_error_capture_process_msg(struct intel_guc *guc, + const u32 *msg, u32 len) +{ + int status; + + if (unlikely(len != 1)) { + drm_dbg(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len); + return -EPROTO; + } + + status = msg[0]; + drm_info(&guc_to_gt(guc)->i915->drm, "Got error capture: status = %d", status); + + /* FIXME: Do something with the capture */ + + return 0; +} + static struct intel_engine_cs * guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance) { @@ -4109,6 +4097,26 @@ guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance) return gt->engine_class[engine_class][instance]; } +static void reset_fail_worker_func(struct work_struct *w) +{ + struct intel_guc *guc = container_of(w, struct intel_guc, + submission_state.reset_fail_worker); + struct intel_gt *gt = guc_to_gt(guc); + intel_engine_mask_t reset_fail_mask; + unsigned long flags; + + spin_lock_irqsave(&guc->submission_state.lock, flags); + reset_fail_mask = guc->submission_state.reset_fail_mask; + guc->submission_state.reset_fail_mask = 0; + spin_unlock_irqrestore(&guc->submission_state.lock, flags); + + if (likely(reset_fail_mask)) + intel_gt_handle_error(gt, reset_fail_mask, + I915_ERROR_CAPTURE, + "GuC failed to reset engine mask=0x%x\n", + reset_fail_mask); +} + int intel_guc_engine_failure_process_msg(struct intel_guc *guc, const u32 *msg, u32 len) { @@ -4116,6 +4124,7 @@ int intel_guc_engine_failure_process_msg(struct intel_guc *guc, struct intel_gt *gt = guc_to_gt(guc); u8 guc_class, instance; u32 reason; + unsigned long flags; if (unlikely(len != 3)) { drm_err(>->i915->drm, "Invalid length %u", len); @@ -4140,10 +4149,15 @@ int intel_guc_engine_failure_process_msg(struct intel_guc *guc, drm_err(>->i915->drm, "GuC engine reset request failed on %d:%d (%s) because 0x%08X", guc_class, instance, engine->name, reason); - intel_gt_handle_error(gt, engine->mask, - I915_ERROR_CAPTURE, - "GuC failed to reset %s (reason=0x%08x)\n", - engine->name, reason); + spin_lock_irqsave(&guc->submission_state.lock, flags); + guc->submission_state.reset_fail_mask |= engine->mask; + spin_unlock_irqrestore(&guc->submission_state.lock, flags); + + /* + * A GT reset flushes this worker queue (G2H handler) so we must use + * another worker to trigger a GT reset. + */ + queue_work(system_unbound_wq, &guc->submission_state.reset_fail_worker); return 0; } @@ -4512,27 +4526,31 @@ static inline bool skip_handshake(struct i915_request *rq) return test_bit(I915_FENCE_FLAG_SKIP_PARALLEL, &rq->fence.flags); } +#define NON_SKIP_LEN 6 static u32 * emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq, u32 *cs) { struct intel_context *ce = rq->context; + __maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs; + __maybe_unused u32 *start_fini_breadcrumb_cs = cs; GEM_BUG_ON(!intel_context_is_parent(ce)); if (unlikely(skip_handshake(rq))) { /* * NOP everything in __emit_fini_breadcrumb_parent_no_preempt_mid_batch, - * the -6 comes from the length of the emits below. + * the NON_SKIP_LEN comes from the length of the emits below. */ memset(cs, 0, sizeof(u32) * - (ce->engine->emit_fini_breadcrumb_dw - 6)); - cs += ce->engine->emit_fini_breadcrumb_dw - 6; + (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN)); + cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN; } else { cs = __emit_fini_breadcrumb_parent_no_preempt_mid_batch(rq, cs); } /* Emit fini breadcrumb */ + before_fini_breadcrumb_user_interrupt_cs = cs; cs = gen8_emit_ggtt_write(cs, rq->fence.seqno, i915_request_active_timeline(rq)->hwsp_offset, @@ -4542,6 +4560,12 @@ emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq, *cs++ = MI_USER_INTERRUPT; *cs++ = MI_NOOP; + /* Ensure our math for skip + emit is correct */ + GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN != + cs); + GEM_BUG_ON(start_fini_breadcrumb_cs + + ce->engine->emit_fini_breadcrumb_dw != cs); + rq->tail = intel_ring_offset(rq, cs); return cs; @@ -4584,22 +4608,25 @@ emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq, u32 *cs) { struct intel_context *ce = rq->context; + __maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs; + __maybe_unused u32 *start_fini_breadcrumb_cs = cs; GEM_BUG_ON(!intel_context_is_child(ce)); if (unlikely(skip_handshake(rq))) { /* * NOP everything in __emit_fini_breadcrumb_child_no_preempt_mid_batch, - * the -6 comes from the length of the emits below. + * the NON_SKIP_LEN comes from the length of the emits below. */ memset(cs, 0, sizeof(u32) * - (ce->engine->emit_fini_breadcrumb_dw - 6)); - cs += ce->engine->emit_fini_breadcrumb_dw - 6; + (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN)); + cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN; } else { cs = __emit_fini_breadcrumb_child_no_preempt_mid_batch(rq, cs); } /* Emit fini breadcrumb */ + before_fini_breadcrumb_user_interrupt_cs = cs; cs = gen8_emit_ggtt_write(cs, rq->fence.seqno, i915_request_active_timeline(rq)->hwsp_offset, @@ -4609,11 +4636,19 @@ emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq, *cs++ = MI_USER_INTERRUPT; *cs++ = MI_NOOP; + /* Ensure our math for skip + emit is correct */ + GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN != + cs); + GEM_BUG_ON(start_fini_breadcrumb_cs + + ce->engine->emit_fini_breadcrumb_dw != cs); + rq->tail = intel_ring_offset(rq, cs); return cs; } +#undef NON_SKIP_LEN + static struct intel_context * guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count, unsigned long flags) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c index d10b227ac4aa..556829de9c17 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c @@ -124,6 +124,7 @@ int intel_huc_auth(struct intel_huc *huc) } intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_RUNNING); + drm_info(>->i915->drm, "HuC authenticated\n"); return 0; fail: diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.h b/drivers/gpu/drm/i915/gt/uc/intel_huc.h index ae8c8a6c8cc8..73ec670800f2 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.h @@ -6,7 +6,7 @@ #ifndef _INTEL_HUC_H_ #define _INTEL_HUC_H_ -#include "i915_reg.h" +#include "i915_reg_defs.h" #include "intel_uc_fw.h" #include "intel_huc_fw.h" diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c index 09ed29df67bc..da199aa6989f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c @@ -432,6 +432,15 @@ static int __uc_check_hw(struct intel_uc *uc) return 0; } +static void print_fw_ver(struct intel_uc *uc, struct intel_uc_fw *fw) +{ + struct drm_i915_private *i915 = uc_to_gt(uc)->i915; + + drm_info(&i915->drm, "%s firmware %s version %u.%u\n", + intel_uc_fw_type_repr(fw->type), fw->path, + fw->major_ver_found, fw->minor_ver_found); +} + static int __uc_init_hw(struct intel_uc *uc) { struct drm_i915_private *i915 = uc_to_gt(uc)->i915; @@ -442,6 +451,11 @@ static int __uc_init_hw(struct intel_uc *uc) GEM_BUG_ON(!intel_uc_supports_guc(uc)); GEM_BUG_ON(!intel_uc_wants_guc(uc)); + print_fw_ver(uc, &guc->fw); + + if (intel_uc_uses_huc(uc)) + print_fw_ver(uc, &huc->fw); + if (!intel_uc_fw_is_loadable(&guc->fw)) { ret = __uc_check_hw(uc) || intel_uc_fw_is_overridden(&guc->fw) || @@ -507,24 +521,11 @@ static int __uc_init_hw(struct intel_uc *uc) intel_rps_lower_unslice(&uc_to_gt(uc)->rps); } - drm_info(&i915->drm, "%s firmware %s version %u.%u %s:%s\n", - intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_GUC), guc->fw.path, - guc->fw.major_ver_found, guc->fw.minor_ver_found, - "submission", + drm_info(&i915->drm, "GuC submission %s\n", enableddisabled(intel_uc_uses_guc_submission(uc))); - - drm_info(&i915->drm, "GuC SLPC: %s\n", + drm_info(&i915->drm, "GuC SLPC %s\n", enableddisabled(intel_uc_uses_guc_slpc(uc))); - if (intel_uc_uses_huc(uc)) { - drm_info(&i915->drm, "%s firmware %s version %u.%u %s:%s\n", - intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_HUC), - huc->fw.path, - huc->fw.major_ver_found, huc->fw.minor_ver_found, - "authenticated", - yesno(intel_huc_is_authenticated(huc))); - } - return 0; /* diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index a5af05bde6f2..c88113044494 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -5,12 +5,15 @@ #include <linux/bitfield.h> #include <linux/firmware.h> + +#include <drm/drm_cache.h> #include <drm/drm_print.h> #include "gem/i915_gem_lmem.h" #include "intel_uc_fw.h" #include "intel_uc_fw_abi.h" #include "i915_drv.h" +#include "i915_reg.h" static inline struct intel_gt * ____uc_fw_to_gt(struct intel_uc_fw *uc_fw, enum intel_uc_fw_type type) @@ -49,21 +52,21 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, * firmware as TGL. */ #define INTEL_GUC_FIRMWARE_DEFS(fw_def, guc_def) \ - fw_def(ALDERLAKE_P, 0, guc_def(adlp, 62, 0, 3)) \ - fw_def(ALDERLAKE_S, 0, guc_def(tgl, 62, 0, 0)) \ - fw_def(DG1, 0, guc_def(dg1, 62, 0, 0)) \ - fw_def(ROCKETLAKE, 0, guc_def(tgl, 62, 0, 0)) \ - fw_def(TIGERLAKE, 0, guc_def(tgl, 62, 0, 0)) \ - fw_def(JASPERLAKE, 0, guc_def(ehl, 62, 0, 0)) \ - fw_def(ELKHARTLAKE, 0, guc_def(ehl, 62, 0, 0)) \ - fw_def(ICELAKE, 0, guc_def(icl, 62, 0, 0)) \ - fw_def(COMETLAKE, 5, guc_def(cml, 62, 0, 0)) \ - fw_def(COMETLAKE, 0, guc_def(kbl, 62, 0, 0)) \ - fw_def(COFFEELAKE, 0, guc_def(kbl, 62, 0, 0)) \ - fw_def(GEMINILAKE, 0, guc_def(glk, 62, 0, 0)) \ - fw_def(KABYLAKE, 0, guc_def(kbl, 62, 0, 0)) \ - fw_def(BROXTON, 0, guc_def(bxt, 62, 0, 0)) \ - fw_def(SKYLAKE, 0, guc_def(skl, 62, 0, 0)) + fw_def(ALDERLAKE_P, 0, guc_def(adlp, 69, 0, 3)) \ + fw_def(ALDERLAKE_S, 0, guc_def(tgl, 69, 0, 3)) \ + fw_def(DG1, 0, guc_def(dg1, 69, 0, 3)) \ + fw_def(ROCKETLAKE, 0, guc_def(tgl, 69, 0, 3)) \ + fw_def(TIGERLAKE, 0, guc_def(tgl, 69, 0, 3)) \ + fw_def(JASPERLAKE, 0, guc_def(ehl, 69, 0, 3)) \ + fw_def(ELKHARTLAKE, 0, guc_def(ehl, 69, 0, 3)) \ + fw_def(ICELAKE, 0, guc_def(icl, 69, 0, 3)) \ + fw_def(COMETLAKE, 5, guc_def(cml, 69, 0, 3)) \ + fw_def(COMETLAKE, 0, guc_def(kbl, 69, 0, 3)) \ + fw_def(COFFEELAKE, 0, guc_def(kbl, 69, 0, 3)) \ + fw_def(GEMINILAKE, 0, guc_def(glk, 69, 0, 3)) \ + fw_def(KABYLAKE, 0, guc_def(kbl, 69, 0, 3)) \ + fw_def(BROXTON, 0, guc_def(bxt, 69, 0, 3)) \ + fw_def(SKYLAKE, 0, guc_def(skl, 69, 0, 3)) #define INTEL_HUC_FIRMWARE_DEFS(fw_def, huc_def) \ fw_def(ALDERLAKE_P, 0, huc_def(tgl, 7, 9, 3)) \ @@ -448,20 +451,19 @@ static void uc_fw_bind_ggtt(struct intel_uc_fw *uc_fw) { struct drm_i915_gem_object *obj = uc_fw->obj; struct i915_ggtt *ggtt = __uc_fw_to_gt(uc_fw)->ggtt; - struct i915_vma *dummy = &uc_fw->dummy; + struct i915_vma_resource *dummy = &uc_fw->dummy; u32 pte_flags = 0; - dummy->node.start = uc_fw_ggtt_offset(uc_fw); - dummy->node.size = obj->base.size; - dummy->pages = obj->mm.pages; - dummy->vm = &ggtt->vm; + dummy->start = uc_fw_ggtt_offset(uc_fw); + dummy->node_size = obj->base.size; + dummy->bi.pages = obj->mm.pages; GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); - GEM_BUG_ON(dummy->node.size > ggtt->uc_fw.size); + GEM_BUG_ON(dummy->node_size > ggtt->uc_fw.size); /* uc_fw->obj cache domains were not controlled across suspend */ if (i915_gem_object_has_struct_page(obj)) - drm_clflush_sg(dummy->pages); + drm_clflush_sg(dummy->bi.pages); if (i915_gem_object_is_lmem(obj)) pte_flags |= PTE_LM; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h index d9d1dc0b4cbb..3229018877d3 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h @@ -85,7 +85,7 @@ struct intel_uc_fw { * threaded as it done during driver load (inherently single threaded) * or during a GT reset (mutex guarantees single threaded). */ - struct i915_vma dummy; + struct i915_vma_resource dummy; struct i915_vma *rsa_data; /* diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c index d3327b802b76..a115894d5896 100644 --- a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c @@ -157,7 +157,7 @@ static int intel_guc_steal_guc_ids(void *arg) wakeref = intel_runtime_pm_get(gt->uncore->rpm); engine = intel_selftest_find_any_engine(gt); sv = guc->submission_state.num_guc_ids; - guc->submission_state.num_guc_ids = 4096; + guc->submission_state.num_guc_ids = 512; /* Create spinner to block requests in below loop */ ce[context_index] = intel_context_create(engine); diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c index 1297ddbf7f88..812220a43df8 100644 --- a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c +++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c @@ -154,6 +154,10 @@ static int intel_guc_multi_lrc_basic(void *arg) int ret; for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) { + /* We don't support breadcrumb handshake on these classes */ + if (class == COMPUTE_CLASS || class == RENDER_CLASS) + continue; + ret = __intel_guc_multi_lrc_basic(gt, class); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c index 0d6d59871308..557f3314291a 100644 --- a/drivers/gpu/drm/i915/gvt/aperture_gm.c +++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c @@ -35,6 +35,7 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "gt/intel_ggtt_fencing.h" #include "gvt.h" @@ -63,7 +64,7 @@ static int alloc_gm(struct intel_vgpu *vgpu, bool high_gm) mutex_lock(>->ggtt->vm.mutex); mmio_hw_access_pre(gt); - ret = i915_gem_gtt_insert(>->ggtt->vm, node, + ret = i915_gem_gtt_insert(>->ggtt->vm, NULL, node, size, I915_GTT_PAGE_SIZE, I915_COLOR_UNEVICTABLE, start, end, flags); diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index c4118b808268..2459213b6c87 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -37,7 +37,9 @@ #include <linux/slab.h> #include "i915_drv.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_gpu_commands.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_lrc.h" #include "gt/intel_ring.h" #include "gt/intel_gt_requests.h" @@ -1144,7 +1146,7 @@ struct cmd_interrupt_event { int mi_user_interrupt; }; -static struct cmd_interrupt_event cmd_interrupt_events[] = { +static const struct cmd_interrupt_event cmd_interrupt_events[] = { [RCS0] = { .pipe_control_notify = RCS_PIPE_CONTROL, .mi_flush_dw = INTEL_GVT_EVENT_RESERVED, diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c index 034c060f89d4..c7722c818b4d 100644 --- a/drivers/gpu/drm/i915/gvt/display.c +++ b/drivers/gpu/drm/i915/gvt/display.c @@ -33,6 +33,7 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "gvt.h" static int get_edp_pipe(struct intel_vgpu *vgpu) @@ -184,10 +185,10 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) for_each_pipe(dev_priv, pipe) { vgpu_vreg_t(vgpu, PIPECONF(pipe)) &= - ~(PIPECONF_ENABLE | I965_PIPECONF_ACTIVE); - vgpu_vreg_t(vgpu, DSPCNTR(pipe)) &= ~DISPLAY_PLANE_ENABLE; + ~(PIPECONF_ENABLE | PIPECONF_STATE_ENABLE); + vgpu_vreg_t(vgpu, DSPCNTR(pipe)) &= ~DISP_ENABLE; vgpu_vreg_t(vgpu, SPRCTL(pipe)) &= ~SPRITE_ENABLE; - vgpu_vreg_t(vgpu, CURCNTR(pipe)) &= ~MCURSOR_MODE; + vgpu_vreg_t(vgpu, CURCNTR(pipe)) &= ~MCURSOR_MODE_MASK; vgpu_vreg_t(vgpu, CURCNTR(pipe)) |= MCURSOR_MODE_DISABLE; } @@ -245,7 +246,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) * setup_virtual_dp_monitor. */ vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE; - vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= I965_PIPECONF_ACTIVE; + vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_STATE_ENABLE; /* * Golden M/N are calculated based on: @@ -253,7 +254,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) * DP link clk 1620 MHz and non-constant_n. * TODO: calculate DP link symbol clk and stream clk m/n. */ - vgpu_vreg_t(vgpu, PIPE_DATA_M1(TRANSCODER_A)) = 63 << TU_SIZE_SHIFT; + vgpu_vreg_t(vgpu, PIPE_DATA_M1(TRANSCODER_A)) = TU_SIZE(64); vgpu_vreg_t(vgpu, PIPE_DATA_M1(TRANSCODER_A)) |= 0x5b425e; vgpu_vreg_t(vgpu, PIPE_DATA_N1(TRANSCODER_A)) = 0x800000; vgpu_vreg_t(vgpu, PIPE_LINK_M1(TRANSCODER_A)) = 0x3cd6e; @@ -387,7 +388,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) * DP link clk 1620 MHz and non-constant_n. * TODO: calculate DP link symbol clk and stream clk m/n. */ - vgpu_vreg_t(vgpu, PIPE_DATA_M1(TRANSCODER_A)) = 63 << TU_SIZE_SHIFT; + vgpu_vreg_t(vgpu, PIPE_DATA_M1(TRANSCODER_A)) = TU_SIZE(64); vgpu_vreg_t(vgpu, PIPE_DATA_M1(TRANSCODER_A)) |= 0x5b425e; vgpu_vreg_t(vgpu, PIPE_DATA_N1(TRANSCODER_A)) = 0x800000; vgpu_vreg_t(vgpu, PIPE_LINK_M1(TRANSCODER_A)) = 0x3cd6e; @@ -496,9 +497,9 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) /* Disable Primary/Sprite/Cursor plane */ for_each_pipe(dev_priv, pipe) { - vgpu_vreg_t(vgpu, DSPCNTR(pipe)) &= ~DISPLAY_PLANE_ENABLE; + vgpu_vreg_t(vgpu, DSPCNTR(pipe)) &= ~DISP_ENABLE; vgpu_vreg_t(vgpu, SPRCTL(pipe)) &= ~SPRITE_ENABLE; - vgpu_vreg_t(vgpu, CURCNTR(pipe)) &= ~MCURSOR_MODE; + vgpu_vreg_t(vgpu, CURCNTR(pipe)) &= ~MCURSOR_MODE_MASK; vgpu_vreg_t(vgpu, CURCNTR(pipe)) |= MCURSOR_MODE_DISABLE; } diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c index 8e65cd8258b9..c95c25d2addb 100644 --- a/drivers/gpu/drm/i915/gvt/dmabuf.c +++ b/drivers/gpu/drm/i915/gvt/dmabuf.c @@ -31,7 +31,13 @@ #include <linux/dma-buf.h> #include <linux/vfio.h> +#include <drm/drm_fourcc.h> +#include <drm/drm_plane.h> + +#include "gem/i915_gem_dmabuf.h" + #include "i915_drv.h" +#include "i915_reg.h" #include "gvt.h" #define GEN8_DECODE_PTE(pte) (pte & GENMASK_ULL(63, 12)) @@ -84,7 +90,7 @@ static int vgpu_gem_get_pages( kfree(st); return ret; } - gtt_entries = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + + gtt_entries = (gen8_pte_t __iomem *)to_gt(dev_priv)->ggtt->gsm + (fb_info->start >> PAGE_SHIFT); for_each_sg(st->sgl, sg, page_num, i) { dma_addr_t dma_addr = @@ -148,8 +154,7 @@ static void dmabuf_gem_object_free(struct kref *kref) if (vgpu && vgpu->active && !list_empty(&vgpu->dmabuf_obj_list_head)) { list_for_each(pos, &vgpu->dmabuf_obj_list_head) { - dmabuf_obj = container_of(pos, - struct intel_vgpu_dmabuf_obj, list); + dmabuf_obj = list_entry(pos, struct intel_vgpu_dmabuf_obj, list); if (dmabuf_obj == obj) { list_del(pos); intel_gvt_hypervisor_put_vfio_device(vgpu); @@ -357,10 +362,8 @@ pick_dmabuf_by_info(struct intel_vgpu *vgpu, struct intel_vgpu_dmabuf_obj *ret = NULL; list_for_each(pos, &vgpu->dmabuf_obj_list_head) { - dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, - list); - if ((dmabuf_obj == NULL) || - (dmabuf_obj->info == NULL)) + dmabuf_obj = list_entry(pos, struct intel_vgpu_dmabuf_obj, list); + if (!dmabuf_obj->info) continue; fb_info = (struct intel_vgpu_fb_info *)dmabuf_obj->info; @@ -387,11 +390,7 @@ pick_dmabuf_by_num(struct intel_vgpu *vgpu, u32 id) struct intel_vgpu_dmabuf_obj *ret = NULL; list_for_each(pos, &vgpu->dmabuf_obj_list_head) { - dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, - list); - if (!dmabuf_obj) - continue; - + dmabuf_obj = list_entry(pos, struct intel_vgpu_dmabuf_obj, list); if (dmabuf_obj->dmabuf_id == id) { ret = dmabuf_obj; break; @@ -600,8 +599,7 @@ void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu) mutex_lock(&vgpu->dmabuf_lock); list_for_each_safe(pos, n, &vgpu->dmabuf_obj_list_head) { - dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, - list); + dmabuf_obj = list_entry(pos, struct intel_vgpu_dmabuf_obj, list); dmabuf_obj->vgpu = NULL; idr_remove(&vgpu->object_idr, dmabuf_obj->dmabuf_id); diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c index 22247805c345..a30ba2d7b7ba 100644 --- a/drivers/gpu/drm/i915/gvt/edid.c +++ b/drivers/gpu/drm/i915/gvt/edid.c @@ -33,6 +33,7 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "gvt.h" #define GMBUS1_TOTAL_BYTES_SHIFT 16 diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c index c8dcda6d4f0d..66d354c4195b 100644 --- a/drivers/gpu/drm/i915/gvt/execlist.c +++ b/drivers/gpu/drm/i915/gvt/execlist.c @@ -163,7 +163,7 @@ static void emulate_csb_update(struct intel_vgpu_execlist *execlist, hwsp_gpa + I915_HWS_CSB_BUF0_INDEX * 4 + write_pointer * 8, status, 8); intel_gvt_hypervisor_write_gpa(vgpu, - hwsp_gpa + intel_hws_csb_write_index(execlist->engine->i915) * 4, + hwsp_gpa + INTEL_HWS_CSB_WRITE_INDEX(execlist->engine->i915) * 4, &write_pointer, 4); } diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.c b/drivers/gpu/drm/i915/gvt/fb_decoder.c index 9ec064199364..835c3fde8a20 100644 --- a/drivers/gpu/drm/i915/gvt/fb_decoder.c +++ b/drivers/gpu/drm/i915/gvt/fb_decoder.c @@ -37,15 +37,16 @@ #include "i915_drv.h" #include "gvt.h" #include "i915_pvinfo.h" +#include "i915_reg.h" #define PRIMARY_FORMAT_NUM 16 struct pixel_format { - int drm_format; /* Pixel format in DRM definition */ - int bpp; /* Bits per pixel, 0 indicates invalid */ - char *desc; /* The description */ + int drm_format; /* Pixel format in DRM definition */ + int bpp; /* Bits per pixel, 0 indicates invalid */ + const char *desc; /* The description */ }; -static struct pixel_format bdw_pixel_formats[] = { +static const struct pixel_format bdw_pixel_formats[] = { {DRM_FORMAT_C8, 8, "8-bit Indexed"}, {DRM_FORMAT_RGB565, 16, "16-bit BGRX (5:6:5 MSB-R:G:B)"}, {DRM_FORMAT_XRGB8888, 32, "32-bit BGRX (8:8:8:8 MSB-X:R:G:B)"}, @@ -58,7 +59,7 @@ static struct pixel_format bdw_pixel_formats[] = { {0, 0, NULL}, }; -static struct pixel_format skl_pixel_formats[] = { +static const struct pixel_format skl_pixel_formats[] = { {DRM_FORMAT_YUYV, 16, "16-bit packed YUYV (8:8:8:8 MSB-V:Y2:U:Y1)"}, {DRM_FORMAT_UYVY, 16, "16-bit packed UYVY (8:8:8:8 MSB-Y2:V:Y1:U)"}, {DRM_FORMAT_YVYU, 16, "16-bit packed YVYU (8:8:8:8 MSB-U:Y2:V:Y1)"}, @@ -83,22 +84,22 @@ static int bdw_format_to_drm(int format) int bdw_pixel_formats_index = 6; switch (format) { - case DISPPLANE_8BPP: + case DISP_FORMAT_8BPP: bdw_pixel_formats_index = 0; break; - case DISPPLANE_BGRX565: + case DISP_FORMAT_BGRX565: bdw_pixel_formats_index = 1; break; - case DISPPLANE_BGRX888: + case DISP_FORMAT_BGRX888: bdw_pixel_formats_index = 2; break; - case DISPPLANE_RGBX101010: + case DISP_FORMAT_RGBX101010: bdw_pixel_formats_index = 3; break; - case DISPPLANE_BGRX101010: + case DISP_FORMAT_BGRX101010: bdw_pixel_formats_index = 4; break; - case DISPPLANE_RGBX888: + case DISP_FORMAT_RGBX888: bdw_pixel_formats_index = 5; break; @@ -211,14 +212,14 @@ int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, return -ENODEV; val = vgpu_vreg_t(vgpu, DSPCNTR(pipe)); - plane->enabled = !!(val & DISPLAY_PLANE_ENABLE); + plane->enabled = !!(val & DISP_ENABLE); if (!plane->enabled) return -ENODEV; if (GRAPHICS_VER(dev_priv) >= 9) { plane->tiled = val & PLANE_CTL_TILED_MASK; fmt = skl_format_to_drm( - val & PLANE_CTL_FORMAT_MASK, + val & PLANE_CTL_FORMAT_MASK_SKL, val & PLANE_CTL_ORDER_RGBX, val & PLANE_CTL_ALPHA_MASK, val & PLANE_CTL_YUV422_ORDER_MASK); @@ -231,8 +232,8 @@ int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, plane->bpp = skl_pixel_formats[fmt].bpp; plane->drm_format = skl_pixel_formats[fmt].drm_format; } else { - plane->tiled = val & DISPPLANE_TILED; - fmt = bdw_format_to_drm(val & DISPPLANE_PIXFORMAT_MASK); + plane->tiled = val & DISP_TILED; + fmt = bdw_format_to_drm(val & DISP_FORMAT_MASK); plane->bpp = bdw_pixel_formats[fmt].bpp; plane->drm_format = bdw_pixel_formats[fmt].drm_format; } @@ -278,14 +279,14 @@ int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu, #define CURSOR_FORMAT_NUM (1 << 6) struct cursor_mode_format { - int drm_format; /* Pixel format in DRM definition */ - u8 bpp; /* Bits per pixel; 0 indicates invalid */ - u32 width; /* In pixel */ - u32 height; /* In lines */ - char *desc; /* The description */ + int drm_format; /* Pixel format in DRM definition */ + u8 bpp; /* Bits per pixel; 0 indicates invalid */ + u32 width; /* In pixel */ + u32 height; /* In lines */ + const char *desc; /* The description */ }; -static struct cursor_mode_format cursor_pixel_formats[] = { +static const struct cursor_mode_format cursor_pixel_formats[] = { {DRM_FORMAT_ARGB8888, 32, 128, 128, "128x128 32bpp ARGB"}, {DRM_FORMAT_ARGB8888, 32, 256, 256, "256x256 32bpp ARGB"}, {DRM_FORMAT_ARGB8888, 32, 64, 64, "64x64 32bpp ARGB"}, @@ -342,7 +343,7 @@ int intel_vgpu_decode_cursor_plane(struct intel_vgpu *vgpu, return -ENODEV; val = vgpu_vreg_t(vgpu, CURCNTR(pipe)); - mode = val & MCURSOR_MODE; + mode = val & MCURSOR_MODE_MASK; plane->enabled = (mode != MCURSOR_MODE_DISABLE); if (!plane->enabled) return -ENODEV; @@ -391,7 +392,7 @@ int intel_vgpu_decode_cursor_plane(struct intel_vgpu *vgpu, #define SPRITE_FORMAT_NUM (1 << 3) -static struct pixel_format sprite_pixel_formats[SPRITE_FORMAT_NUM] = { +static const struct pixel_format sprite_pixel_formats[SPRITE_FORMAT_NUM] = { [0x0] = {DRM_FORMAT_YUV422, 16, "YUV 16-bit 4:2:2 packed"}, [0x1] = {DRM_FORMAT_XRGB2101010, 32, "RGB 32-bit 2:10:10:10"}, [0x2] = {DRM_FORMAT_XRGB8888, 32, "RGB 32-bit 8:8:8:8"}, @@ -430,7 +431,7 @@ int intel_vgpu_decode_sprite_plane(struct intel_vgpu *vgpu, yuv_order = (val & SPRITE_YUV_ORDER_MASK) >> _SPRITE_YUV_ORDER_SHIFT; - fmt = (val & SPRITE_PIXFORMAT_MASK) >> _SPRITE_FMT_SHIFT; + fmt = (val & SPRITE_FORMAT_MASK) >> _SPRITE_FMT_SHIFT; if (!sprite_pixel_formats[fmt].bpp) { gvt_vgpu_err("Non-supported pixel format (0x%x)\n", fmt); return -EINVAL; diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index af79b39048f7..d4082f4b9be1 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -38,6 +38,8 @@ #include "i915_pvinfo.h" #include "trace.h" +#include "gt/intel_gt_regs.h" + #if defined(VERBOSE_DEBUG) #define gvt_vdbg_mm(fmt, args...) gvt_dbg_mm(fmt, ##args) #else @@ -185,7 +187,7 @@ struct gtt_type_table_entry { .pse_entry_type = pse_type, \ } -static struct gtt_type_table_entry gtt_type_table[] = { +static const struct gtt_type_table_entry gtt_type_table[] = { GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY, GTT_TYPE_PPGTT_ROOT_L4_ENTRY, GTT_TYPE_INVALID, @@ -499,7 +501,7 @@ DEFINE_PPGTT_GMA_TO_INDEX(gen8, l3_pdp, (gma >> 30 & 0x3)); DEFINE_PPGTT_GMA_TO_INDEX(gen8, l4_pdp, (gma >> 30 & 0x1ff)); DEFINE_PPGTT_GMA_TO_INDEX(gen8, pml4, (gma >> 39 & 0x1ff)); -static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = { +static const struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = { .get_entry = gtt_get_entry64, .set_entry = gtt_set_entry64, .clear_present = gtt_entry_clear_present, @@ -516,7 +518,7 @@ static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = { .set_pfn = gen8_gtt_set_pfn, }; -static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = { +static const struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = { .gma_to_ggtt_pte_index = gma_to_ggtt_pte_index, .gma_to_pte_index = gen8_gma_to_pte_index, .gma_to_pde_index = gen8_gma_to_pde_index, @@ -526,7 +528,7 @@ static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = { }; /* Update entry type per pse and ips bit. */ -static void update_entry_type_for_real(struct intel_gvt_gtt_pte_ops *pte_ops, +static void update_entry_type_for_real(const struct intel_gvt_gtt_pte_ops *pte_ops, struct intel_gvt_gtt_entry *entry, bool ips) { switch (entry->type) { @@ -553,7 +555,7 @@ static void _ppgtt_get_root_entry(struct intel_vgpu_mm *mm, struct intel_gvt_gtt_entry *entry, unsigned long index, bool guest) { - struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; GEM_BUG_ON(mm->type != INTEL_GVT_MM_PPGTT); @@ -580,7 +582,7 @@ static void _ppgtt_set_root_entry(struct intel_vgpu_mm *mm, struct intel_gvt_gtt_entry *entry, unsigned long index, bool guest) { - struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; pte_ops->set_entry(guest ? mm->ppgtt_mm.guest_pdps : mm->ppgtt_mm.shadow_pdps, @@ -596,7 +598,7 @@ static inline void ppgtt_set_shadow_root_entry(struct intel_vgpu_mm *mm, static void ggtt_get_guest_entry(struct intel_vgpu_mm *mm, struct intel_gvt_gtt_entry *entry, unsigned long index) { - struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); @@ -608,7 +610,7 @@ static void ggtt_get_guest_entry(struct intel_vgpu_mm *mm, static void ggtt_set_guest_entry(struct intel_vgpu_mm *mm, struct intel_gvt_gtt_entry *entry, unsigned long index) { - struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); @@ -619,7 +621,7 @@ static void ggtt_set_guest_entry(struct intel_vgpu_mm *mm, static void ggtt_get_host_entry(struct intel_vgpu_mm *mm, struct intel_gvt_gtt_entry *entry, unsigned long index) { - struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); @@ -629,7 +631,7 @@ static void ggtt_get_host_entry(struct intel_vgpu_mm *mm, static void ggtt_set_host_entry(struct intel_vgpu_mm *mm, struct intel_gvt_gtt_entry *entry, unsigned long index) { - struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; unsigned long offset = index; GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); @@ -655,7 +657,7 @@ static inline int ppgtt_spt_get_entry( bool guest) { struct intel_gvt *gvt = spt->vgpu->gvt; - struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; int ret; e->type = get_entry_type(type); @@ -684,7 +686,7 @@ static inline int ppgtt_spt_set_entry( bool guest) { struct intel_gvt *gvt = spt->vgpu->gvt; - struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n")) return -EINVAL; @@ -947,7 +949,7 @@ static int ppgtt_invalidate_spt_by_shadow_entry(struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *e) { struct drm_i915_private *i915 = vgpu->gvt->gt->i915; - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *s; enum intel_gvt_gtt_type cur_pt_type; @@ -984,7 +986,7 @@ static inline void ppgtt_invalidate_pte(struct intel_vgpu_ppgtt_spt *spt, struct intel_gvt_gtt_entry *entry) { struct intel_vgpu *vgpu = spt->vgpu; - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; unsigned long pfn; int type; @@ -1072,7 +1074,7 @@ static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt); static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry( struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we) { - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *spt = NULL; bool ips = false; int ret; @@ -1136,7 +1138,7 @@ err: static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se, struct intel_vgpu_ppgtt_spt *s, struct intel_gvt_gtt_entry *ge) { - struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops; se->type = ge->type; se->val64 = ge->val64; @@ -1159,7 +1161,7 @@ static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se, static int is_2MB_gtt_possible(struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *entry) { - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; unsigned long pfn; if (!HAS_PAGE_SIZES(vgpu->gvt->gt->i915, I915_GTT_PAGE_SIZE_2M)) @@ -1176,7 +1178,7 @@ static int split_2MB_gtt_entry(struct intel_vgpu *vgpu, struct intel_vgpu_ppgtt_spt *spt, unsigned long index, struct intel_gvt_gtt_entry *se) { - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *sub_spt; struct intel_gvt_gtt_entry sub_se; unsigned long start_gfn; @@ -1223,7 +1225,7 @@ static int split_64KB_gtt_entry(struct intel_vgpu *vgpu, struct intel_vgpu_ppgtt_spt *spt, unsigned long index, struct intel_gvt_gtt_entry *se) { - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; struct intel_gvt_gtt_entry entry = *se; unsigned long start_gfn; dma_addr_t dma_addr; @@ -1254,7 +1256,7 @@ static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu, struct intel_vgpu_ppgtt_spt *spt, unsigned long index, struct intel_gvt_gtt_entry *ge) { - struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; struct intel_gvt_gtt_entry se = *ge; unsigned long gfn, page_size = PAGE_SIZE; dma_addr_t dma_addr; @@ -1308,7 +1310,7 @@ static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt) { struct intel_vgpu *vgpu = spt->vgpu; struct intel_gvt *gvt = vgpu->gvt; - struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *s; struct intel_gvt_gtt_entry se, ge; unsigned long gfn, i; @@ -1351,7 +1353,7 @@ static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_ppgtt_spt *spt, struct intel_gvt_gtt_entry *se, unsigned long index) { struct intel_vgpu *vgpu = spt->vgpu; - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; int ret; trace_spt_guest_change(spt->vgpu->id, "remove", spt, @@ -1432,7 +1434,7 @@ static int sync_oos_page(struct intel_vgpu *vgpu, { const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; struct intel_gvt *gvt = vgpu->gvt; - struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *spt = oos_page->spt; struct intel_gvt_gtt_entry old, new; int index; @@ -1603,7 +1605,7 @@ static int ppgtt_handle_guest_write_page_table( { struct intel_vgpu *vgpu = spt->vgpu; int type = spt->shadow_page.type; - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; struct intel_gvt_gtt_entry old_se; int new_present; int i, ret; @@ -1720,7 +1722,7 @@ static int ppgtt_handle_guest_write_page_table_bytes( u64 pa, void *p_data, int bytes) { struct intel_vgpu *vgpu = spt->vgpu; - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; struct intel_gvt_gtt_entry we, se; unsigned long index; @@ -1785,7 +1787,7 @@ static void invalidate_ppgtt_mm(struct intel_vgpu_mm *mm) struct intel_vgpu *vgpu = mm->vgpu; struct intel_gvt *gvt = vgpu->gvt; struct intel_gvt_gtt *gtt = &gvt->gtt; - struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops; struct intel_gvt_gtt_entry se; int index; @@ -1815,7 +1817,7 @@ static int shadow_ppgtt_mm(struct intel_vgpu_mm *mm) struct intel_vgpu *vgpu = mm->vgpu; struct intel_gvt *gvt = vgpu->gvt; struct intel_gvt_gtt *gtt = &gvt->gtt; - struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops; struct intel_vgpu_ppgtt_spt *spt; struct intel_gvt_gtt_entry ge, se; int index, ret; @@ -2067,7 +2069,7 @@ static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm, struct intel_gvt_gtt_entry *e, unsigned long index, bool guest) { struct intel_vgpu *vgpu = mm->vgpu; - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; struct intel_vgpu_ppgtt_spt *s; s = intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(e)); @@ -2096,8 +2098,8 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma) { struct intel_vgpu *vgpu = mm->vgpu; struct intel_gvt *gvt = vgpu->gvt; - struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops; - struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops; + const struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops; unsigned long gpa = INTEL_GVT_INVALID_ADDR; unsigned long gma_index[4]; struct intel_gvt_gtt_entry e; @@ -2221,7 +2223,7 @@ int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off, static void ggtt_invalidate_pte(struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *entry) { - struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; unsigned long pfn; pfn = pte_ops->get_pfn(entry); @@ -2236,7 +2238,7 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, struct intel_gvt *gvt = vgpu->gvt; const struct intel_gvt_device_info *info = &gvt->device_info; struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm; - struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; unsigned long g_gtt_index = off >> info->gtt_entry_size_shift; unsigned long gma, gfn; struct intel_gvt_gtt_entry e = {.val64 = 0, .type = GTT_TYPE_GGTT_PTE}; @@ -2391,7 +2393,7 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu, { struct drm_i915_private *i915 = vgpu->gvt->gt->i915; struct intel_vgpu_gtt *gtt = &vgpu->gtt; - struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; int page_entry_num = I915_GTT_PAGE_SIZE >> vgpu->gvt->device_info.gtt_entry_size_shift; void *scratch_pt; @@ -2822,7 +2824,7 @@ void intel_vgpu_invalidate_ppgtt(struct intel_vgpu *vgpu) void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu, bool invalidate_old) { struct intel_gvt *gvt = vgpu->gvt; - struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops; struct intel_gvt_gtt_entry entry = {.type = GTT_TYPE_GGTT_PTE}; struct intel_gvt_gtt_entry old_entry; u32 index; diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h index 3bf45672ef98..a3b0f59ec8bd 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.h +++ b/drivers/gpu/drm/i915/gvt/gtt.h @@ -91,8 +91,8 @@ struct intel_gvt_gtt_gma_ops { }; struct intel_gvt_gtt { - struct intel_gvt_gtt_pte_ops *pte_ops; - struct intel_gvt_gtt_gma_ops *gma_ops; + const struct intel_gvt_gtt_pte_ops *pte_ops; + const struct intel_gvt_gtt_gma_ops *gma_ops; int (*mm_alloc_page_table)(struct intel_vgpu_mm *mm); void (*mm_free_page_table)(struct intel_vgpu_mm *mm); struct list_head oos_page_use_list_head; diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 0c0615602343..0ebffc327528 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -272,7 +272,7 @@ struct intel_gvt_mmio { /* Value of command write of this reg needs to be patched */ #define F_CMD_WRITE_PATCH (1 << 8) - struct gvt_mmio_block *mmio_block; + const struct gvt_mmio_block *mmio_block; unsigned int num_mmio_block; DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS); diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index cde0a477fb49..520a7e1942f3 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -37,9 +37,14 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "gvt.h" #include "i915_pvinfo.h" +#include "intel_mchbar_regs.h" #include "display/intel_display_types.h" +#include "display/intel_fbc.h" +#include "display/vlv_dsi_pll_regs.h" +#include "gt/intel_gt_regs.h" /* XXX FIXME i915 has changed PP_XXX definition */ #define PCH_PP_STATUS _MMIO(0xc7200) @@ -701,11 +706,11 @@ static int pipeconf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, data = vgpu_vreg(vgpu, offset); if (data & PIPECONF_ENABLE) { - vgpu_vreg(vgpu, offset) |= I965_PIPECONF_ACTIVE; + vgpu_vreg(vgpu, offset) |= PIPECONF_STATE_ENABLE; vgpu_update_refresh_rate(vgpu); vgpu_update_vblank_emulation(vgpu, true); } else { - vgpu_vreg(vgpu, offset) &= ~I965_PIPECONF_ACTIVE; + vgpu_vreg(vgpu, offset) &= ~PIPECONF_STATE_ENABLE; vgpu_update_vblank_emulation(vgpu, false); } return 0; @@ -2647,12 +2652,12 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(_MMIO(_TRANSA_CHICKEN2), D_ALL); MMIO_D(_MMIO(_TRANSB_CHICKEN2), D_ALL); - MMIO_D(ILK_DPFC_CB_BASE, D_ALL); - MMIO_D(ILK_DPFC_CONTROL, D_ALL); - MMIO_D(ILK_DPFC_RECOMP_CTL, D_ALL); - MMIO_D(ILK_DPFC_STATUS, D_ALL); - MMIO_D(ILK_DPFC_FENCE_YOFF, D_ALL); - MMIO_D(ILK_DPFC_CHICKEN, D_ALL); + MMIO_D(ILK_DPFC_CB_BASE(INTEL_FBC_A), D_ALL); + MMIO_D(ILK_DPFC_CONTROL(INTEL_FBC_A), D_ALL); + MMIO_D(ILK_DPFC_RECOMP_CTL(INTEL_FBC_A), D_ALL); + MMIO_D(ILK_DPFC_STATUS(INTEL_FBC_A), D_ALL); + MMIO_D(ILK_DPFC_FENCE_YOFF(INTEL_FBC_A), D_ALL); + MMIO_D(ILK_DPFC_CHICKEN(INTEL_FBC_A), D_ALL); MMIO_D(ILK_FBC_RT_BASE, D_ALL); MMIO_D(IPS_CTL, D_ALL); @@ -2876,9 +2881,9 @@ static int init_generic_mmio_info(struct intel_gvt *gvt) MMIO_D(_MMIO(0x3c), D_ALL); MMIO_D(_MMIO(0x860), D_ALL); - MMIO_D(ECOSKPD, D_ALL); + MMIO_D(ECOSKPD(RENDER_RING_BASE), D_ALL); MMIO_D(_MMIO(0x121d0), D_ALL); - MMIO_D(GEN6_BLITTER_ECOSKPD, D_ALL); + MMIO_D(ECOSKPD(BLT_RING_BASE), D_ALL); MMIO_D(_MMIO(0x41d0), D_ALL); MMIO_D(GAC_ECO_BITS, D_ALL); MMIO_D(_MMIO(0x6200), D_ALL); @@ -3436,6 +3441,7 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_DFH(GAMT_CHKN_BIT_REG, D_KBL | D_CFL, F_CMD_ACCESS, NULL, NULL); MMIO_D(GEN9_CTX_PREEMPT_REG, D_SKL_PLUS & ~D_BXT); + MMIO_DFH(_MMIO(0xe4cc), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); return 0; } @@ -3627,11 +3633,11 @@ static int init_bxt_mmio_info(struct intel_gvt *gvt) return 0; } -static struct gvt_mmio_block *find_mmio_block(struct intel_gvt *gvt, - unsigned int offset) +static const struct gvt_mmio_block *find_mmio_block(struct intel_gvt *gvt, + unsigned int offset) { unsigned long device = intel_gvt_get_device_type(gvt); - struct gvt_mmio_block *block = gvt->mmio.mmio_block; + const struct gvt_mmio_block *block = gvt->mmio.mmio_block; int num = gvt->mmio.num_mmio_block; int i; @@ -3670,7 +3676,7 @@ void intel_gvt_clean_mmio_info(struct intel_gvt *gvt) * accessible (should have no F_CMD_ACCESS flag). * otherwise, need to update cmd_reg_handler in cmd_parser.c */ -static struct gvt_mmio_block mmio_blocks[] = { +static const struct gvt_mmio_block mmio_blocks[] = { {D_SKL_PLUS, _MMIO(DMC_MMIO_START_RANGE), 0x3000, NULL, NULL}, {D_ALL, _MMIO(MCHBAR_MIRROR_BASE_SNB), 0x40000, NULL, NULL}, {D_ALL, _MMIO(VGT_PVINFO_PAGE), VGT_PVINFO_SIZE, @@ -3753,7 +3759,7 @@ int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt, int (*handler)(struct intel_gvt *gvt, u32 offset, void *data), void *data) { - struct gvt_mmio_block *block = gvt->mmio.mmio_block; + const struct gvt_mmio_block *block = gvt->mmio.mmio_block; struct intel_gvt_mmio_info *e; int i, j, ret; @@ -3871,7 +3877,7 @@ int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset, struct drm_i915_private *i915 = vgpu->gvt->gt->i915; struct intel_gvt *gvt = vgpu->gvt; struct intel_gvt_mmio_info *mmio_info; - struct gvt_mmio_block *mmio_block; + const struct gvt_mmio_block *mmio_block; gvt_mmio_func func; int ret; diff --git a/drivers/gpu/drm/i915/gvt/interrupt.c b/drivers/gpu/drm/i915/gvt/interrupt.c index 614b951d919f..228f623d466d 100644 --- a/drivers/gpu/drm/i915/gvt/interrupt.c +++ b/drivers/gpu/drm/i915/gvt/interrupt.c @@ -30,6 +30,7 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "gvt.h" #include "trace.h" @@ -176,7 +177,7 @@ int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu, unsigned int reg, void *p_data, unsigned int bytes) { struct intel_gvt *gvt = vgpu->gvt; - struct intel_gvt_irq_ops *ops = gvt->irq.ops; + const struct intel_gvt_irq_ops *ops = gvt->irq.ops; u32 imr = *(u32 *)p_data; trace_write_ir(vgpu->id, "IMR", reg, imr, vgpu_vreg(vgpu, reg), @@ -206,7 +207,7 @@ int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu, unsigned int reg, void *p_data, unsigned int bytes) { struct intel_gvt *gvt = vgpu->gvt; - struct intel_gvt_irq_ops *ops = gvt->irq.ops; + const struct intel_gvt_irq_ops *ops = gvt->irq.ops; u32 ier = *(u32 *)p_data; u32 virtual_ier = vgpu_vreg(vgpu, reg); @@ -246,7 +247,7 @@ int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu, { struct intel_gvt *gvt = vgpu->gvt; struct drm_i915_private *i915 = gvt->gt->i915; - struct intel_gvt_irq_ops *ops = gvt->irq.ops; + const struct intel_gvt_irq_ops *ops = gvt->irq.ops; struct intel_gvt_irq_info *info; u32 ier = *(u32 *)p_data; @@ -604,7 +605,7 @@ static void gen8_init_irq( SET_BIT_INFO(irq, 25, PCU_PCODE2DRIVER_MAILBOX, INTEL_GVT_IRQ_INFO_PCU); } -static struct intel_gvt_irq_ops gen8_irq_ops = { +static const struct intel_gvt_irq_ops gen8_irq_ops = { .init_irq = gen8_init_irq, .check_pending_irq = gen8_check_pending_irq, }; @@ -626,7 +627,7 @@ void intel_vgpu_trigger_virtual_event(struct intel_vgpu *vgpu, struct intel_gvt *gvt = vgpu->gvt; struct intel_gvt_irq *irq = &gvt->irq; gvt_event_virt_handler_t handler; - struct intel_gvt_irq_ops *ops = gvt->irq.ops; + const struct intel_gvt_irq_ops *ops = gvt->irq.ops; handler = get_event_virt_handler(irq, event); drm_WARN_ON(&i915->drm, !handler); diff --git a/drivers/gpu/drm/i915/gvt/interrupt.h b/drivers/gpu/drm/i915/gvt/interrupt.h index 6c47d3e33161..b62f04ab47cb 100644 --- a/drivers/gpu/drm/i915/gvt/interrupt.h +++ b/drivers/gpu/drm/i915/gvt/interrupt.h @@ -35,7 +35,7 @@ #include <linux/hrtimer.h> #include <linux/kernel.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" enum intel_gvt_event_type { RCS_MI_USER_INTERRUPT = 0, @@ -203,7 +203,7 @@ struct intel_gvt_irq_map { /* structure containing device specific IRQ state */ struct intel_gvt_irq { - struct intel_gvt_irq_ops *ops; + const struct intel_gvt_irq_ops *ops; struct intel_gvt_irq_info *info[INTEL_GVT_IRQ_INFO_MAX]; DECLARE_BITMAP(irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX); struct intel_gvt_event_info events[INTEL_GVT_EVENT_MAX]; diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 20b82fb036f8..057ec4490104 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -46,6 +46,8 @@ #include <linux/nospec.h> +#include <drm/drm_edid.h> + #include "i915_drv.h" #include "gvt.h" @@ -186,14 +188,29 @@ static ssize_t description_show(struct mdev_type *mtype, type->weight); } +static ssize_t name_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, char *buf) +{ + struct intel_vgpu_type *type; + struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; + + type = &gvt->types[mtype_get_type_group_id(mtype)]; + if (!type) + return 0; + + return sprintf(buf, "%s\n", type->name); +} + static MDEV_TYPE_ATTR_RO(available_instances); static MDEV_TYPE_ATTR_RO(device_api); static MDEV_TYPE_ATTR_RO(description); +static MDEV_TYPE_ATTR_RO(name); static struct attribute *gvt_type_attrs[] = { &mdev_type_attr_available_instances.attr, &mdev_type_attr_device_api.attr, &mdev_type_attr_description.attr, + &mdev_type_attr_name.attr, NULL, }; diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c index 24210b1eaec5..5db0ef83d522 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.c +++ b/drivers/gpu/drm/i915/gvt/mmio.c @@ -34,8 +34,11 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "gvt.h" +#include "gt/intel_gt_regs.h" + /** * intel_vgpu_gpa_to_mmio_offset - translate a GPA to MMIO offset * @vgpu: a vGPU diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c index f776c470914d..c85bafe7539e 100644 --- a/drivers/gpu/drm/i915/gvt/mmio_context.c +++ b/drivers/gpu/drm/i915/gvt/mmio_context.c @@ -35,7 +35,9 @@ #include "i915_drv.h" #include "gt/intel_context.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_gpu_commands.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_ring.h" #include "gvt.h" #include "trace.h" @@ -44,7 +46,7 @@ /* Raw offset is appened to each line for convenience. */ static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = { - {RCS0, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */ + {RCS0, RING_MODE_GEN7(RENDER_RING_BASE), 0xffff, false}, /* 0x229c */ {RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */ {RCS0, HWSTAM, 0x0, false}, /* 0x2098 */ {RCS0, INSTPM, 0xffff, true}, /* 0x20c0 */ @@ -76,7 +78,7 @@ static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = { }; static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = { - {RCS0, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */ + {RCS0, RING_MODE_GEN7(RENDER_RING_BASE), 0xffff, false}, /* 0x229c */ {RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */ {RCS0, HWSTAM, 0x0, false}, /* 0x2098 */ {RCS0, INSTPM, 0xffff, true}, /* 0x20c0 */ diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.h b/drivers/gpu/drm/i915/gvt/mmio_context.h index b6b69777af49..9540813b88e5 100644 --- a/drivers/gpu/drm/i915/gvt/mmio_context.h +++ b/drivers/gpu/drm/i915/gvt/mmio_context.h @@ -38,9 +38,9 @@ #include <linux/types.h> +#include "gt/intel_engine_regs.h" #include "gt/intel_engine_types.h" #include "gt/intel_lrc_reg.h" -#include "i915_reg.h" struct i915_request; struct intel_context; diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h index 244cc7320b54..7d666d34f9ff 100644 --- a/drivers/gpu/drm/i915/gvt/reg.h +++ b/drivers/gpu/drm/i915/gvt/reg.h @@ -62,7 +62,6 @@ #define SKL_FLIP_EVENT(pipe, plane) (PRIMARY_A_FLIP_DONE + (plane) * 3 + (pipe)) -#define PLANE_CTL_ASYNC_FLIP (1 << 9) #define REG50080_FLIP_TYPE_MASK 0x3 #define REG50080_FLIP_TYPE_ASYNC 0x1 diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c index 036b74fe9298..c077fb4674f0 100644 --- a/drivers/gpu/drm/i915/gvt/sched_policy.c +++ b/drivers/gpu/drm/i915/gvt/sched_policy.c @@ -368,7 +368,7 @@ static void tbs_sched_stop_schedule(struct intel_vgpu *vgpu) vgpu_data->active = false; } -static struct intel_gvt_sched_policy_ops tbs_schedule_ops = { +static const struct intel_gvt_sched_policy_ops tbs_schedule_ops = { .init = tbs_sched_init, .clean = tbs_sched_clean, .init_vgpu = tbs_sched_init_vgpu, diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index 42a0c9ae0a73..679476da0640 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -38,11 +38,13 @@ #include "gem/i915_gem_pm.h" #include "gt/intel_context.h" #include "gt/intel_execlists_submission.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_lrc.h" #include "gt/intel_ring.h" #include "i915_drv.h" #include "i915_gem_gtt.h" +#include "i915_perf_oa_regs.h" #include "gvt.h" #define RING_CTX_OFF(x) \ diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h index 7c86984a842f..1f391b3da2cc 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.h +++ b/drivers/gpu/drm/i915/gvt/scheduler.h @@ -56,7 +56,7 @@ struct intel_gvt_workload_scheduler { wait_queue_head_t waitq[I915_NUM_ENGINES]; void *sched_data; - struct intel_gvt_sched_policy_ops *sched_ops; + const struct intel_gvt_sched_policy_ops *sched_ops; }; #define INDIRECT_CTX_ADDR_MASK 0xffffffc0 diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index fa6b92615799..8dddd0a940a1 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -77,7 +77,7 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu) #define VGPU_WEIGHT(vgpu_num) \ (VGPU_MAX_WEIGHT / (vgpu_num)) -static struct { +static const struct { unsigned int low_mm; unsigned int high_mm; unsigned int fence; @@ -88,7 +88,7 @@ static struct { */ unsigned int weight; enum intel_vgpu_edid edid; - char *name; + const char *name; } vgpu_types[] = { /* Fixed vGPU type table */ { MB_TO_BYTES(64), MB_TO_BYTES(384), 4, VGPU_WEIGHT(8), GVT_EDID_1024_768, "8" }, diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c deleted file mode 100644 index 6e2ad68f8f3f..000000000000 --- a/drivers/gpu/drm/i915/i915_buddy.c +++ /dev/null @@ -1,466 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include <linux/kmemleak.h> -#include <linux/sizes.h> - -#include "i915_buddy.h" - -#include "i915_gem.h" -#include "i915_utils.h" - -static struct kmem_cache *slab_blocks; - -static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm, - struct i915_buddy_block *parent, - unsigned int order, - u64 offset) -{ - struct i915_buddy_block *block; - - GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER); - - block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL); - if (!block) - return NULL; - - block->header = offset; - block->header |= order; - block->parent = parent; - - GEM_BUG_ON(block->header & I915_BUDDY_HEADER_UNUSED); - return block; -} - -static void i915_block_free(struct i915_buddy_mm *mm, - struct i915_buddy_block *block) -{ - kmem_cache_free(slab_blocks, block); -} - -static void mark_allocated(struct i915_buddy_block *block) -{ - block->header &= ~I915_BUDDY_HEADER_STATE; - block->header |= I915_BUDDY_ALLOCATED; - - list_del(&block->link); -} - -static void mark_free(struct i915_buddy_mm *mm, - struct i915_buddy_block *block) -{ - block->header &= ~I915_BUDDY_HEADER_STATE; - block->header |= I915_BUDDY_FREE; - - list_add(&block->link, - &mm->free_list[i915_buddy_block_order(block)]); -} - -static void mark_split(struct i915_buddy_block *block) -{ - block->header &= ~I915_BUDDY_HEADER_STATE; - block->header |= I915_BUDDY_SPLIT; - - list_del(&block->link); -} - -int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size) -{ - unsigned int i; - u64 offset; - - if (size < chunk_size) - return -EINVAL; - - if (chunk_size < PAGE_SIZE) - return -EINVAL; - - if (!is_power_of_2(chunk_size)) - return -EINVAL; - - size = round_down(size, chunk_size); - - mm->size = size; - mm->avail = size; - mm->chunk_size = chunk_size; - mm->max_order = ilog2(size) - ilog2(chunk_size); - - GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER); - - mm->free_list = kmalloc_array(mm->max_order + 1, - sizeof(struct list_head), - GFP_KERNEL); - if (!mm->free_list) - return -ENOMEM; - - for (i = 0; i <= mm->max_order; ++i) - INIT_LIST_HEAD(&mm->free_list[i]); - - mm->n_roots = hweight64(size); - - mm->roots = kmalloc_array(mm->n_roots, - sizeof(struct i915_buddy_block *), - GFP_KERNEL); - if (!mm->roots) - goto out_free_list; - - offset = 0; - i = 0; - - /* - * Split into power-of-two blocks, in case we are given a size that is - * not itself a power-of-two. - */ - do { - struct i915_buddy_block *root; - unsigned int order; - u64 root_size; - - root_size = rounddown_pow_of_two(size); - order = ilog2(root_size) - ilog2(chunk_size); - - root = i915_block_alloc(mm, NULL, order, offset); - if (!root) - goto out_free_roots; - - mark_free(mm, root); - - GEM_BUG_ON(i > mm->max_order); - GEM_BUG_ON(i915_buddy_block_size(mm, root) < chunk_size); - - mm->roots[i] = root; - - offset += root_size; - size -= root_size; - i++; - } while (size); - - return 0; - -out_free_roots: - while (i--) - i915_block_free(mm, mm->roots[i]); - kfree(mm->roots); -out_free_list: - kfree(mm->free_list); - return -ENOMEM; -} - -void i915_buddy_fini(struct i915_buddy_mm *mm) -{ - int i; - - for (i = 0; i < mm->n_roots; ++i) { - GEM_WARN_ON(!i915_buddy_block_is_free(mm->roots[i])); - i915_block_free(mm, mm->roots[i]); - } - - GEM_WARN_ON(mm->avail != mm->size); - - kfree(mm->roots); - kfree(mm->free_list); -} - -static int split_block(struct i915_buddy_mm *mm, - struct i915_buddy_block *block) -{ - unsigned int block_order = i915_buddy_block_order(block) - 1; - u64 offset = i915_buddy_block_offset(block); - - GEM_BUG_ON(!i915_buddy_block_is_free(block)); - GEM_BUG_ON(!i915_buddy_block_order(block)); - - block->left = i915_block_alloc(mm, block, block_order, offset); - if (!block->left) - return -ENOMEM; - - block->right = i915_block_alloc(mm, block, block_order, - offset + (mm->chunk_size << block_order)); - if (!block->right) { - i915_block_free(mm, block->left); - return -ENOMEM; - } - - mark_free(mm, block->left); - mark_free(mm, block->right); - - mark_split(block); - - return 0; -} - -static struct i915_buddy_block * -get_buddy(struct i915_buddy_block *block) -{ - struct i915_buddy_block *parent; - - parent = block->parent; - if (!parent) - return NULL; - - if (parent->left == block) - return parent->right; - - return parent->left; -} - -static void __i915_buddy_free(struct i915_buddy_mm *mm, - struct i915_buddy_block *block) -{ - struct i915_buddy_block *parent; - - while ((parent = block->parent)) { - struct i915_buddy_block *buddy; - - buddy = get_buddy(block); - - if (!i915_buddy_block_is_free(buddy)) - break; - - list_del(&buddy->link); - - i915_block_free(mm, block); - i915_block_free(mm, buddy); - - block = parent; - } - - mark_free(mm, block); -} - -void i915_buddy_free(struct i915_buddy_mm *mm, - struct i915_buddy_block *block) -{ - GEM_BUG_ON(!i915_buddy_block_is_allocated(block)); - mm->avail += i915_buddy_block_size(mm, block); - __i915_buddy_free(mm, block); -} - -void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects) -{ - struct i915_buddy_block *block, *on; - - list_for_each_entry_safe(block, on, objects, link) { - i915_buddy_free(mm, block); - cond_resched(); - } - INIT_LIST_HEAD(objects); -} - -/* - * Allocate power-of-two block. The order value here translates to: - * - * 0 = 2^0 * mm->chunk_size - * 1 = 2^1 * mm->chunk_size - * 2 = 2^2 * mm->chunk_size - * ... - */ -struct i915_buddy_block * -i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order) -{ - struct i915_buddy_block *block = NULL; - unsigned int i; - int err; - - for (i = order; i <= mm->max_order; ++i) { - block = list_first_entry_or_null(&mm->free_list[i], - struct i915_buddy_block, - link); - if (block) - break; - } - - if (!block) - return ERR_PTR(-ENOSPC); - - GEM_BUG_ON(!i915_buddy_block_is_free(block)); - - while (i != order) { - err = split_block(mm, block); - if (unlikely(err)) - goto out_free; - - /* Go low */ - block = block->left; - i--; - } - - mark_allocated(block); - mm->avail -= i915_buddy_block_size(mm, block); - kmemleak_update_trace(block); - return block; - -out_free: - if (i != order) - __i915_buddy_free(mm, block); - return ERR_PTR(err); -} - -static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) -{ - return s1 <= e2 && e1 >= s2; -} - -static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) -{ - return s1 <= s2 && e1 >= e2; -} - -/* - * Allocate range. Note that it's safe to chain together multiple alloc_ranges - * with the same blocks list. - * - * Intended for pre-allocating portions of the address space, for example to - * reserve a block for the initial framebuffer or similar, hence the expectation - * here is that i915_buddy_alloc() is still the main vehicle for - * allocations, so if that's not the case then the drm_mm range allocator is - * probably a much better fit, and so you should probably go use that instead. - */ -int i915_buddy_alloc_range(struct i915_buddy_mm *mm, - struct list_head *blocks, - u64 start, u64 size) -{ - struct i915_buddy_block *block; - struct i915_buddy_block *buddy; - LIST_HEAD(allocated); - LIST_HEAD(dfs); - u64 end; - int err; - int i; - - if (size < mm->chunk_size) - return -EINVAL; - - if (!IS_ALIGNED(size | start, mm->chunk_size)) - return -EINVAL; - - if (range_overflows(start, size, mm->size)) - return -EINVAL; - - for (i = 0; i < mm->n_roots; ++i) - list_add_tail(&mm->roots[i]->tmp_link, &dfs); - - end = start + size - 1; - - do { - u64 block_start; - u64 block_end; - - block = list_first_entry_or_null(&dfs, - struct i915_buddy_block, - tmp_link); - if (!block) - break; - - list_del(&block->tmp_link); - - block_start = i915_buddy_block_offset(block); - block_end = block_start + i915_buddy_block_size(mm, block) - 1; - - if (!overlaps(start, end, block_start, block_end)) - continue; - - if (i915_buddy_block_is_allocated(block)) { - err = -ENOSPC; - goto err_free; - } - - if (contains(start, end, block_start, block_end)) { - if (!i915_buddy_block_is_free(block)) { - err = -ENOSPC; - goto err_free; - } - - mark_allocated(block); - mm->avail -= i915_buddy_block_size(mm, block); - list_add_tail(&block->link, &allocated); - continue; - } - - if (!i915_buddy_block_is_split(block)) { - err = split_block(mm, block); - if (unlikely(err)) - goto err_undo; - } - - list_add(&block->right->tmp_link, &dfs); - list_add(&block->left->tmp_link, &dfs); - } while (1); - - list_splice_tail(&allocated, blocks); - return 0; - -err_undo: - /* - * We really don't want to leave around a bunch of split blocks, since - * bigger is better, so make sure we merge everything back before we - * free the allocated blocks. - */ - buddy = get_buddy(block); - if (buddy && - (i915_buddy_block_is_free(block) && - i915_buddy_block_is_free(buddy))) - __i915_buddy_free(mm, block); - -err_free: - i915_buddy_free_list(mm, &allocated); - return err; -} - -void i915_buddy_block_print(struct i915_buddy_mm *mm, - struct i915_buddy_block *block, - struct drm_printer *p) -{ - u64 start = i915_buddy_block_offset(block); - u64 size = i915_buddy_block_size(mm, block); - - drm_printf(p, "%#018llx-%#018llx: %llu\n", start, start + size, size); -} - -void i915_buddy_print(struct i915_buddy_mm *mm, struct drm_printer *p) -{ - int order; - - drm_printf(p, "chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB\n", - mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20); - - for (order = mm->max_order; order >= 0; order--) { - struct i915_buddy_block *block; - u64 count = 0, free; - - list_for_each_entry(block, &mm->free_list[order], link) { - GEM_BUG_ON(!i915_buddy_block_is_free(block)); - count++; - } - - drm_printf(p, "order-%d ", order); - - free = count * (mm->chunk_size << order); - if (free < SZ_1M) - drm_printf(p, "free: %lluKiB", free >> 10); - else - drm_printf(p, "free: %lluMiB", free >> 20); - - drm_printf(p, ", pages: %llu\n", count); - } -} - -#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) -#include "selftests/i915_buddy.c" -#endif - -void i915_buddy_module_exit(void) -{ - kmem_cache_destroy(slab_blocks); -} - -int __init i915_buddy_module_init(void) -{ - slab_blocks = KMEM_CACHE(i915_buddy_block, 0); - if (!slab_blocks) - return -ENOMEM; - - return 0; -} diff --git a/drivers/gpu/drm/i915/i915_buddy.h b/drivers/gpu/drm/i915/i915_buddy.h deleted file mode 100644 index 7077742112ac..000000000000 --- a/drivers/gpu/drm/i915/i915_buddy.h +++ /dev/null @@ -1,143 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2021 Intel Corporation - */ - -#ifndef __I915_BUDDY_H__ -#define __I915_BUDDY_H__ - -#include <linux/bitops.h> -#include <linux/list.h> -#include <linux/slab.h> - -#include <drm/drm_print.h> - -struct i915_buddy_block { -#define I915_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) -#define I915_BUDDY_HEADER_STATE GENMASK_ULL(11, 10) -#define I915_BUDDY_ALLOCATED (1 << 10) -#define I915_BUDDY_FREE (2 << 10) -#define I915_BUDDY_SPLIT (3 << 10) -/* Free to be used, if needed in the future */ -#define I915_BUDDY_HEADER_UNUSED GENMASK_ULL(9, 6) -#define I915_BUDDY_HEADER_ORDER GENMASK_ULL(5, 0) - u64 header; - - struct i915_buddy_block *left; - struct i915_buddy_block *right; - struct i915_buddy_block *parent; - - void *private; /* owned by creator */ - - /* - * While the block is allocated by the user through i915_buddy_alloc*, - * the user has ownership of the link, for example to maintain within - * a list, if so desired. As soon as the block is freed with - * i915_buddy_free* ownership is given back to the mm. - */ - struct list_head link; - struct list_head tmp_link; -}; - -/* Order-zero must be at least PAGE_SIZE */ -#define I915_BUDDY_MAX_ORDER (63 - PAGE_SHIFT) - -/* - * Binary Buddy System. - * - * Locking should be handled by the user, a simple mutex around - * i915_buddy_alloc* and i915_buddy_free* should suffice. - */ -struct i915_buddy_mm { - /* Maintain a free list for each order. */ - struct list_head *free_list; - - /* - * Maintain explicit binary tree(s) to track the allocation of the - * address space. This gives us a simple way of finding a buddy block - * and performing the potentially recursive merge step when freeing a - * block. Nodes are either allocated or free, in which case they will - * also exist on the respective free list. - */ - struct i915_buddy_block **roots; - - /* - * Anything from here is public, and remains static for the lifetime of - * the mm. Everything above is considered do-not-touch. - */ - unsigned int n_roots; - unsigned int max_order; - - /* Must be at least PAGE_SIZE */ - u64 chunk_size; - u64 size; - u64 avail; -}; - -static inline u64 -i915_buddy_block_offset(struct i915_buddy_block *block) -{ - return block->header & I915_BUDDY_HEADER_OFFSET; -} - -static inline unsigned int -i915_buddy_block_order(struct i915_buddy_block *block) -{ - return block->header & I915_BUDDY_HEADER_ORDER; -} - -static inline unsigned int -i915_buddy_block_state(struct i915_buddy_block *block) -{ - return block->header & I915_BUDDY_HEADER_STATE; -} - -static inline bool -i915_buddy_block_is_allocated(struct i915_buddy_block *block) -{ - return i915_buddy_block_state(block) == I915_BUDDY_ALLOCATED; -} - -static inline bool -i915_buddy_block_is_free(struct i915_buddy_block *block) -{ - return i915_buddy_block_state(block) == I915_BUDDY_FREE; -} - -static inline bool -i915_buddy_block_is_split(struct i915_buddy_block *block) -{ - return i915_buddy_block_state(block) == I915_BUDDY_SPLIT; -} - -static inline u64 -i915_buddy_block_size(struct i915_buddy_mm *mm, - struct i915_buddy_block *block) -{ - return mm->chunk_size << i915_buddy_block_order(block); -} - -int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size); - -void i915_buddy_fini(struct i915_buddy_mm *mm); - -struct i915_buddy_block * -i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order); - -int i915_buddy_alloc_range(struct i915_buddy_mm *mm, - struct list_head *blocks, - u64 start, u64 size); - -void i915_buddy_free(struct i915_buddy_mm *mm, struct i915_buddy_block *block); - -void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects); - -void i915_buddy_print(struct i915_buddy_mm *mm, struct drm_printer *p); -void i915_buddy_block_print(struct i915_buddy_mm *mm, - struct i915_buddy_block *block, - struct drm_printer *p); - -void i915_buddy_module_exit(void); -int i915_buddy_module_init(void); - -#endif diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index e0403ce9ce69..5f6e41636655 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -25,11 +25,17 @@ * */ +#include <drm/drm_cache.h> + #include "gt/intel_engine.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_gpu_commands.h" +#include "gt/intel_gt_regs.h" +#include "i915_cmd_parser.h" #include "i915_drv.h" #include "i915_memcpy.h" +#include "i915_reg.h" /** * DOC: batch buffer command parser @@ -591,6 +597,10 @@ struct drm_i915_reg_descriptor { { .addr = _reg(idx) }, \ { .addr = _reg ## _UDW(idx) } +#define REG64_BASE_IDX(_reg, base, idx) \ + { .addr = _reg(base, idx) }, \ + { .addr = _reg ## _UDW(base, idx) } + static const struct drm_i915_reg_descriptor gen7_render_regs[] = { REG64(GPGPU_THREADS_DISPATCHED), REG64(HS_INVOCATION_COUNT), @@ -605,8 +615,8 @@ static const struct drm_i915_reg_descriptor gen7_render_regs[] = { REG64(PS_INVOCATION_COUNT), REG64(PS_DEPTH_COUNT), REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE), - REG64(MI_PREDICATE_SRC0), - REG64(MI_PREDICATE_SRC1), + REG64_IDX(MI_PREDICATE_SRC0, RENDER_RING_BASE), + REG64_IDX(MI_PREDICATE_SRC1, RENDER_RING_BASE), REG32(GEN7_3DPRIM_END_OFFSET), REG32(GEN7_3DPRIM_START_VERTEX), REG32(GEN7_3DPRIM_VERTEX_COUNT), @@ -636,22 +646,22 @@ static const struct drm_i915_reg_descriptor gen7_render_regs[] = { }; static const struct drm_i915_reg_descriptor hsw_render_regs[] = { - REG64_IDX(HSW_CS_GPR, 0), - REG64_IDX(HSW_CS_GPR, 1), - REG64_IDX(HSW_CS_GPR, 2), - REG64_IDX(HSW_CS_GPR, 3), - REG64_IDX(HSW_CS_GPR, 4), - REG64_IDX(HSW_CS_GPR, 5), - REG64_IDX(HSW_CS_GPR, 6), - REG64_IDX(HSW_CS_GPR, 7), - REG64_IDX(HSW_CS_GPR, 8), - REG64_IDX(HSW_CS_GPR, 9), - REG64_IDX(HSW_CS_GPR, 10), - REG64_IDX(HSW_CS_GPR, 11), - REG64_IDX(HSW_CS_GPR, 12), - REG64_IDX(HSW_CS_GPR, 13), - REG64_IDX(HSW_CS_GPR, 14), - REG64_IDX(HSW_CS_GPR, 15), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 0), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 1), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 2), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 3), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 4), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 5), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 6), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 7), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 8), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 9), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 10), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 11), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 12), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 13), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 14), + REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 15), REG32(HSW_SCRATCH1, .mask = ~HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE, .value = 0), @@ -674,22 +684,22 @@ static const struct drm_i915_reg_descriptor gen9_blt_regs[] = { REG32(BCS_SWCTRL), REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE), REG32_IDX(RING_CTX_TIMESTAMP, BLT_RING_BASE), - REG64_IDX(BCS_GPR, 0), - REG64_IDX(BCS_GPR, 1), - REG64_IDX(BCS_GPR, 2), - REG64_IDX(BCS_GPR, 3), - REG64_IDX(BCS_GPR, 4), - REG64_IDX(BCS_GPR, 5), - REG64_IDX(BCS_GPR, 6), - REG64_IDX(BCS_GPR, 7), - REG64_IDX(BCS_GPR, 8), - REG64_IDX(BCS_GPR, 9), - REG64_IDX(BCS_GPR, 10), - REG64_IDX(BCS_GPR, 11), - REG64_IDX(BCS_GPR, 12), - REG64_IDX(BCS_GPR, 13), - REG64_IDX(BCS_GPR, 14), - REG64_IDX(BCS_GPR, 15), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 0), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 1), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 2), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 3), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 4), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 5), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 6), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 7), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 8), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 9), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 10), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 11), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 12), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 13), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 14), + REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 15), }; #undef REG64 diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.h b/drivers/gpu/drm/i915/i915_cmd_parser.h new file mode 100644 index 000000000000..ba70ac6c97cd --- /dev/null +++ b/drivers/gpu/drm/i915/i915_cmd_parser.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_CMD_PARSER_H__ +#define __I915_CMD_PARSER_H__ + +#include <linux/types.h> + +struct drm_i915_private; +struct intel_engine_cs; +struct i915_vma; + +int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv); +int intel_engine_init_cmd_parser(struct intel_engine_cs *engine); +void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine); +int intel_engine_cmd_parser(struct intel_engine_cs *engine, + struct i915_vma *batch, + unsigned long batch_offset, + unsigned long batch_length, + struct i915_vma *shadow, + bool trampoline); +#define I915_CMD_PARSER_TRAMPOLINE_SIZE 8 + +#endif /* __I915_CMD_PARSER_H__ */ diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index e0e052cdf8b8..946bbe57bfe5 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -38,6 +38,7 @@ #include "gt/intel_gt_debugfs.h" #include "gt/intel_gt_pm.h" #include "gt/intel_gt_pm_debugfs.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_gt_requests.h" #include "gt/intel_rc6.h" #include "gt/intel_reset.h" @@ -48,6 +49,7 @@ #include "i915_debugfs_params.h" #include "i915_irq.h" #include "i915_scheduler.h" +#include "intel_mchbar_regs.h" #include "intel_pm.h" static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node) @@ -136,6 +138,17 @@ static const char *stringify_vma_type(const struct i915_vma *vma) return "ppgtt"; } +static const char *i915_cache_level_str(struct drm_i915_private *i915, int type) +{ + switch (type) { + case I915_CACHE_NONE: return " uncached"; + case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped"; + case I915_CACHE_L3_LLC: return " L3+LLC"; + case I915_CACHE_WT: return " WT"; + default: return ""; + } +} + void i915_debugfs_describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) { @@ -170,7 +183,8 @@ i915_debugfs_describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) seq_printf(m, " (%s offset: %08llx, size: %08llx, pages: %s", stringify_vma_type(vma), vma->node.start, vma->node.size, - stringify_page_sizes(vma->page_sizes.gtt, NULL, 0)); + stringify_page_sizes(vma->resource->page_sizes_gtt, + NULL, 0)); if (i915_vma_is_ggtt(vma) || i915_vma_is_dpt(vma)) { switch (vma->ggtt_view.type) { case I915_GGTT_VIEW_NORMAL: @@ -390,9 +404,9 @@ static int i915_swizzle_info(struct seq_file *m, void *data) intel_wakeref_t wakeref; seq_printf(m, "bit6 swizzle for X-tiling = %s\n", - swizzle_string(dev_priv->ggtt.bit_6_swizzle_x)); + swizzle_string(to_gt(dev_priv)->ggtt->bit_6_swizzle_x)); seq_printf(m, "bit6 swizzle for Y-tiling = %s\n", - swizzle_string(dev_priv->ggtt.bit_6_swizzle_y)); + swizzle_string(to_gt(dev_priv)->ggtt->bit_6_swizzle_y)); if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) seq_puts(m, "L-shaped memory detected\n"); diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c index 95174938b160..62b3f332bbf5 100644 --- a/drivers/gpu/drm/i915/i915_driver.c +++ b/drivers/gpu/drm/i915/i915_driver.c @@ -62,6 +62,8 @@ #include "display/intel_vga.h" #include "gem/i915_gem_context.h" +#include "gem/i915_gem_create.h" +#include "gem/i915_gem_dmabuf.h" #include "gem/i915_gem_ioctls.h" #include "gem/i915_gem_mman.h" #include "gem/i915_gem_pm.h" @@ -71,10 +73,13 @@ #include "pxp/intel_pxp_pm.h" +#include "i915_file_private.h" #include "i915_debugfs.h" #include "i915_driver.h" #include "i915_drv.h" +#include "i915_getparam.h" #include "i915_ioc32.h" +#include "i915_ioctl.h" #include "i915_irq.h" #include "i915_memcpy.h" #include "i915_perf.h" @@ -86,6 +91,7 @@ #include "intel_dram.h" #include "intel_gvt.h" #include "intel_memory_region.h" +#include "intel_pci_config.h" #include "intel_pcode.h" #include "intel_pm.h" #include "intel_region_ttm.h" @@ -571,6 +577,10 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) i915_perf_init(dev_priv); + ret = intel_gt_assign_ggtt(to_gt(dev_priv)); + if (ret) + goto err_perf; + ret = i915_ggtt_probe_hw(dev_priv); if (ret) goto err_perf; @@ -587,8 +597,6 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) if (ret) goto err_ggtt; - intel_gt_init_hw_early(to_gt(dev_priv), &dev_priv->ggtt); - ret = intel_gt_probe_lmem(to_gt(dev_priv)); if (ret) goto err_mem_regions; @@ -827,21 +835,6 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (!i915->params.nuclear_pageflip && match_info->graphics.ver < 5) i915->drm.driver_features &= ~DRIVER_ATOMIC; - /* - * Check if we support fake LMEM -- for now we only unleash this for - * the live selftests(test-and-exit). - */ -#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) - if (IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) { - if (GRAPHICS_VER(i915) >= 9 && i915_selftest.live < 0 && - i915->params.fake_lmem_start) { - mkwrite_device_info(i915)->memory_regions = - REGION_SMEM | REGION_LMEM | REGION_STOLEN_SMEM; - GEM_BUG_ON(!HAS_LMEM(i915)); - } - } -#endif - ret = pci_enable_device(pdev); if (ret) goto out_fini; @@ -1146,7 +1139,7 @@ static int i915_drm_suspend(struct drm_device *dev) /* Must be called before GGTT is suspended. */ intel_dpt_suspend(dev_priv); - i915_ggtt_suspend(&dev_priv->ggtt); + i915_ggtt_suspend(to_gt(dev_priv)->ggtt); i915_save_display(dev_priv); @@ -1270,7 +1263,7 @@ static int i915_drm_resume(struct drm_device *dev) if (ret) drm_err(&dev_priv->drm, "failed to re-enable GGTT\n"); - i915_ggtt_resume(&dev_priv->ggtt); + i915_ggtt_resume(to_gt(dev_priv)->ggtt); /* Must be called after GGTT is resumed. */ intel_dpt_resume(dev_priv); @@ -1818,6 +1811,21 @@ static const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_GEM_VM_DESTROY, i915_gem_vm_destroy_ioctl, DRM_RENDER_ALLOW), }; +/* + * Interface history: + * + * 1.1: Original. + * 1.2: Add Power Management + * 1.3: Add vblank support + * 1.4: Fix cmdbuffer path, add heap destroy + * 1.5: Add vblank pipe configuration + * 1.6: - New ioctl for scheduling buffer swaps on vertical blank + * - Support vertical blank on secondary display pipe + */ +#define DRIVER_MAJOR 1 +#define DRIVER_MINOR 6 +#define DRIVER_PATCHLEVEL 0 + static const struct drm_driver i915_drm_driver = { /* Don't use MTRRs here; the Xserver or userspace app should * deal with them for Intel hardware. diff --git a/drivers/gpu/drm/i915/i915_driver.h b/drivers/gpu/drm/i915/i915_driver.h index 9ef8db4aa0a6..9d11de65daaf 100644 --- a/drivers/gpu/drm/i915/i915_driver.h +++ b/drivers/gpu/drm/i915/i915_driver.h @@ -12,6 +12,11 @@ struct pci_dev; struct pci_device_id; struct drm_i915_private; +#define DRIVER_NAME "i915" +#define DRIVER_DESC "Intel Graphics" +#define DRIVER_DATE "20201103" +#define DRIVER_TIMESTAMP 1604406085 + extern const struct dev_pm_ops i915_pm_ops; int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 0c70ab08fc0c..d134838b3458 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -31,54 +31,34 @@ #define _I915_DRV_H_ #include <uapi/drm/i915_drm.h> -#include <uapi/drm/drm_fourcc.h> #include <asm/hypervisor.h> -#include <linux/io-mapping.h> #include <linux/i2c.h> #include <linux/i2c-algo-bit.h> -#include <linux/backlight.h> -#include <linux/hash.h> #include <linux/intel-iommu.h> -#include <linux/kref.h> -#include <linux/mm_types.h> -#include <linux/perf_event.h> #include <linux/pm_qos.h> -#include <linux/dma-resv.h> -#include <linux/shmem_fs.h> -#include <linux/stackdepot.h> -#include <linux/xarray.h> - -#include <drm/drm_gem.h> -#include <drm/drm_auth.h> -#include <drm/drm_cache.h> -#include <drm/drm_util.h> -#include <drm/drm_dsc.h> -#include <drm/drm_atomic.h> + #include <drm/drm_connector.h> -#include <drm/i915_mei_hdcp_interface.h> #include <drm/ttm/ttm_device.h> -#include "i915_params.h" -#include "i915_reg.h" -#include "i915_utils.h" - #include "display/intel_bios.h" +#include "display/intel_cdclk.h" #include "display/intel_display.h" #include "display/intel_display_power.h" #include "display/intel_dmc.h" #include "display/intel_dpll_mgr.h" #include "display/intel_dsb.h" +#include "display/intel_fbc.h" #include "display/intel_frontbuffer.h" #include "display/intel_global_state.h" #include "display/intel_gmbus.h" #include "display/intel_opregion.h" #include "gem/i915_gem_context_types.h" +#include "gem/i915_gem_lmem.h" #include "gem/i915_gem_shrinker.h" #include "gem/i915_gem_stolen.h" -#include "gem/i915_gem_lmem.h" #include "gt/intel_engine.h" #include "gt/intel_gt_types.h" @@ -86,6 +66,12 @@ #include "gt/intel_workarounds.h" #include "gt/uc/intel_uc.h" +#include "i915_gem.h" +#include "i915_gpu_error.h" +#include "i915_params.h" +#include "i915_perf_types.h" +#include "i915_scheduler.h" +#include "i915_utils.h" #include "intel_device_info.h" #include "intel_memory_region.h" #include "intel_pch.h" @@ -93,29 +79,32 @@ #include "intel_runtime_pm.h" #include "intel_step.h" #include "intel_uncore.h" -#include "intel_wakeref.h" #include "intel_wopcm.h" -#include "i915_gem.h" -#include "i915_gem_gtt.h" -#include "i915_gpu_error.h" -#include "i915_perf_types.h" -#include "i915_request.h" -#include "i915_scheduler.h" -#include "gt/intel_timeline.h" -#include "i915_vma.h" -#include "i915_irq.h" - - -/* General customization: - */ - -#define DRIVER_NAME "i915" -#define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20201103" -#define DRIVER_TIMESTAMP 1604406085 - +struct dpll; +struct drm_i915_clock_gating_funcs; struct drm_i915_gem_object; +struct drm_i915_private; +struct intel_atomic_state; +struct intel_audio_funcs; +struct intel_cdclk_config; +struct intel_cdclk_funcs; +struct intel_cdclk_state; +struct intel_cdclk_vals; +struct intel_color_funcs; +struct intel_connector; +struct intel_crtc; +struct intel_dp; +struct intel_dpll_funcs; +struct intel_encoder; +struct intel_fbdev; +struct intel_fdi_funcs; +struct intel_hotplug_funcs; +struct intel_initial_plane_config; +struct intel_limit; +struct intel_overlay; +struct intel_overlay_error_state; +struct vlv_s0ix_state; /* Threshold == 5 for long IRQs, 50 for short */ #define HPD_STORM_DEFAULT_THRESHOLD 50 @@ -166,117 +155,6 @@ struct i915_hotplug { I915_GEM_DOMAIN_INSTRUCTION | \ I915_GEM_DOMAIN_VERTEX) -struct drm_i915_private; - -struct drm_i915_file_private { - struct drm_i915_private *dev_priv; - - union { - struct drm_file *file; - struct rcu_head rcu; - }; - - /** @proto_context_lock: Guards all struct i915_gem_proto_context - * operations - * - * This not only guards @proto_context_xa, but is always held - * whenever we manipulate any struct i915_gem_proto_context, - * including finalizing it on first actual use of the GEM context. - * - * See i915_gem_proto_context. - */ - struct mutex proto_context_lock; - - /** @proto_context_xa: xarray of struct i915_gem_proto_context - * - * Historically, the context uAPI allowed for two methods of - * setting context parameters: SET_CONTEXT_PARAM and - * CONTEXT_CREATE_EXT_SETPARAM. The former is allowed to be called - * at any time while the later happens as part of - * GEM_CONTEXT_CREATE. Everything settable via one was settable - * via the other. While some params are fairly simple and setting - * them on a live context is harmless such as the context priority, - * others are far trickier such as the VM or the set of engines. - * In order to swap out the VM, for instance, we have to delay - * until all current in-flight work is complete, swap in the new - * VM, and then continue. This leads to a plethora of potential - * race conditions we'd really rather avoid. - * - * We have since disallowed setting these more complex parameters - * on active contexts. This works by delaying the creation of the - * actual context until after the client is done configuring it - * with SET_CONTEXT_PARAM. From the perspective of the client, it - * has the same u32 context ID the whole time. From the - * perspective of i915, however, it's a struct i915_gem_proto_context - * right up until the point where we attempt to do something which - * the proto-context can't handle. Then the struct i915_gem_context - * gets created. - * - * This is accomplished via a little xarray dance. When - * GEM_CONTEXT_CREATE is called, we create a struct - * i915_gem_proto_context, reserve a slot in @context_xa but leave - * it NULL, and place the proto-context in the corresponding slot - * in @proto_context_xa. Then, in i915_gem_context_lookup(), we - * first check @context_xa. If it's there, we return the struct - * i915_gem_context and we're done. If it's not, we look in - * @proto_context_xa and, if we find it there, we create the actual - * context and kill the proto-context. - * - * In order for this dance to work properly, everything which ever - * touches a struct i915_gem_proto_context is guarded by - * @proto_context_lock, including context creation. Yes, this - * means context creation now takes a giant global lock but it - * can't really be helped and that should never be on any driver's - * fast-path anyway. - */ - struct xarray proto_context_xa; - - /** @context_xa: xarray of fully created i915_gem_context - * - * Write access to this xarray is guarded by @proto_context_lock. - * Otherwise, writers may race with finalize_create_context_locked(). - * - * See @proto_context_xa. - */ - struct xarray context_xa; - struct xarray vm_xa; - - unsigned int bsd_engine; - -/* - * Every context ban increments per client ban score. Also - * hangs in short succession increments ban score. If ban threshold - * is reached, client is considered banned and submitting more work - * will fail. This is a stop gap measure to limit the badly behaving - * clients access to gpu. Note that unbannable contexts never increment - * the client ban score. - */ -#define I915_CLIENT_SCORE_HANG_FAST 1 -#define I915_CLIENT_FAST_HANG_JIFFIES (60 * HZ) -#define I915_CLIENT_SCORE_CONTEXT_BAN 3 -#define I915_CLIENT_SCORE_BANNED 9 - /** ban_score: Accumulated score of all ctx bans and fast hangs. */ - atomic_t ban_score; - unsigned long hang_timestamp; -}; - -/* Interface history: - * - * 1.1: Original. - * 1.2: Add Power Management - * 1.3: Add vblank support - * 1.4: Fix cmdbuffer path, add heap destroy - * 1.5: Add vblank pipe configuration - * 1.6: - New ioctl for scheduling buffer swaps on vertical blank - * - Support vertical blank on secondary display pipe - */ -#define DRIVER_MAJOR 1 -#define DRIVER_MINOR 6 -#define DRIVER_PATCHLEVEL 0 - -struct intel_overlay; -struct intel_overlay_error_state; - struct sdvo_device_mapping { u8 initialized; u8 dvo_port; @@ -286,22 +164,6 @@ struct sdvo_device_mapping { u8 ddc_pin; }; -struct intel_connector; -struct intel_encoder; -struct intel_atomic_state; -struct intel_cdclk_config; -struct intel_cdclk_state; -struct intel_cdclk_vals; -struct intel_initial_plane_config; -struct intel_crtc; -struct intel_limit; -struct dpll; - -/* functions used internal in intel_pm.c */ -struct drm_i915_clock_gating_funcs { - void (*init_clock_gating)(struct drm_i915_private *dev_priv); -}; - /* functions used for watermark calcs for display. */ struct drm_i915_wm_disp_funcs { /* update_wm is for legacy wm management */ @@ -319,49 +181,6 @@ struct drm_i915_wm_disp_funcs { int (*compute_global_watermarks)(struct intel_atomic_state *state); }; -struct intel_color_funcs { - int (*color_check)(struct intel_crtc_state *crtc_state); - /* - * Program double buffered color management registers during - * vblank evasion. The registers should then latch during the - * next vblank start, alongside any other double buffered registers - * involved with the same commit. - */ - void (*color_commit)(const struct intel_crtc_state *crtc_state); - /* - * Load LUTs (and other single buffered color management - * registers). Will (hopefully) be called during the vblank - * following the latching of any double buffered registers - * involved with the same commit. - */ - void (*load_luts)(const struct intel_crtc_state *crtc_state); - void (*read_luts)(struct intel_crtc_state *crtc_state); -}; - -struct intel_cdclk_funcs { - void (*get_cdclk)(struct drm_i915_private *dev_priv, - struct intel_cdclk_config *cdclk_config); - void (*set_cdclk)(struct drm_i915_private *dev_priv, - const struct intel_cdclk_config *cdclk_config, - enum pipe pipe); - int (*bw_calc_min_cdclk)(struct intel_atomic_state *state); - int (*modeset_calc_cdclk)(struct intel_cdclk_state *state); - u8 (*calc_voltage_level)(int cdclk); -}; - -struct intel_hotplug_funcs { - void (*hpd_irq_setup)(struct drm_i915_private *dev_priv); -}; - -struct intel_fdi_funcs { - void (*fdi_link_train)(struct intel_crtc *crtc, - const struct intel_crtc_state *crtc_state); -}; - -struct intel_dpll_funcs { - int (*crtc_compute_clock)(struct intel_crtc_state *crtc_state); -}; - struct drm_i915_display_funcs { /* Returns the active state of the crtc, and if the crtc is active, * fills out the pipe-config with the hw state. */ @@ -395,7 +214,6 @@ enum drrs_support_type { SEAMLESS_DRRS_SUPPORT = 2 }; -struct intel_dp; struct i915_drrs { struct mutex mutex; struct delayed_work work; @@ -413,8 +231,6 @@ struct i915_drrs { #define QUIRK_INCREASE_DDI_DISABLED_TIME (1<<7) #define QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK (1<<8) -struct intel_fbdev; - struct intel_gmbus { struct i2c_adapter adapter; #define GMBUS_FORCE_BIT_RETRY (1U << 31) @@ -433,8 +249,6 @@ struct i915_suspend_saved_registers { u16 saveGCDGMBUS; }; -struct vlv_s0ix_state; - #define MAX_L3_SLICES 2 struct intel_l3_parity { u32 *remap_info[MAX_L3_SLICES]; @@ -523,13 +337,6 @@ i915_fence_timeout(const struct drm_i915_private *i915) /* Amount of PSF GV points, BSpec precisely defines this */ #define I915_NUM_PSF_GV_POINTS 3 -enum psr_lines_to_wait { - PSR_0_LINES_TO_WAIT = 0, - PSR_1_LINE_TO_WAIT, - PSR_4_LINES_TO_WAIT, - PSR_8_LINES_TO_WAIT -}; - struct intel_vbt_data { /* bdb version */ u16 version; @@ -550,6 +357,9 @@ struct intel_vbt_data { unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */ enum drm_panel_orientation orientation; + bool override_afc_startup; + u8 override_afc_startup_val; + enum drrs_support_type drrs_type; struct { @@ -569,7 +379,6 @@ struct intel_vbt_data { bool full_link; bool require_aux_wakeup; int idle_frames; - enum psr_lines_to_wait lines_to_wait; int tp1_wakeup_time_us; int tp2_tp3_wakeup_time_us; int psr2_tp2_tp3_wakeup_time_us; @@ -625,18 +434,12 @@ struct i915_virtual_gpu { u32 caps; }; -struct intel_cdclk_config { - unsigned int cdclk, vco, ref, bypass; - u8 voltage_level; -}; - struct i915_selftest_stash { atomic_t counter; struct ida mock_region_instances; }; /* intel_audio.c private */ -struct intel_audio_funcs; struct intel_audio_private { /* Display internal audio functions */ const struct intel_audio_funcs *funcs; @@ -749,7 +552,7 @@ struct drm_i915_private { u32 pipestat_irq_mask[I915_MAX_PIPES]; struct i915_hotplug hotplug; - struct intel_fbc *fbc; + struct intel_fbc *fbc[I915_MAX_FBCS]; struct i915_drrs drrs; struct intel_opregion opregion; struct intel_vbt_data vbt; @@ -838,8 +641,6 @@ struct drm_i915_private { struct drm_atomic_state *modeset_restore_state; struct drm_modeset_acquire_ctx reset_ctx; - struct i915_ggtt ggtt; /* VM representing the global address space */ - struct i915_gem_mm mm; /* Kernel Modesetting */ @@ -1286,8 +1087,12 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, IS_SUBPLATFORM(dev_priv, INTEL_DG2, INTEL_SUBPLATFORM_G10) #define IS_DG2_G11(dev_priv) \ IS_SUBPLATFORM(dev_priv, INTEL_DG2, INTEL_SUBPLATFORM_G11) +#define IS_DG2_G12(dev_priv) \ + IS_SUBPLATFORM(dev_priv, INTEL_DG2, INTEL_SUBPLATFORM_G12) #define IS_ADLS_RPLS(dev_priv) \ IS_SUBPLATFORM(dev_priv, INTEL_ALDERLAKE_S, INTEL_SUBPLATFORM_RPL_S) +#define IS_ADLP_N(dev_priv) \ + IS_SUBPLATFORM(dev_priv, INTEL_ALDERLAKE_P, INTEL_SUBPLATFORM_N) #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) #define IS_BDW_ULT(dev_priv) \ @@ -1342,11 +1147,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define IS_ICL_WITH_PORT_F(dev_priv) \ IS_SUBPLATFORM(dev_priv, INTEL_ICELAKE, INTEL_SUBPLATFORM_PORTF) -#define IS_TGL_U(dev_priv) \ - IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_ULT) - -#define IS_TGL_Y(dev_priv) \ - IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_ULX) +#define IS_TGL_UY(dev_priv) \ + IS_SUBPLATFORM(dev_priv, INTEL_TIGERLAKE, INTEL_SUBPLATFORM_UY) #define IS_SKL_GRAPHICS_STEP(p, since, until) (IS_SKYLAKE(p) && IS_GRAPHICS_STEP(p, since, until)) @@ -1365,11 +1167,11 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, IS_DISPLAY_STEP(__i915, since, until)) #define IS_TGL_UY_GRAPHICS_STEP(__i915, since, until) \ - ((IS_TGL_U(__i915) || IS_TGL_Y(__i915)) && \ + (IS_TGL_UY(__i915) && \ IS_GRAPHICS_STEP(__i915, since, until)) #define IS_TGL_GRAPHICS_STEP(__i915, since, until) \ - (IS_TIGERLAKE(__i915) && !(IS_TGL_U(__i915) || IS_TGL_Y(__i915)) && \ + (IS_TIGERLAKE(__i915) && !IS_TGL_UY(__i915)) && \ IS_GRAPHICS_STEP(__i915, since, until)) #define IS_RKL_DISPLAY_STEP(p, since, until) \ @@ -1400,16 +1202,17 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, (IS_XEHPSDV(__i915) && IS_GRAPHICS_STEP(__i915, since, until)) /* - * DG2 hardware steppings are a bit unusual. The hardware design was forked - * to create two variants (G10 and G11) which have distinct workaround sets. - * The G11 fork of the DG2 design resets the GT stepping back to "A0" for its - * first iteration, even though it's more similar to a G10 B0 stepping in terms - * of functionality and workarounds. However the display stepping does not - * reset in the same manner --- a specific stepping like "B0" has a consistent - * meaning regardless of whether it belongs to a G10 or G11 DG2. + * DG2 hardware steppings are a bit unusual. The hardware design was forked to + * create three variants (G10, G11, and G12) which each have distinct + * workaround sets. The G11 and G12 forks of the DG2 design reset the GT + * stepping back to "A0" for their first iterations, even though they're more + * similar to a G10 B0 stepping and G10 C0 stepping respectively in terms of + * functionality and workarounds. However the display stepping does not reset + * in the same manner --- a specific stepping like "B0" has a consistent + * meaning regardless of whether it belongs to a G10, G11, or G12 DG2. * * TLDR: All GT workarounds and stepping-specific logic must be applied in - * relation to a specific subplatform (G10 or G11), whereas display workarounds + * relation to a specific subplatform (G10/G11/G12), whereas display workarounds * and stepping-specific logic will be applied with a general DG2-wide stepping * number. */ @@ -1438,6 +1241,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, ENGINE_INSTANCES_MASK(gt, VCS0, I915_MAX_VCS) #define VEBOX_MASK(gt) \ ENGINE_INSTANCES_MASK(gt, VECS0, I915_MAX_VECS) +#define CCS_MASK(gt) \ + ENGINE_INSTANCES_MASK(gt, CCS0, I915_MAX_CCS) /* * The Gen7 cmdparser copies the scanned buffer to the ggtt for execution @@ -1485,8 +1290,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ (IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv)) -#define HAS_GMBUS_IRQ(dev_priv) (GRAPHICS_VER(dev_priv) >= 4) -#define HAS_GMBUS_BURST_READ(dev_priv) (GRAPHICS_VER(dev_priv) >= 11 || \ +#define HAS_GMBUS_IRQ(dev_priv) (DISPLAY_VER(dev_priv) >= 4) +#define HAS_GMBUS_BURST_READ(dev_priv) (DISPLAY_VER(dev_priv) >= 11 || \ IS_GEMINILAKE(dev_priv) || \ IS_KABYLAKE(dev_priv)) @@ -1498,9 +1303,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define SUPPORTS_TV(dev_priv) (INTEL_INFO(dev_priv)->display.supports_tv) #define I915_HAS_HOTPLUG(dev_priv) (INTEL_INFO(dev_priv)->display.has_hotplug) -#define HAS_FW_BLC(dev_priv) (GRAPHICS_VER(dev_priv) > 2) -#define HAS_FBC(dev_priv) (INTEL_INFO(dev_priv)->display.has_fbc) -#define HAS_CUR_FBC(dev_priv) (!HAS_GMCH(dev_priv) && GRAPHICS_VER(dev_priv) >= 7) +#define HAS_FW_BLC(dev_priv) (DISPLAY_VER(dev_priv) > 2) +#define HAS_FBC(dev_priv) (INTEL_INFO(dev_priv)->display.fbc_mask != 0) +#define HAS_CUR_FBC(dev_priv) (!HAS_GMCH(dev_priv) && DISPLAY_VER(dev_priv) >= 7) #define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) @@ -1513,7 +1318,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_PSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_psr) #define HAS_PSR_HW_TRACKING(dev_priv) \ (INTEL_INFO(dev_priv)->display.has_psr_hw_tracking) -#define HAS_PSR2_SEL_FETCH(dev_priv) (GRAPHICS_VER(dev_priv) >= 12) +#define HAS_PSR2_SEL_FETCH(dev_priv) (DISPLAY_VER(dev_priv) >= 12) #define HAS_TRANSCODER(dev_priv, trans) ((INTEL_INFO(dev_priv)->display.cpu_transcoder_mask & BIT(trans)) != 0) #define HAS_RC6(dev_priv) (INTEL_INFO(dev_priv)->has_rc6) @@ -1524,7 +1329,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_DMC(dev_priv) (INTEL_INFO(dev_priv)->display.has_dmc) -#define HAS_MSO(i915) (GRAPHICS_VER(i915) >= 12) +#define HAS_MSO(i915) (DISPLAY_VER(i915) >= 12) #define HAS_RUNTIME_PM(dev_priv) (INTEL_INFO(dev_priv)->has_runtime_pm) #define HAS_64BIT_RELOC(dev_priv) (INTEL_INFO(dev_priv)->has_64bit_reloc) @@ -1534,17 +1339,28 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, /* * Set this flag, when platform requires 64K GTT page sizes or larger for - * device local memory access. Also this flag implies that we require or - * at least support the compact PT layout for the ppGTT when using the 64K - * GTT pages. + * device local memory access. */ #define HAS_64K_PAGES(dev_priv) (INTEL_INFO(dev_priv)->has_64k_pages) +/* + * Set this flag when platform doesn't allow both 64k pages and 4k pages in + * the same PT. this flag means we need to support compact PT layout for the + * ppGTT when using the 64K GTT pages. + */ +#define NEEDS_COMPACT_PT(dev_priv) (INTEL_INFO(dev_priv)->needs_compact_pt) + #define HAS_IPC(dev_priv) (INTEL_INFO(dev_priv)->display.has_ipc) #define HAS_REGION(i915, i) (INTEL_INFO(i915)->memory_regions & (i)) #define HAS_LMEM(i915) HAS_REGION(i915, REGION_LMEM) +/* + * Platform has the dedicated compression control state for each lmem surfaces + * stored in lmem to support the 3D and media compression formats. + */ +#define HAS_FLAT_CCS(dev_priv) (INTEL_INFO(dev_priv)->has_flat_ccs) + #define HAS_GT_UC(dev_priv) (INTEL_INFO(dev_priv)->has_gt_uc) #define HAS_POOLED_EU(dev_priv) (INTEL_INFO(dev_priv)->has_pooled_eu) @@ -1557,7 +1373,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_GMCH(dev_priv) (INTEL_INFO(dev_priv)->display.has_gmch) -#define HAS_LSPCON(dev_priv) (IS_GRAPHICS_VER(dev_priv, 9, 10)) +#define HAS_LSPCON(dev_priv) (IS_DISPLAY_VER(dev_priv, 9, 10)) /* DPF == dynamic parity feature */ #define HAS_L3_DPF(dev_priv) (INTEL_INFO(dev_priv)->has_l3_dpf) @@ -1571,7 +1387,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_DISPLAY(dev_priv) (INTEL_INFO(dev_priv)->display.pipe_mask != 0) -#define HAS_VRR(i915) (GRAPHICS_VER(i915) >= 11) +#define HAS_VRR(i915) (DISPLAY_VER(i915) >= 11) #define HAS_ASYNC_FLIPS(i915) (DISPLAY_VER(i915) >= 5) @@ -1579,6 +1395,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define INTEL_DISPLAY_ENABLED(dev_priv) \ (drm_WARN_ON(&(dev_priv)->drm, !HAS_DISPLAY(dev_priv)), !(dev_priv)->params.disable_display) +#define HAS_GUC_DEPRIVILEGE(dev_priv) \ + (INTEL_INFO(dev_priv)->has_guc_deprivilege) + static inline bool run_as_guest(void) { return !hypervisor_is_type(X86_HYPER_NATIVE); @@ -1601,7 +1420,7 @@ i915_print_iommu_status(struct drm_i915_private *i915, struct drm_printer *p); static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) { - return GRAPHICS_VER(dev_priv) >= 6 && intel_vtd_active(dev_priv); + return DISPLAY_VER(dev_priv) >= 6 && intel_vtd_active(dev_priv); } static inline bool @@ -1616,13 +1435,7 @@ intel_vm_no_concurrent_access_wa(struct drm_i915_private *i915) return IS_CHERRYVIEW(i915) || intel_ggtt_update_needs_vtd_wa(i915); } -/* i915_getparam.c */ -int i915_getparam_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv); - /* i915_gem.c */ -int i915_gem_init_userptr(struct drm_i915_private *dev_priv); -void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv); void i915_gem_init_early(struct drm_i915_private *dev_priv); void i915_gem_cleanup_early(struct drm_i915_private *dev_priv); @@ -1681,107 +1494,29 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj, #define I915_GEM_OBJECT_UNBIND_BARRIER BIT(1) #define I915_GEM_OBJECT_UNBIND_TEST BIT(2) #define I915_GEM_OBJECT_UNBIND_VM_TRYLOCK BIT(3) +#define I915_GEM_OBJECT_UNBIND_ASYNC BIT(4) void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); -int i915_gem_dumb_create(struct drm_file *file_priv, - struct drm_device *dev, - struct drm_mode_create_dumb *args); - int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); -static inline u32 i915_reset_count(struct i915_gpu_error *error) -{ - return atomic_read(&error->reset_count); -} - -static inline u32 i915_reset_engine_count(struct i915_gpu_error *error, - const struct intel_engine_cs *engine) -{ - return atomic_read(&error->reset_engine_count[engine->uabi_class]); -} - int __must_check i915_gem_init(struct drm_i915_private *dev_priv); void i915_gem_driver_register(struct drm_i915_private *i915); void i915_gem_driver_unregister(struct drm_i915_private *i915); void i915_gem_driver_remove(struct drm_i915_private *dev_priv); void i915_gem_driver_release(struct drm_i915_private *dev_priv); -void i915_gem_suspend(struct drm_i915_private *dev_priv); -void i915_gem_suspend_late(struct drm_i915_private *dev_priv); -void i915_gem_resume(struct drm_i915_private *dev_priv); int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); -int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, - enum i915_cache_level cache_level); - -struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, - struct dma_buf *dma_buf); - -struct dma_buf *i915_gem_prime_export(struct drm_gem_object *gem_obj, int flags); - -static inline struct i915_address_space * -i915_gem_vm_lookup(struct drm_i915_file_private *file_priv, u32 id) -{ - struct i915_address_space *vm; - - xa_lock(&file_priv->vm_xa); - vm = xa_load(&file_priv->vm_xa, id); - if (vm) - kref_get(&vm->ref); - xa_unlock(&file_priv->vm_xa); - - return vm; -} - -/* i915_gem_evict.c */ -int __must_check i915_gem_evict_something(struct i915_address_space *vm, - u64 min_size, u64 alignment, - unsigned long color, - u64 start, u64 end, - unsigned flags); -int __must_check i915_gem_evict_for_node(struct i915_address_space *vm, - struct drm_mm_node *node, - unsigned int flags); -int i915_gem_evict_vm(struct i915_address_space *vm); - -/* i915_gem_internal.c */ -struct drm_i915_gem_object * -i915_gem_object_create_internal(struct drm_i915_private *dev_priv, - phys_addr_t size); -struct drm_i915_gem_object * -__i915_gem_object_create_internal(struct drm_i915_private *dev_priv, - const struct drm_i915_gem_object_ops *ops, - phys_addr_t size); - /* i915_gem_tiling.c */ static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - return i915->ggtt.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && + return to_gt(i915)->ggtt->bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && i915_gem_object_is_tiled(obj); } -u32 i915_gem_fence_size(struct drm_i915_private *dev_priv, u32 size, - unsigned int tiling, unsigned int stride); -u32 i915_gem_fence_alignment(struct drm_i915_private *dev_priv, u32 size, - unsigned int tiling, unsigned int stride); - -const char *i915_cache_level_str(struct drm_i915_private *i915, int type); - -/* i915_cmd_parser.c */ -int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv); -int intel_engine_init_cmd_parser(struct intel_engine_cs *engine); -void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine); -int intel_engine_cmd_parser(struct intel_engine_cs *engine, - struct i915_vma *batch, - unsigned long batch_offset, - unsigned long batch_length, - struct i915_vma *shadow, - bool trampoline); -#define I915_CMD_PARSER_TRAMPOLINE_SIZE 8 - /* intel_device_info.c */ static inline struct intel_device_info * mkwrite_device_info(struct drm_i915_private *dev_priv) @@ -1789,17 +1524,6 @@ mkwrite_device_info(struct drm_i915_private *dev_priv) return (struct intel_device_info *)INTEL_INFO(dev_priv); } -int i915_reg_read_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); - -static inline int intel_hws_csb_write_index(struct drm_i915_private *i915) -{ - if (GRAPHICS_VER(i915) >= 11) - return ICL_HWS_CSB_WRITE_INDEX; - else - return I915_HWS_CSB_WRITE_INDEX; -} - static inline enum i915_map_type i915_coherent_map_type(struct drm_i915_private *i915, struct drm_i915_gem_object *obj, bool always_coherent) diff --git a/drivers/gpu/drm/i915/i915_file_private.h b/drivers/gpu/drm/i915/i915_file_private.h new file mode 100644 index 000000000000..fb16cc431b2a --- /dev/null +++ b/drivers/gpu/drm/i915/i915_file_private.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_FILE_PRIVATE_H__ +#define __I915_FILE_PRIVATE_H__ + +#include <linux/mutex.h> +#include <linux/types.h> +#include <linux/xarray.h> + +struct drm_i915_private; +struct drm_file; + +struct drm_i915_file_private { + struct drm_i915_private *dev_priv; + + union { + struct drm_file *file; + struct rcu_head rcu; + }; + + /** @proto_context_lock: Guards all struct i915_gem_proto_context + * operations + * + * This not only guards @proto_context_xa, but is always held + * whenever we manipulate any struct i915_gem_proto_context, + * including finalizing it on first actual use of the GEM context. + * + * See i915_gem_proto_context. + */ + struct mutex proto_context_lock; + + /** @proto_context_xa: xarray of struct i915_gem_proto_context + * + * Historically, the context uAPI allowed for two methods of + * setting context parameters: SET_CONTEXT_PARAM and + * CONTEXT_CREATE_EXT_SETPARAM. The former is allowed to be called + * at any time while the later happens as part of + * GEM_CONTEXT_CREATE. Everything settable via one was settable + * via the other. While some params are fairly simple and setting + * them on a live context is harmless such as the context priority, + * others are far trickier such as the VM or the set of engines. + * In order to swap out the VM, for instance, we have to delay + * until all current in-flight work is complete, swap in the new + * VM, and then continue. This leads to a plethora of potential + * race conditions we'd really rather avoid. + * + * We have since disallowed setting these more complex parameters + * on active contexts. This works by delaying the creation of the + * actual context until after the client is done configuring it + * with SET_CONTEXT_PARAM. From the perspective of the client, it + * has the same u32 context ID the whole time. From the + * perspective of i915, however, it's a struct i915_gem_proto_context + * right up until the point where we attempt to do something which + * the proto-context can't handle. Then the struct i915_gem_context + * gets created. + * + * This is accomplished via a little xarray dance. When + * GEM_CONTEXT_CREATE is called, we create a struct + * i915_gem_proto_context, reserve a slot in @context_xa but leave + * it NULL, and place the proto-context in the corresponding slot + * in @proto_context_xa. Then, in i915_gem_context_lookup(), we + * first check @context_xa. If it's there, we return the struct + * i915_gem_context and we're done. If it's not, we look in + * @proto_context_xa and, if we find it there, we create the actual + * context and kill the proto-context. + * + * In order for this dance to work properly, everything which ever + * touches a struct i915_gem_proto_context is guarded by + * @proto_context_lock, including context creation. Yes, this + * means context creation now takes a giant global lock but it + * can't really be helped and that should never be on any driver's + * fast-path anyway. + */ + struct xarray proto_context_xa; + + /** @context_xa: xarray of fully created i915_gem_context + * + * Write access to this xarray is guarded by @proto_context_lock. + * Otherwise, writers may race with finalize_create_context_locked(). + * + * See @proto_context_xa. + */ + struct xarray context_xa; + struct xarray vm_xa; + + unsigned int bsd_engine; + +/* + * Every context ban increments per client ban score. Also + * hangs in short succession increments ban score. If ban threshold + * is reached, client is considered banned and submitting more work + * will fail. This is a stop gap measure to limit the badly behaving + * clients access to gpu. Note that unbannable contexts never increment + * the client ban score. + */ +#define I915_CLIENT_SCORE_HANG_FAST 1 +#define I915_CLIENT_FAST_HANG_JIFFIES (60 * HZ) +#define I915_CLIENT_SCORE_CONTEXT_BAN 3 +#define I915_CLIENT_SCORE_BANNED 9 + /** ban_score: Accumulated score of all ctx bans and fast hangs. */ + atomic_t ban_score; + unsigned long hang_timestamp; +}; + +#endif /* __I915_FILE_PRIVATE_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 915bf431f320..2e10187cd0a0 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -25,7 +25,6 @@ * */ -#include <drm/drm_vma_manager.h> #include <linux/dma-fence-array.h> #include <linux/kthread.h> #include <linux/dma-resv.h> @@ -37,6 +36,9 @@ #include <linux/dma-buf.h> #include <linux/mman.h> +#include <drm/drm_cache.h> +#include <drm/drm_vma_manager.h> + #include "display/intel_display.h" #include "display/intel_frontbuffer.h" @@ -44,16 +46,18 @@ #include "gem/i915_gem_context.h" #include "gem/i915_gem_ioctls.h" #include "gem/i915_gem_mman.h" +#include "gem/i915_gem_pm.h" #include "gem/i915_gem_region.h" +#include "gem/i915_gem_userptr.h" #include "gt/intel_engine_user.h" #include "gt/intel_gt.h" #include "gt/intel_gt_pm.h" #include "gt/intel_workarounds.h" #include "i915_drv.h" +#include "i915_file_private.h" #include "i915_trace.h" #include "i915_vgpu.h" - #include "intel_pm.h" static int @@ -88,7 +92,8 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct i915_ggtt *ggtt = &to_i915(dev)->ggtt; + struct drm_i915_private *i915 = to_i915(dev); + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct drm_i915_gem_get_aperture *args = data; struct i915_vma *vma; u64 pinned; @@ -118,6 +123,8 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj, struct i915_vma *vma; int ret; + assert_object_held(obj); + if (list_empty(&obj->vma.list)) return 0; @@ -155,10 +162,16 @@ try_again: spin_unlock(&obj->vma.lock); if (vma) { + bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK); ret = -EBUSY; - if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE || - !i915_vma_is_active(vma)) { - if (flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK) { + if (flags & I915_GEM_OBJECT_UNBIND_ASYNC) { + assert_object_held(vma->obj); + ret = i915_vma_unbind_async(vma, vm_trylock); + } + + if (ret == -EBUSY && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE || + !i915_vma_is_active(vma))) { + if (vm_trylock) { if (mutex_trylock(&vma->vm->mutex)) { ret = __i915_vma_unbind(vma); mutex_unlock(&vma->vm->mutex); @@ -289,7 +302,7 @@ static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj, bool write) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct i915_vma *vma; struct i915_gem_ww_ctx ww; int ret; @@ -350,7 +363,7 @@ static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj, struct i915_vma *vma) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; i915_gem_object_unpin_pages(obj); if (drm_mm_node_allocated(node)) { @@ -366,7 +379,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj, const struct drm_i915_gem_pread *args) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; intel_wakeref_t wakeref; struct drm_mm_node node; void __user *user_data; @@ -522,7 +535,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, const struct drm_i915_gem_pwrite *args) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct intel_runtime_pm *rpm = &i915->runtime_pm; intel_wakeref_t wakeref; struct drm_mm_node node; @@ -823,7 +836,7 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915) */ list_for_each_entry_safe(obj, on, - &i915->ggtt.userfault_list, userfault_link) + &to_gt(i915)->ggtt->userfault_list, userfault_link) __i915_gem_object_release_mmap_gtt(obj); /* @@ -831,8 +844,8 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915) * in use by hardware (i.e. they are pinned), we should not be powering * down! All other fences will be reacquired by the user upon waking. */ - for (i = 0; i < i915->ggtt.num_fences; i++) { - struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i]; + for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) { + struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i]; /* * Ideally we want to assert that the fence register is not @@ -873,7 +886,7 @@ i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj, u64 size, u64 alignment, u64 flags) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct i915_vma *vma; int ret; @@ -1123,7 +1136,7 @@ err_unlock: /* Minimal basic recovery for KMS */ ret = i915_ggtt_enable_hw(dev_priv); - i915_ggtt_resume(&dev_priv->ggtt); + i915_ggtt_resume(to_gt(dev_priv)->ggtt); intel_init_clock_gating(dev_priv); } @@ -1146,7 +1159,7 @@ void i915_gem_driver_unregister(struct drm_i915_private *i915) void i915_gem_driver_remove(struct drm_i915_private *dev_priv) { - intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref); + intel_wakeref_auto_fini(&to_gt(dev_priv)->ggtt->userfault_wakeref); i915_gem_suspend_late(dev_priv); intel_gt_driver_remove(to_gt(dev_priv)); diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index 2b73ddb11c66..f025ee4fa526 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -31,12 +31,18 @@ #include "gt/intel_gt_requests.h" #include "i915_drv.h" +#include "i915_gem_evict.h" #include "i915_trace.h" I915_SELFTEST_DECLARE(static struct igt_evict_ctl { bool fail_if_busy:1; } igt_evict_ctl;) +static bool dying_vma(struct i915_vma *vma) +{ + return !kref_read(&vma->obj->base.refcount); +} + static int ggtt_flush(struct intel_gt *gt) { /* @@ -49,8 +55,37 @@ static int ggtt_flush(struct intel_gt *gt) return intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT); } +static bool grab_vma(struct i915_vma *vma, struct i915_gem_ww_ctx *ww) +{ + /* + * We add the extra refcount so the object doesn't drop to zero until + * after ungrab_vma(), this way trylock is always paired with unlock. + */ + if (i915_gem_object_get_rcu(vma->obj)) { + if (!i915_gem_object_trylock(vma->obj, ww)) { + i915_gem_object_put(vma->obj); + return false; + } + } else { + /* Dead objects don't need pins */ + atomic_and(~I915_VMA_PIN_MASK, &vma->flags); + } + + return true; +} + +static void ungrab_vma(struct i915_vma *vma) +{ + if (dying_vma(vma)) + return; + + i915_gem_object_unlock(vma->obj); + i915_gem_object_put(vma->obj); +} + static bool mark_free(struct drm_mm_scan *scan, + struct i915_gem_ww_ctx *ww, struct i915_vma *vma, unsigned int flags, struct list_head *unwind) @@ -58,6 +93,9 @@ mark_free(struct drm_mm_scan *scan, if (i915_vma_is_pinned(vma)) return false; + if (!grab_vma(vma, ww)) + return false; + list_add(&vma->evict_link, unwind); return drm_mm_scan_add_block(scan, &vma->node); } @@ -76,6 +114,7 @@ static bool defer_evict(struct i915_vma *vma) /** * i915_gem_evict_something - Evict vmas to make room for binding a new one * @vm: address space to evict from + * @ww: An optional struct i915_gem_ww_ctx. * @min_size: size of the desired free space * @alignment: alignment constraint of the desired free space * @color: color for the desired space @@ -98,6 +137,7 @@ static bool defer_evict(struct i915_vma *vma) */ int i915_gem_evict_something(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, u64 min_size, u64 alignment, unsigned long color, u64 start, u64 end, @@ -170,7 +210,7 @@ search_again: continue; } - if (mark_free(&scan, vma, flags, &eviction_list)) + if (mark_free(&scan, ww, vma, flags, &eviction_list)) goto found; } @@ -178,6 +218,7 @@ search_again: list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { ret = drm_mm_scan_remove_block(&scan, &vma->node); BUG_ON(ret); + ungrab_vma(vma); } /* @@ -222,10 +263,12 @@ found: * of any of our objects, thus corrupting the list). */ list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { - if (drm_mm_scan_remove_block(&scan, &vma->node)) + if (drm_mm_scan_remove_block(&scan, &vma->node)) { __i915_vma_pin(vma); - else + } else { list_del(&vma->evict_link); + ungrab_vma(vma); + } } /* Unbinding will emit any required flushes */ @@ -234,16 +277,20 @@ found: __i915_vma_unpin(vma); if (ret == 0) ret = __i915_vma_unbind(vma); + ungrab_vma(vma); } while (ret == 0 && (node = drm_mm_scan_color_evict(&scan))) { vma = container_of(node, struct i915_vma, node); /* If we find any non-objects (!vma), we cannot evict them */ - if (vma->node.color != I915_COLOR_UNEVICTABLE) + if (vma->node.color != I915_COLOR_UNEVICTABLE && + grab_vma(vma, ww)) { ret = __i915_vma_unbind(vma); - else - ret = -ENOSPC; /* XXX search failed, try again? */ + ungrab_vma(vma); + } else { + ret = -ENOSPC; + } } return ret; @@ -252,6 +299,7 @@ found: /** * i915_gem_evict_for_node - Evict vmas to make room for binding a new one * @vm: address space to evict from + * @ww: An optional struct i915_gem_ww_ctx. * @target: range (and color) to evict for * @flags: additional flags to control the eviction algorithm * @@ -261,6 +309,7 @@ found: * memory in e.g. the shrinker. */ int i915_gem_evict_for_node(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, struct drm_mm_node *target, unsigned int flags) { @@ -333,6 +382,11 @@ int i915_gem_evict_for_node(struct i915_address_space *vm, break; } + if (!grab_vma(vma, ww)) { + ret = -ENOSPC; + break; + } + /* * Never show fear in the face of dragons! * @@ -350,6 +404,8 @@ int i915_gem_evict_for_node(struct i915_address_space *vm, __i915_vma_unpin(vma); if (ret == 0) ret = __i915_vma_unbind(vma); + + ungrab_vma(vma); } return ret; @@ -358,6 +414,8 @@ int i915_gem_evict_for_node(struct i915_address_space *vm, /** * i915_gem_evict_vm - Evict all idle vmas from a vm * @vm: Address space to cleanse + * @ww: An optional struct i915_gem_ww_ctx. If not NULL, i915_gem_evict_vm + * will be able to evict vma's locked by the ww as well. * * This function evicts all vmas from a vm. * @@ -367,7 +425,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm, * To clarify: This is for freeing up virtual address space, not for freeing * memory in e.g. the shrinker. */ -int i915_gem_evict_vm(struct i915_address_space *vm) +int i915_gem_evict_vm(struct i915_address_space *vm, struct i915_gem_ww_ctx *ww) { int ret = 0; @@ -388,24 +446,52 @@ int i915_gem_evict_vm(struct i915_address_space *vm) do { struct i915_vma *vma, *vn; LIST_HEAD(eviction_list); + LIST_HEAD(locked_eviction_list); list_for_each_entry(vma, &vm->bound_list, vm_link) { if (i915_vma_is_pinned(vma)) continue; + /* + * If we already own the lock, trylock fails. In case + * the resv is shared among multiple objects, we still + * need the object ref. + */ + if (dying_vma(vma) || + (ww && (dma_resv_locking_ctx(vma->obj->base.resv) == &ww->ctx))) { + __i915_vma_pin(vma); + list_add(&vma->evict_link, &locked_eviction_list); + continue; + } + + if (!i915_gem_object_trylock(vma->obj, ww)) + continue; + __i915_vma_pin(vma); list_add(&vma->evict_link, &eviction_list); } - if (list_empty(&eviction_list)) + if (list_empty(&eviction_list) && list_empty(&locked_eviction_list)) break; ret = 0; + /* Unbind locked objects first, before unlocking the eviction_list */ + list_for_each_entry_safe(vma, vn, &locked_eviction_list, evict_link) { + __i915_vma_unpin(vma); + + if (ret == 0) + ret = __i915_vma_unbind(vma); + if (ret != -EINTR) /* "Get me out of here!" */ + ret = 0; + } + list_for_each_entry_safe(vma, vn, &eviction_list, evict_link) { __i915_vma_unpin(vma); if (ret == 0) ret = __i915_vma_unbind(vma); if (ret != -EINTR) /* "Get me out of here!" */ ret = 0; + + i915_gem_object_unlock(vma->obj); } } while (ret == 0); diff --git a/drivers/gpu/drm/i915/i915_gem_evict.h b/drivers/gpu/drm/i915/i915_gem_evict.h new file mode 100644 index 000000000000..e593c530f9bd --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_evict.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_GEM_EVICT_H__ +#define __I915_GEM_EVICT_H__ + +#include <linux/types.h> + +struct drm_mm_node; +struct i915_address_space; +struct i915_gem_ww_ctx; + +int __must_check i915_gem_evict_something(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, + u64 min_size, u64 alignment, + unsigned long color, + u64 start, u64 end, + unsigned flags); +int __must_check i915_gem_evict_for_node(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, + struct drm_mm_node *node, + unsigned int flags); +int i915_gem_evict_vm(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww); + +#endif /* __I915_GEM_EVICT_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index cd5f2348a187..329ff75b80b9 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -20,6 +20,7 @@ #include "gt/intel_gt_requests.h" #include "i915_drv.h" +#include "i915_gem_evict.h" #include "i915_scatterlist.h" #include "i915_trace.h" #include "i915_vgpu.h" @@ -56,7 +57,7 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; /* XXX This does not prevent more requests being submitted! */ if (unlikely(ggtt->do_idle_maps)) @@ -70,6 +71,7 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, /** * i915_gem_gtt_reserve - reserve a node in an address_space (GTT) * @vm: the &struct i915_address_space + * @ww: An optional struct i915_gem_ww_ctx. * @node: the &struct drm_mm_node (typically i915_vma.mode) * @size: how much space to allocate inside the GTT, * must be #I915_GTT_PAGE_SIZE aligned @@ -93,6 +95,7 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, * asked to wait for eviction and interrupted. */ int i915_gem_gtt_reserve(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, struct drm_mm_node *node, u64 size, u64 offset, unsigned long color, unsigned int flags) @@ -103,7 +106,7 @@ int i915_gem_gtt_reserve(struct i915_address_space *vm, GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT)); GEM_BUG_ON(range_overflows(offset, size, vm->total)); - GEM_BUG_ON(vm == &vm->i915->ggtt.alias->vm); + GEM_BUG_ON(vm == &to_gt(vm->i915)->ggtt->alias->vm); GEM_BUG_ON(drm_mm_node_allocated(node)); node->size = size; @@ -117,7 +120,7 @@ int i915_gem_gtt_reserve(struct i915_address_space *vm, if (flags & PIN_NOEVICT) return -ENOSPC; - err = i915_gem_evict_for_node(vm, node, flags); + err = i915_gem_evict_for_node(vm, ww, node, flags); if (err == 0) err = drm_mm_reserve_node(&vm->mm, node); @@ -152,6 +155,7 @@ static u64 random_offset(u64 start, u64 end, u64 len, u64 align) /** * i915_gem_gtt_insert - insert a node into an address_space (GTT) * @vm: the &struct i915_address_space + * @ww: An optional struct i915_gem_ww_ctx. * @node: the &struct drm_mm_node (typically i915_vma.node) * @size: how much space to allocate inside the GTT, * must be #I915_GTT_PAGE_SIZE aligned @@ -184,6 +188,7 @@ static u64 random_offset(u64 start, u64 end, u64 len, u64 align) * asked to wait for eviction and interrupted. */ int i915_gem_gtt_insert(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, struct drm_mm_node *node, u64 size, u64 alignment, unsigned long color, u64 start, u64 end, unsigned int flags) @@ -201,7 +206,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, GEM_BUG_ON(start >= end); GEM_BUG_ON(start > 0 && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); - GEM_BUG_ON(vm == &vm->i915->ggtt.alias->vm); + GEM_BUG_ON(vm == &to_gt(vm->i915)->ggtt->alias->vm); GEM_BUG_ON(drm_mm_node_allocated(node)); if (unlikely(range_overflows(start, size, end))) @@ -269,7 +274,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, */ offset = random_offset(start, end, size, alignment ?: I915_GTT_MIN_ALIGNMENT); - err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags); + err = i915_gem_gtt_reserve(vm, ww, node, size, offset, color, flags); if (err != -ENOSPC) return err; @@ -277,7 +282,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, return -ENOSPC; /* Randomly selected placement is pinned, do a search */ - err = i915_gem_evict_something(vm, size, alignment, color, + err = i915_gem_evict_something(vm, ww, size, alignment, color, start, end, flags); if (err) return err; diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index c9b0ee5e1d23..8c2f57eb5dda 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -16,6 +16,7 @@ struct drm_i915_gem_object; struct i915_address_space; +struct i915_gem_ww_ctx; int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, struct sg_table *pages); @@ -23,11 +24,13 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, struct sg_table *pages); int i915_gem_gtt_reserve(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, struct drm_mm_node *node, u64 size, u64 offset, unsigned long color, unsigned int flags); int i915_gem_gtt_insert(struct i915_address_space *vm, + struct i915_gem_ww_ctx *ww, struct drm_mm_node *node, u64 size, u64 alignment, unsigned long color, u64 start, u64 end, unsigned int flags); @@ -41,6 +44,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, #define PIN_HIGH BIT_ULL(5) #define PIN_OFFSET_BIAS BIT_ULL(6) #define PIN_OFFSET_FIXED BIT_ULL(7) +#define PIN_VALIDATE BIT_ULL(8) /* validate placement only, no need to call unpin() */ #define PIN_GLOBAL BIT_ULL(10) /* I915_VMA_GLOBAL_BIND */ #define PIN_USER BIT_ULL(11) /* I915_VMA_LOCAL_BIND */ diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c index 7f80ad247bc8..c12a0adefda5 100644 --- a/drivers/gpu/drm/i915/i915_getparam.c +++ b/drivers/gpu/drm/i915/i915_getparam.c @@ -5,7 +5,9 @@ #include "gem/i915_gem_mman.h" #include "gt/intel_engine_user.h" +#include "i915_cmd_parser.h" #include "i915_drv.h" +#include "i915_getparam.h" #include "i915_perf.h" int i915_getparam_ioctl(struct drm_device *dev, void *data, @@ -31,7 +33,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, value = pdev->revision; break; case I915_PARAM_NUM_FENCES_AVAIL: - value = i915->ggtt.num_fences; + value = to_gt(i915)->ggtt->num_fences; break; case I915_PARAM_HAS_OVERLAY: value = !!i915->overlay; diff --git a/drivers/gpu/drm/i915/i915_getparam.h b/drivers/gpu/drm/i915/i915_getparam.h new file mode 100644 index 000000000000..18e4752e8f70 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_getparam.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_GETPARAM_H__ +#define __I915_GETPARAM_H__ + +struct drm_device; +struct drm_file; + +int i915_getparam_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +#endif /* __I915_GETPARAM_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 0633888a411e..1d042551619e 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -34,6 +34,7 @@ #include <linux/utsname.h> #include <linux/zlib.h> +#include <drm/drm_cache.h> #include <drm/drm_print.h> #include "display/intel_dmc.h" @@ -41,14 +42,16 @@ #include "gem/i915_gem_context.h" #include "gem/i915_gem_lmem.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_gt.h" #include "gt/intel_gt_pm.h" +#include "gt/intel_gt_regs.h" +#include "i915_driver.h" #include "i915_drv.h" #include "i915_gpu_error.h" #include "i915_memcpy.h" #include "i915_scatterlist.h" -#include "i915_vma_snapshot.h" #define ALLOW_FAIL (__GFP_KSWAPD_RECLAIM | __GFP_RETRY_MAYFAIL | __GFP_NOWARN) #define ATOMIC_MAYFAIL (GFP_ATOMIC | __GFP_NOWARN) @@ -1013,8 +1016,10 @@ void __i915_gpu_coredump_free(struct kref *error_ref) static struct i915_vma_coredump * i915_vma_coredump_create(const struct intel_gt *gt, - const struct i915_vma_snapshot *vsnap, - struct i915_vma_compress *compress) + const struct i915_vma_resource *vma_res, + struct i915_vma_compress *compress, + const char *name) + { struct i915_ggtt *ggtt = gt->ggtt; const u64 slot = ggtt->error_capture.start; @@ -1024,7 +1029,7 @@ i915_vma_coredump_create(const struct intel_gt *gt, might_sleep(); - if (!vsnap || !vsnap->pages || !compress) + if (!vma_res || !vma_res->bi.pages || !compress) return NULL; dst = kmalloc(sizeof(*dst), ALLOW_FAIL); @@ -1037,12 +1042,12 @@ i915_vma_coredump_create(const struct intel_gt *gt, } INIT_LIST_HEAD(&dst->page_list); - strcpy(dst->name, vsnap->name); + strcpy(dst->name, name); dst->next = NULL; - dst->gtt_offset = vsnap->gtt_offset; - dst->gtt_size = vsnap->gtt_size; - dst->gtt_page_sizes = vsnap->page_sizes; + dst->gtt_offset = vma_res->start; + dst->gtt_size = vma_res->node_size; + dst->gtt_page_sizes = vma_res->page_sizes_gtt; dst->unused = 0; ret = -EINVAL; @@ -1050,7 +1055,7 @@ i915_vma_coredump_create(const struct intel_gt *gt, void __iomem *s; dma_addr_t dma; - for_each_sgt_daddr(dma, iter, vsnap->pages) { + for_each_sgt_daddr(dma, iter, vma_res->bi.pages) { mutex_lock(&ggtt->error_mutex); ggtt->vm.insert_page(&ggtt->vm, dma, slot, I915_CACHE_NONE, 0); @@ -1068,11 +1073,11 @@ i915_vma_coredump_create(const struct intel_gt *gt, if (ret) break; } - } else if (vsnap->mr && vsnap->mr->type != INTEL_MEMORY_SYSTEM) { - struct intel_memory_region *mem = vsnap->mr; + } else if (vma_res->bi.lmem) { + struct intel_memory_region *mem = vma_res->mr; dma_addr_t dma; - for_each_sgt_daddr(dma, iter, vsnap->pages) { + for_each_sgt_daddr(dma, iter, vma_res->bi.pages) { void __iomem *s; s = io_mapping_map_wc(&mem->iomap, @@ -1088,7 +1093,7 @@ i915_vma_coredump_create(const struct intel_gt *gt, } else { struct page *page; - for_each_sgt_page(page, iter, vsnap->pages) { + for_each_sgt_page(page, iter, vma_res->bi.pages) { void *s; drm_clflush_pages(&page, 1); @@ -1324,33 +1329,32 @@ static bool record_context(struct i915_gem_context_coredump *e, struct intel_engine_capture_vma { struct intel_engine_capture_vma *next; - struct i915_vma_snapshot *vsnap; + struct i915_vma_resource *vma_res; char name[16]; bool lockdep_cookie; }; static struct intel_engine_capture_vma * capture_vma_snapshot(struct intel_engine_capture_vma *next, - struct i915_vma_snapshot *vsnap, - gfp_t gfp) + struct i915_vma_resource *vma_res, + gfp_t gfp, const char *name) { struct intel_engine_capture_vma *c; - if (!i915_vma_snapshot_present(vsnap)) + if (!vma_res) return next; c = kmalloc(sizeof(*c), gfp); if (!c) return next; - if (!i915_vma_snapshot_resource_pin(vsnap, &c->lockdep_cookie)) { + if (!i915_vma_resource_hold(vma_res, &c->lockdep_cookie)) { kfree(c); return next; } - strcpy(c->name, vsnap->name); - c->vsnap = vsnap; - i915_vma_snapshot_get(vsnap); + strcpy(c->name, name); + c->vma_res = i915_vma_resource_get(vma_res); c->next = next; return c; @@ -1362,8 +1366,6 @@ capture_vma(struct intel_engine_capture_vma *next, const char *name, gfp_t gfp) { - struct i915_vma_snapshot *vsnap; - if (!vma) return next; @@ -1372,19 +1374,10 @@ capture_vma(struct intel_engine_capture_vma *next, * to a struct i915_vma_snapshot at command submission time. * Not here. */ - GEM_WARN_ON(!i915_vma_is_pinned(vma)); - if (!i915_vma_is_pinned(vma)) - return next; - - vsnap = i915_vma_snapshot_alloc(gfp); - if (!vsnap) + if (GEM_WARN_ON(!i915_vma_is_pinned(vma))) return next; - i915_vma_snapshot_init(vsnap, vma, name); - next = capture_vma_snapshot(next, vsnap, gfp); - - /* FIXME: Replace on async unbind. */ - i915_vma_snapshot_put(vsnap); + next = capture_vma_snapshot(next, vma->resource, gfp, name); return next; } @@ -1397,7 +1390,8 @@ capture_user(struct intel_engine_capture_vma *capture, struct i915_capture_list *c; for (c = rq->capture_list; c; c = c->next) - capture = capture_vma_snapshot(capture, c->vma_snapshot, gfp); + capture = capture_vma_snapshot(capture, c->vma_res, gfp, + "user"); return capture; } @@ -1415,16 +1409,19 @@ static struct i915_vma_coredump * create_vma_coredump(const struct intel_gt *gt, struct i915_vma *vma, const char *name, struct i915_vma_compress *compress) { - struct i915_vma_coredump *ret; - struct i915_vma_snapshot tmp; + struct i915_vma_coredump *ret = NULL; + struct i915_vma_resource *vma_res; + bool lockdep_cookie; if (!vma) return NULL; - GEM_WARN_ON(!i915_vma_is_pinned(vma)); - i915_vma_snapshot_init_onstack(&tmp, vma, name); - ret = i915_vma_coredump_create(gt, &tmp, compress); - i915_vma_snapshot_put_onstack(&tmp); + vma_res = vma->resource; + + if (i915_vma_resource_hold(vma_res, &lockdep_cookie)) { + ret = i915_vma_coredump_create(gt, vma_res, compress, name); + i915_vma_resource_unhold(vma_res, lockdep_cookie); + } return ret; } @@ -1471,7 +1468,7 @@ intel_engine_coredump_add_request(struct intel_engine_coredump *ee, * as the simplest method to avoid being overwritten * by userspace. */ - vma = capture_vma_snapshot(vma, &rq->batch_snapshot, gfp); + vma = capture_vma_snapshot(vma, rq->batch_res, gfp, "batch"); vma = capture_user(vma, rq, gfp); vma = capture_vma(vma, rq->ring->vma, "ring", gfp); vma = capture_vma(vma, rq->context->state, "HW context", gfp); @@ -1492,14 +1489,14 @@ intel_engine_coredump_add_vma(struct intel_engine_coredump *ee, while (capture) { struct intel_engine_capture_vma *this = capture; - struct i915_vma_snapshot *vsnap = this->vsnap; + struct i915_vma_resource *vma_res = this->vma_res; add_vma(ee, - i915_vma_coredump_create(engine->gt, - vsnap, compress)); + i915_vma_coredump_create(engine->gt, vma_res, + compress, this->name)); - i915_vma_snapshot_resource_unpin(vsnap, this->lockdep_cookie); - i915_vma_snapshot_put(vsnap); + i915_vma_resource_unhold(vma_res, this->lockdep_cookie); + i915_vma_resource_put(vma_res); capture = this->next; kfree(this); diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h index 5aedf5129814..903d838e2e63 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.h +++ b/drivers/gpu/drm/i915/i915_gpu_error.h @@ -210,6 +210,17 @@ struct drm_i915_error_state_buf { int err; }; +static inline u32 i915_reset_count(struct i915_gpu_error *error) +{ + return atomic_read(&error->reset_count); +} + +static inline u32 i915_reset_engine_count(struct i915_gpu_error *error, + const struct intel_engine_cs *engine) +{ + return atomic_read(&error->reset_engine_count[engine->uabi_class]); +} + #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) __printf(2, 3) diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c index 55b97c3a3dde..33348960f623 100644 --- a/drivers/gpu/drm/i915/i915_ioc32.c +++ b/drivers/gpu/drm/i915/i915_ioc32.c @@ -31,6 +31,7 @@ #include <drm/drm_ioctl.h> #include "i915_drv.h" +#include "i915_getparam.h" #include "i915_ioc32.h" struct drm_i915_getparam32 { diff --git a/drivers/gpu/drm/i915/i915_ioctl.c b/drivers/gpu/drm/i915/i915_ioctl.c new file mode 100644 index 000000000000..06a10ccea80b --- /dev/null +++ b/drivers/gpu/drm/i915/i915_ioctl.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "gt/intel_engine_regs.h" + +#include "i915_drv.h" +#include "i915_gem.h" +#include "i915_ioctl.h" +#include "i915_reg.h" +#include "intel_runtime_pm.h" +#include "intel_uncore.h" + +/* + * This file is for small ioctl functions that are out of place everywhere else, + * and not big enough to warrant a file of their own. + * + * This is not the dumping ground for random ioctls. + */ + +struct reg_whitelist { + i915_reg_t offset_ldw; + i915_reg_t offset_udw; + u8 min_graphics_ver; + u8 max_graphics_ver; + u8 size; +}; + +static const struct reg_whitelist reg_read_whitelist[] = { + { + .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE), + .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE), + .min_graphics_ver = 4, + .max_graphics_ver = 12, + .size = 8 + } +}; + +int i915_reg_read_ioctl(struct drm_device *dev, + void *data, struct drm_file *unused) +{ + struct drm_i915_private *i915 = to_i915(dev); + struct intel_uncore *uncore = &i915->uncore; + struct drm_i915_reg_read *reg = data; + struct reg_whitelist const *entry; + intel_wakeref_t wakeref; + unsigned int flags; + int remain; + int ret = 0; + + entry = reg_read_whitelist; + remain = ARRAY_SIZE(reg_read_whitelist); + while (remain) { + u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw); + + GEM_BUG_ON(!is_power_of_2(entry->size)); + GEM_BUG_ON(entry->size > 8); + GEM_BUG_ON(entry_offset & (entry->size - 1)); + + if (IS_GRAPHICS_VER(i915, entry->min_graphics_ver, entry->max_graphics_ver) && + entry_offset == (reg->offset & -entry->size)) + break; + entry++; + remain--; + } + + if (!remain) + return -EINVAL; + + flags = reg->offset & (entry->size - 1); + + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + if (entry->size == 8 && flags == I915_REG_READ_8B_WA) + reg->val = intel_uncore_read64_2x32(uncore, + entry->offset_ldw, + entry->offset_udw); + else if (entry->size == 8 && flags == 0) + reg->val = intel_uncore_read64(uncore, + entry->offset_ldw); + else if (entry->size == 4 && flags == 0) + reg->val = intel_uncore_read(uncore, entry->offset_ldw); + else if (entry->size == 2 && flags == 0) + reg->val = intel_uncore_read16(uncore, + entry->offset_ldw); + else if (entry->size == 1 && flags == 0) + reg->val = intel_uncore_read8(uncore, + entry->offset_ldw); + else + ret = -EINVAL; + } + + return ret; +} diff --git a/drivers/gpu/drm/i915/i915_ioctl.h b/drivers/gpu/drm/i915/i915_ioctl.h new file mode 100644 index 000000000000..f16ae87b8b8a --- /dev/null +++ b/drivers/gpu/drm/i915/i915_ioctl.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __I915_IOCTL_H__ +#define __I915_IOCTL_H__ + +struct drm_device; +struct drm_file; + +int i915_reg_read_ioctl(struct drm_device *dev, void *data, struct drm_file *file); + +#endif /* __I915_IOCTL_H__ */ diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 21f75b069fa8..73cebc6aa650 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -34,6 +34,7 @@ #include <drm/drm_drv.h> +#include "display/icl_dsi_regs.h" #include "display/intel_de.h" #include "display/intel_display_trace.h" #include "display/intel_display_types.h" @@ -46,8 +47,10 @@ #include "gt/intel_gt.h" #include "gt/intel_gt_irq.h" #include "gt/intel_gt_pm_irq.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_rps.h" +#include "i915_driver.h" #include "i915_drv.h" #include "i915_irq.h" #include "intel_pm.h" @@ -177,6 +180,7 @@ static const u32 hpd_sde_dg1[HPD_NUM_PINS] = { [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_B), [HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_C), [HPD_PORT_D] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_D), + [HPD_PORT_TC1] = SDE_TC_HOTPLUG_DG2(HPD_PORT_TC1), }; static void intel_hpd_init_pins(struct drm_i915_private *dev_priv) @@ -836,10 +840,7 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) if (mode->flags & DRM_MODE_FLAG_INTERLACE) vtotal /= 2; - if (DISPLAY_VER(dev_priv) == 2) - position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; - else - position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; + position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & PIPEDSL_LINE_MASK; /* * On HSW, the DSL reg (0x70000) appears to return 0 if we @@ -858,7 +859,7 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) for (i = 0; i < 100; i++) { udelay(1); - temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; + temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & PIPEDSL_LINE_MASK; if (temp != position) { position = temp; break; @@ -4349,6 +4350,10 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) return ret; } +struct intel_hotplug_funcs { + void (*hpd_irq_setup)(struct drm_i915_private *i915); +}; + #define HPD_FUNCS(platform) \ static const struct intel_hotplug_funcs platform##_hpd_funcs = { \ .hpd_irq_setup = platform##_hpd_irq_setup, \ @@ -4363,6 +4368,12 @@ HPD_FUNCS(spt); HPD_FUNCS(ilk); #undef HPD_FUNCS +void intel_hpd_irq_setup(struct drm_i915_private *i915) +{ + if (i915->display_irqs_enabled && i915->hotplug_funcs) + i915->hotplug_funcs->hpd_irq_setup(i915); +} + /** * intel_irq_init - initializes irq support * @dev_priv: i915 device instance @@ -4415,7 +4426,9 @@ void intel_irq_init(struct drm_i915_private *dev_priv) if (I915_HAS_HOTPLUG(dev_priv)) dev_priv->hotplug_funcs = &i915_hpd_funcs; } else { - if (HAS_PCH_DG1(dev_priv)) + if (HAS_PCH_DG2(dev_priv)) + dev_priv->hotplug_funcs = &icp_hpd_funcs; + else if (HAS_PCH_DG1(dev_priv)) dev_priv->hotplug_funcs = &dg1_hpd_funcs; else if (DISPLAY_VER(dev_priv) >= 11) dev_priv->hotplug_funcs = &gen11_hpd_funcs; diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h index 0eb90d271fa7..82639d9d7e82 100644 --- a/drivers/gpu/drm/i915/i915_irq.h +++ b/drivers/gpu/drm/i915/i915_irq.h @@ -37,6 +37,7 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv); void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv); +void intel_hpd_irq_setup(struct drm_i915_private *i915); void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv, u32 mask, u32 bits); diff --git a/drivers/gpu/drm/i915/i915_mitigations.c b/drivers/gpu/drm/i915/i915_mitigations.c index 84f12598d145..def7302ef7fe 100644 --- a/drivers/gpu/drm/i915/i915_mitigations.c +++ b/drivers/gpu/drm/i915/i915_mitigations.c @@ -8,6 +8,7 @@ #include <linux/slab.h> #include <linux/string.h> +#include "i915_driver.h" #include "i915_drv.h" #include "i915_mitigations.h" diff --git a/drivers/gpu/drm/i915/i915_mm.h b/drivers/gpu/drm/i915/i915_mm.h index 3ad22bbe80eb..04c8974d822b 100644 --- a/drivers/gpu/drm/i915/i915_mm.h +++ b/drivers/gpu/drm/i915/i915_mm.h @@ -23,8 +23,7 @@ int remap_io_mapping(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, struct io_mapping *iomap) { - pr_err("Architecture has no %s() and shouldn't be calling this function\n", __func__); - WARN_ON_ONCE(1); + WARN_ONCE(1, "Architecture has no drm_cache.c support\n"); return 0; } #endif diff --git a/drivers/gpu/drm/i915/i915_module.c b/drivers/gpu/drm/i915/i915_module.c index f6bcd2f89257..65acd7bf75d0 100644 --- a/drivers/gpu/drm/i915/i915_module.c +++ b/drivers/gpu/drm/i915/i915_module.c @@ -9,7 +9,7 @@ #include "gem/i915_gem_context.h" #include "gem/i915_gem_object.h" #include "i915_active.h" -#include "i915_buddy.h" +#include "i915_driver.h" #include "i915_params.h" #include "i915_pci.h" #include "i915_perf.h" @@ -17,6 +17,7 @@ #include "i915_scheduler.h" #include "i915_selftest.h" #include "i915_vma.h" +#include "i915_vma_resource.h" static int i915_check_nomodeset(void) { @@ -50,8 +51,6 @@ static const struct { { .init = i915_check_nomodeset }, { .init = i915_active_module_init, .exit = i915_active_module_exit }, - { .init = i915_buddy_module_init, - .exit = i915_buddy_module_exit }, { .init = i915_context_module_init, .exit = i915_context_module_exit }, { .init = i915_gem_context_module_init, @@ -64,6 +63,8 @@ static const struct { .exit = i915_scheduler_module_exit }, { .init = i915_vma_module_init, .exit = i915_vma_module_exit }, + { .init = i915_vma_resource_module_init, + .exit = i915_vma_resource_module_exit }, { .init = i915_mock_selftests }, { .init = i915_pmu_init, .exit = i915_pmu_exit }, diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index 525ae832aa9a..eea355c2fc28 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -195,11 +195,6 @@ i915_param_named(enable_gvt, bool, 0400, "Enable support for Intel GVT-g graphics virtualization host support(default:false)"); #endif -#if IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM) -i915_param_named_unsafe(fake_lmem_start, ulong, 0400, - "Fake LMEM start offset (default: 0)"); -#endif - #if CONFIG_DRM_I915_REQUEST_TIMEOUT i915_param_named_unsafe(request_timeout_ms, uint, 0600, "Default request/fence/batch buffer expiration timeout."); diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index c9d53ff910a0..c779a6f85c7e 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -72,7 +72,6 @@ struct drm_printer; param(int, fastboot, -1, 0600) \ param(int, enable_dpcd_backlight, -1, 0600) \ param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \ - param(unsigned long, fake_lmem_start, 0, IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM) ? 0400 : 0) \ param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, CONFIG_DRM_I915_REQUEST_TIMEOUT ? 0600 : 0) \ /* leave bools at the end to not create holes */ \ param(bool, enable_hangcheck, true, 0600) \ diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 261294df535c..c32c0c6661c8 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -22,12 +22,14 @@ * */ +#include <drm/drm_color_mgmt.h> #include <drm/drm_drv.h> #include <drm/i915_pciids.h> #include "i915_driver.h" #include "i915_drv.h" #include "i915_pci.h" +#include "i915_reg.h" #define PLATFORM(x) .platform = (x) #define GEN(x) \ @@ -214,13 +216,13 @@ static const struct intel_device_info i845g_info = { static const struct intel_device_info i85x_info = { I830_FEATURES, PLATFORM(INTEL_I85X), - .display.has_fbc = 1, + .display.fbc_mask = BIT(INTEL_FBC_A), }; static const struct intel_device_info i865g_info = { I845_FEATURES, PLATFORM(INTEL_I865G), - .display.has_fbc = 1, + .display.fbc_mask = BIT(INTEL_FBC_A), }; #define GEN3_FEATURES \ @@ -258,7 +260,7 @@ static const struct intel_device_info i915gm_info = { .display.has_overlay = 1, .display.overlay_needs_physical = 1, .display.supports_tv = 1, - .display.has_fbc = 1, + .display.fbc_mask = BIT(INTEL_FBC_A), .hws_needs_physical = 1, .unfenced_needs_alignment = 1, }; @@ -283,7 +285,7 @@ static const struct intel_device_info i945gm_info = { .display.has_overlay = 1, .display.overlay_needs_physical = 1, .display.supports_tv = 1, - .display.has_fbc = 1, + .display.fbc_mask = BIT(INTEL_FBC_A), .hws_needs_physical = 1, .unfenced_needs_alignment = 1, }; @@ -342,7 +344,7 @@ static const struct intel_device_info i965gm_info = { GEN4_FEATURES, PLATFORM(INTEL_I965GM), .is_mobile = 1, - .display.has_fbc = 1, + .display.fbc_mask = BIT(INTEL_FBC_A), .display.has_overlay = 1, .display.supports_tv = 1, .hws_needs_physical = 1, @@ -360,7 +362,7 @@ static const struct intel_device_info gm45_info = { GEN4_FEATURES, PLATFORM(INTEL_GM45), .is_mobile = 1, - .display.has_fbc = 1, + .display.fbc_mask = BIT(INTEL_FBC_A), .display.supports_tv = 1, .platform_engine_mask = BIT(RCS0) | BIT(VCS0), .gpu_reset_clobbers_display = false, @@ -393,7 +395,7 @@ static const struct intel_device_info ilk_m_info = { PLATFORM(INTEL_IRONLAKE), .is_mobile = 1, .has_rps = true, - .display.has_fbc = 1, + .display.fbc_mask = BIT(INTEL_FBC_A), }; #define GEN6_FEATURES \ @@ -401,7 +403,7 @@ static const struct intel_device_info ilk_m_info = { .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_hotplug = 1, \ - .display.has_fbc = 1, \ + .display.fbc_mask = BIT(INTEL_FBC_A), \ .platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ .has_coherent_ggtt = true, \ .has_llc = 1, \ @@ -452,7 +454,7 @@ static const struct intel_device_info snb_m_gt2_info = { .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \ .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), \ .display.has_hotplug = 1, \ - .display.has_fbc = 1, \ + .display.fbc_mask = BIT(INTEL_FBC_A), \ .platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ .has_coherent_ggtt = true, \ .has_llc = 1, \ @@ -693,7 +695,7 @@ static const struct intel_device_info skl_gt4_info = { .has_64bit_reloc = 1, \ .display.has_ddi = 1, \ .display.has_fpga_dbg = 1, \ - .display.has_fbc = 1, \ + .display.fbc_mask = BIT(INTEL_FBC_A), \ .display.has_hdcp = 1, \ .display.has_psr = 1, \ .display.has_psr_hw_tracking = 1, \ @@ -948,7 +950,7 @@ static const struct intel_device_info adl_s_info = { .display.has_dp_mst = 1, \ .display.has_dsb = 1, \ .display.has_dsc = 1, \ - .display.has_fbc = 1, \ + .display.fbc_mask = BIT(INTEL_FBC_A), \ .display.has_fpga_dbg = 1, \ .display.has_hdcp = 1, \ .display.has_hotplug = 1, \ @@ -1003,6 +1005,7 @@ static const struct intel_device_info adl_p_info = { XE_HP_PAGE_SIZES, \ .dma_mask_size = 46, \ .has_64bit_reloc = 1, \ + .has_flat_ccs = 1, \ .has_global_mocs = 1, \ .has_gt_uc = 1, \ .has_llc = 1, \ @@ -1028,6 +1031,7 @@ static const struct intel_device_info xehpsdv_info = { PLATFORM(INTEL_XEHPSDV), .display = { }, .has_64k_pages = 1, + .needs_compact_pt = 1, .platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VECS1) | BIT(VECS2) | BIT(VECS3) | @@ -1045,7 +1049,9 @@ static const struct intel_device_info dg2_info = { .graphics.rel = 55, .media.rel = 55, PLATFORM(INTEL_DG2), + .has_guc_deprivilege = 1, .has_64k_pages = 1, + .needs_compact_pt = 1, .platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VECS1) | @@ -1131,6 +1137,7 @@ static const struct pci_device_id pciidlist[] = { INTEL_RKL_IDS(&rkl_info), INTEL_ADLS_IDS(&adl_s_info), INTEL_ADLP_IDS(&adl_p_info), + INTEL_ADLN_IDS(&adl_p_info), INTEL_DG1_IDS(&dg1_info), INTEL_RPLS_IDS(&adl_s_info), {0, 0, 0} diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index e27f3b7cf094..0a9c3fcc09b1 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -196,17 +196,23 @@ #include <linux/uuid.h> #include "gem/i915_gem_context.h" +#include "gem/i915_gem_internal.h" #include "gt/intel_engine_pm.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_engine_user.h" #include "gt/intel_execlists_submission.h" #include "gt/intel_gpu_commands.h" #include "gt/intel_gt.h" #include "gt/intel_gt_clock_utils.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_lrc.h" +#include "gt/intel_lrc_reg.h" #include "gt/intel_ring.h" #include "i915_drv.h" +#include "i915_file_private.h" #include "i915_perf.h" +#include "i915_perf_oa_regs.h" /* HW requires this to be a power of two, between 128k and 16M, though driver * is currently generally designed assuming the largest 16M size is used such @@ -1630,8 +1636,8 @@ static int alloc_noa_wait(struct i915_perf_stream *stream) struct drm_i915_gem_object *bo; struct i915_vma *vma; const u64 delay_ticks = 0xffffffffffffffff - - intel_gt_ns_to_clock_interval(stream->perf->i915->ggtt.vm.gt, - atomic64_read(&stream->perf->noa_programming_delay)); + intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915), + atomic64_read(&stream->perf->noa_programming_delay)); const u32 base = stream->engine->mmio_base; #define CS_GPR(x) GEN8_RING_CS_GPR(base, x) u32 *batch, *ts0, *cs, *jump; @@ -1682,7 +1688,7 @@ retry: stream, cs, true /* save */, CS_GPR(i), INTEL_GT_SCRATCH_FIELD_PERF_CS_GPR + 8 * i, 2); cs = save_restore_register( - stream, cs, true /* save */, MI_PREDICATE_RESULT_1, + stream, cs, true /* save */, MI_PREDICATE_RESULT_1(RENDER_RING_BASE), INTEL_GT_SCRATCH_FIELD_PERF_PREDICATE_RESULT_1, 1); /* First timestamp snapshot location. */ @@ -1736,7 +1742,7 @@ retry: */ *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); *cs++ = i915_mmio_reg_offset(CS_GPR(JUMP_PREDICATE)); - *cs++ = i915_mmio_reg_offset(MI_PREDICATE_RESULT_1); + *cs++ = i915_mmio_reg_offset(MI_PREDICATE_RESULT_1(RENDER_RING_BASE)); /* Restart from the beginning if we had timestamps roll over. */ *cs++ = (GRAPHICS_VER(i915) < 8 ? @@ -1773,7 +1779,7 @@ retry: */ *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); *cs++ = i915_mmio_reg_offset(CS_GPR(JUMP_PREDICATE)); - *cs++ = i915_mmio_reg_offset(MI_PREDICATE_RESULT_1); + *cs++ = i915_mmio_reg_offset(MI_PREDICATE_RESULT_1(RENDER_RING_BASE)); /* Predicate the jump. */ *cs++ = (GRAPHICS_VER(i915) < 8 ? @@ -1789,7 +1795,7 @@ retry: stream, cs, false /* restore */, CS_GPR(i), INTEL_GT_SCRATCH_FIELD_PERF_CS_GPR + 8 * i, 2); cs = save_restore_register( - stream, cs, false /* restore */, MI_PREDICATE_RESULT_1, + stream, cs, false /* restore */, MI_PREDICATE_RESULT_1(RENDER_RING_BASE), INTEL_GT_SCRATCH_FIELD_PERF_PREDICATE_RESULT_1, 1); /* And return to the ring. */ @@ -2114,7 +2120,7 @@ gen8_update_reg_state_unlocked(const struct intel_context *ce, u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; /* The MMIO offsets for Flex EU registers aren't contiguous */ - i915_reg_t flex_regs[] = { + static const i915_reg_t flex_regs[] = { EU_PERF_CNTL0, EU_PERF_CNTL1, EU_PERF_CNTL2, @@ -2418,7 +2424,7 @@ gen12_configure_all_contexts(struct i915_perf_stream *stream, { struct flex regs[] = { { - GEN8_R_PWR_CLK_STATE, + GEN8_R_PWR_CLK_STATE(RENDER_RING_BASE), CTX_R_PWR_CLK_STATE, }, }; @@ -2438,7 +2444,7 @@ lrc_configure_all_contexts(struct i915_perf_stream *stream, #define ctx_flexeuN(N) (ctx_flexeu0 + 2 * (N) + 1) struct flex regs[] = { { - GEN8_R_PWR_CLK_STATE, + GEN8_R_PWR_CLK_STATE(RENDER_RING_BASE), CTX_R_PWR_CLK_STATE, }, { @@ -3542,7 +3548,7 @@ err: static u64 oa_exponent_to_ns(struct i915_perf *perf, int exponent) { - return intel_gt_clock_interval_to_ns(perf->i915->ggtt.vm.gt, + return intel_gt_clock_interval_to_ns(to_gt(perf->i915), 2ULL << exponent); } @@ -3862,80 +3868,116 @@ static bool gen8_is_valid_flex_addr(struct i915_perf *perf, u32 addr) return false; } -#define ADDR_IN_RANGE(addr, start, end) \ - ((addr) >= (start) && \ - (addr) <= (end)) +static bool reg_in_range_table(u32 addr, const struct i915_range *table) +{ + while (table->start || table->end) { + if (addr >= table->start && addr <= table->end) + return true; -#define REG_IN_RANGE(addr, start, end) \ - ((addr) >= i915_mmio_reg_offset(start) && \ - (addr) <= i915_mmio_reg_offset(end)) + table++; + } + + return false; +} #define REG_EQUAL(addr, mmio) \ ((addr) == i915_mmio_reg_offset(mmio)) -static bool gen7_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr) -{ - return REG_IN_RANGE(addr, OASTARTTRIG1, OASTARTTRIG8) || - REG_IN_RANGE(addr, OAREPORTTRIG1, OAREPORTTRIG8) || - REG_IN_RANGE(addr, OACEC0_0, OACEC7_1); -} +static const struct i915_range gen7_oa_b_counters[] = { + { .start = 0x2710, .end = 0x272c }, /* OASTARTTRIG[1-8] */ + { .start = 0x2740, .end = 0x275c }, /* OAREPORTTRIG[1-8] */ + { .start = 0x2770, .end = 0x27ac }, /* OACEC[0-7][0-1] */ + {} +}; + +static const struct i915_range gen12_oa_b_counters[] = { + { .start = 0x2b2c, .end = 0x2b2c }, /* GEN12_OAG_OA_PESS */ + { .start = 0xd900, .end = 0xd91c }, /* GEN12_OAG_OASTARTTRIG[1-8] */ + { .start = 0xd920, .end = 0xd93c }, /* GEN12_OAG_OAREPORTTRIG1[1-8] */ + { .start = 0xd940, .end = 0xd97c }, /* GEN12_OAG_CEC[0-7][0-1] */ + { .start = 0xdc00, .end = 0xdc3c }, /* GEN12_OAG_SCEC[0-7][0-1] */ + { .start = 0xdc40, .end = 0xdc40 }, /* GEN12_OAG_SPCTR_CNF */ + { .start = 0xdc44, .end = 0xdc44 }, /* GEN12_OAA_DBG_REG */ + {} +}; -static bool gen7_is_valid_mux_addr(struct i915_perf *perf, u32 addr) +static const struct i915_range gen7_oa_mux_regs[] = { + { .start = 0x91b8, .end = 0x91cc }, /* OA_PERFCNT[1-2], OA_PERFMATRIX */ + { .start = 0x9800, .end = 0x9888 }, /* MICRO_BP0_0 - NOA_WRITE */ + { .start = 0xe180, .end = 0xe180 }, /* HALF_SLICE_CHICKEN2 */ + {} +}; + +static const struct i915_range hsw_oa_mux_regs[] = { + { .start = 0x09e80, .end = 0x09ea4 }, /* HSW_MBVID2_NOA[0-9] */ + { .start = 0x09ec0, .end = 0x09ec0 }, /* HSW_MBVID2_MISR0 */ + { .start = 0x25100, .end = 0x2ff90 }, + {} +}; + +static const struct i915_range chv_oa_mux_regs[] = { + { .start = 0x182300, .end = 0x1823a4 }, + {} +}; + +static const struct i915_range gen8_oa_mux_regs[] = { + { .start = 0x0d00, .end = 0x0d2c }, /* RPM_CONFIG[0-1], NOA_CONFIG[0-8] */ + { .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */ + {} +}; + +static const struct i915_range gen11_oa_mux_regs[] = { + { .start = 0x91c8, .end = 0x91dc }, /* OA_PERFCNT[3-4] */ + {} +}; + +static const struct i915_range gen12_oa_mux_regs[] = { + { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */ + { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */ + { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */ + { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */ + { .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */ + {} +}; + +static bool gen7_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr) { - return REG_EQUAL(addr, HALF_SLICE_CHICKEN2) || - REG_IN_RANGE(addr, MICRO_BP0_0, NOA_WRITE) || - REG_IN_RANGE(addr, OA_PERFCNT1_LO, OA_PERFCNT2_HI) || - REG_IN_RANGE(addr, OA_PERFMATRIX_LO, OA_PERFMATRIX_HI); + return reg_in_range_table(addr, gen7_oa_b_counters); } static bool gen8_is_valid_mux_addr(struct i915_perf *perf, u32 addr) { - return gen7_is_valid_mux_addr(perf, addr) || - REG_EQUAL(addr, WAIT_FOR_RC6_EXIT) || - REG_IN_RANGE(addr, RPM_CONFIG0, NOA_CONFIG(8)); + return reg_in_range_table(addr, gen7_oa_mux_regs) || + reg_in_range_table(addr, gen8_oa_mux_regs); } static bool gen11_is_valid_mux_addr(struct i915_perf *perf, u32 addr) { - return gen8_is_valid_mux_addr(perf, addr) || - REG_EQUAL(addr, GEN10_NOA_WRITE_HIGH) || - REG_IN_RANGE(addr, OA_PERFCNT3_LO, OA_PERFCNT4_HI); + return reg_in_range_table(addr, gen7_oa_mux_regs) || + reg_in_range_table(addr, gen8_oa_mux_regs) || + reg_in_range_table(addr, gen11_oa_mux_regs); } static bool hsw_is_valid_mux_addr(struct i915_perf *perf, u32 addr) { - return gen7_is_valid_mux_addr(perf, addr) || - ADDR_IN_RANGE(addr, 0x25100, 0x2FF90) || - REG_IN_RANGE(addr, HSW_MBVID2_NOA0, HSW_MBVID2_NOA9) || - REG_EQUAL(addr, HSW_MBVID2_MISR0); + return reg_in_range_table(addr, gen7_oa_mux_regs) || + reg_in_range_table(addr, hsw_oa_mux_regs); } static bool chv_is_valid_mux_addr(struct i915_perf *perf, u32 addr) { - return gen7_is_valid_mux_addr(perf, addr) || - ADDR_IN_RANGE(addr, 0x182300, 0x1823A4); + return reg_in_range_table(addr, gen7_oa_mux_regs) || + reg_in_range_table(addr, chv_oa_mux_regs); } static bool gen12_is_valid_b_counter_addr(struct i915_perf *perf, u32 addr) { - return REG_IN_RANGE(addr, GEN12_OAG_OASTARTTRIG1, GEN12_OAG_OASTARTTRIG8) || - REG_IN_RANGE(addr, GEN12_OAG_OAREPORTTRIG1, GEN12_OAG_OAREPORTTRIG8) || - REG_IN_RANGE(addr, GEN12_OAG_CEC0_0, GEN12_OAG_CEC7_1) || - REG_IN_RANGE(addr, GEN12_OAG_SCEC0_0, GEN12_OAG_SCEC7_1) || - REG_EQUAL(addr, GEN12_OAA_DBG_REG) || - REG_EQUAL(addr, GEN12_OAG_OA_PESS) || - REG_EQUAL(addr, GEN12_OAG_SPCTR_CNF); + return reg_in_range_table(addr, gen12_oa_b_counters); } static bool gen12_is_valid_mux_addr(struct i915_perf *perf, u32 addr) { - return REG_EQUAL(addr, NOA_WRITE) || - REG_EQUAL(addr, GEN10_NOA_WRITE_HIGH) || - REG_EQUAL(addr, GDT_CHICKEN_BITS) || - REG_EQUAL(addr, WAIT_FOR_RC6_EXIT) || - REG_EQUAL(addr, RPM_CONFIG0) || - REG_EQUAL(addr, RPM_CONFIG1) || - REG_IN_RANGE(addr, NOA_CONFIG(0), NOA_CONFIG(8)); + return reg_in_range_table(addr, gen12_oa_mux_regs); } static u32 mask_reg_value(u32 reg, u32 val) @@ -4332,6 +4374,10 @@ void i915_perf_init(struct drm_i915_private *i915) /* XXX const struct i915_perf_ops! */ + /* i915_perf is not enabled for DG2 yet */ + if (IS_DG2(i915)) + return; + perf->oa_formats = oa_formats; if (IS_HASWELL(i915)) { perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr; diff --git a/drivers/gpu/drm/i915/i915_perf_oa_regs.h b/drivers/gpu/drm/i915/i915_perf_oa_regs.h new file mode 100644 index 000000000000..f31c9f13a9fc --- /dev/null +++ b/drivers/gpu/drm/i915/i915_perf_oa_regs.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_PERF_OA_REGS__ +#define __INTEL_PERF_OA_REGS__ + +#include "i915_reg_defs.h" + +#define GEN7_OACONTROL _MMIO(0x2360) +#define GEN7_OACONTROL_CTX_MASK 0xFFFFF000 +#define GEN7_OACONTROL_TIMER_PERIOD_MASK 0x3F +#define GEN7_OACONTROL_TIMER_PERIOD_SHIFT 6 +#define GEN7_OACONTROL_TIMER_ENABLE (1 << 5) +#define GEN7_OACONTROL_FORMAT_A13 (0 << 2) +#define GEN7_OACONTROL_FORMAT_A29 (1 << 2) +#define GEN7_OACONTROL_FORMAT_A13_B8_C8 (2 << 2) +#define GEN7_OACONTROL_FORMAT_A29_B8_C8 (3 << 2) +#define GEN7_OACONTROL_FORMAT_B4_C8 (4 << 2) +#define GEN7_OACONTROL_FORMAT_A45_B8_C8 (5 << 2) +#define GEN7_OACONTROL_FORMAT_B4_C8_A16 (6 << 2) +#define GEN7_OACONTROL_FORMAT_C4_B8 (7 << 2) +#define GEN7_OACONTROL_FORMAT_SHIFT 2 +#define GEN7_OACONTROL_PER_CTX_ENABLE (1 << 1) +#define GEN7_OACONTROL_ENABLE (1 << 0) + +#define GEN8_OACTXID _MMIO(0x2364) + +#define GEN8_OA_DEBUG _MMIO(0x2B04) +#define GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS (1 << 5) +#define GEN9_OA_DEBUG_INCLUDE_CLK_RATIO (1 << 6) +#define GEN9_OA_DEBUG_DISABLE_GO_1_0_REPORTS (1 << 2) +#define GEN9_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS (1 << 1) + +#define GEN8_OACONTROL _MMIO(0x2B00) +#define GEN8_OA_REPORT_FORMAT_A12 (0 << 2) +#define GEN8_OA_REPORT_FORMAT_A12_B8_C8 (2 << 2) +#define GEN8_OA_REPORT_FORMAT_A36_B8_C8 (5 << 2) +#define GEN8_OA_REPORT_FORMAT_C4_B8 (7 << 2) +#define GEN8_OA_REPORT_FORMAT_SHIFT 2 +#define GEN8_OA_SPECIFIC_CONTEXT_ENABLE (1 << 1) +#define GEN8_OA_COUNTER_ENABLE (1 << 0) + +#define GEN8_OACTXCONTROL _MMIO(0x2360) +#define GEN8_OA_TIMER_PERIOD_MASK 0x3F +#define GEN8_OA_TIMER_PERIOD_SHIFT 2 +#define GEN8_OA_TIMER_ENABLE (1 << 1) +#define GEN8_OA_COUNTER_RESUME (1 << 0) + +#define GEN7_OABUFFER _MMIO(0x23B0) /* R/W */ +#define GEN7_OABUFFER_OVERRUN_DISABLE (1 << 3) +#define GEN7_OABUFFER_EDGE_TRIGGER (1 << 2) +#define GEN7_OABUFFER_STOP_RESUME_ENABLE (1 << 1) +#define GEN7_OABUFFER_RESUME (1 << 0) + +#define GEN8_OABUFFER_UDW _MMIO(0x23b4) +#define GEN8_OABUFFER _MMIO(0x2b14) +#define GEN8_OABUFFER_MEM_SELECT_GGTT (1 << 0) /* 0: PPGTT, 1: GGTT */ + +#define GEN7_OASTATUS1 _MMIO(0x2364) +#define GEN7_OASTATUS1_TAIL_MASK 0xffffffc0 +#define GEN7_OASTATUS1_COUNTER_OVERFLOW (1 << 2) +#define GEN7_OASTATUS1_OABUFFER_OVERFLOW (1 << 1) +#define GEN7_OASTATUS1_REPORT_LOST (1 << 0) + +#define GEN7_OASTATUS2 _MMIO(0x2368) +#define GEN7_OASTATUS2_HEAD_MASK 0xffffffc0 +#define GEN7_OASTATUS2_MEM_SELECT_GGTT (1 << 0) /* 0: PPGTT, 1: GGTT */ + +#define GEN8_OASTATUS _MMIO(0x2b08) +#define GEN8_OASTATUS_TAIL_POINTER_WRAP (1 << 17) +#define GEN8_OASTATUS_HEAD_POINTER_WRAP (1 << 16) +#define GEN8_OASTATUS_OVERRUN_STATUS (1 << 3) +#define GEN8_OASTATUS_COUNTER_OVERFLOW (1 << 2) +#define GEN8_OASTATUS_OABUFFER_OVERFLOW (1 << 1) +#define GEN8_OASTATUS_REPORT_LOST (1 << 0) + +#define GEN8_OAHEADPTR _MMIO(0x2B0C) +#define GEN8_OAHEADPTR_MASK 0xffffffc0 +#define GEN8_OATAILPTR _MMIO(0x2B10) +#define GEN8_OATAILPTR_MASK 0xffffffc0 + +#define OABUFFER_SIZE_128K (0 << 3) +#define OABUFFER_SIZE_256K (1 << 3) +#define OABUFFER_SIZE_512K (2 << 3) +#define OABUFFER_SIZE_1M (3 << 3) +#define OABUFFER_SIZE_2M (4 << 3) +#define OABUFFER_SIZE_4M (5 << 3) +#define OABUFFER_SIZE_8M (6 << 3) +#define OABUFFER_SIZE_16M (7 << 3) + +#define GEN12_OA_TLB_INV_CR _MMIO(0xceec) + +/* Gen12 OAR unit */ +#define GEN12_OAR_OACONTROL _MMIO(0x2960) +#define GEN12_OAR_OACONTROL_COUNTER_FORMAT_SHIFT 1 +#define GEN12_OAR_OACONTROL_COUNTER_ENABLE (1 << 0) + +#define GEN12_OACTXCONTROL _MMIO(0x2360) +#define GEN12_OAR_OASTATUS _MMIO(0x2968) + +/* Gen12 OAG unit */ +#define GEN12_OAG_OAHEADPTR _MMIO(0xdb00) +#define GEN12_OAG_OAHEADPTR_MASK 0xffffffc0 +#define GEN12_OAG_OATAILPTR _MMIO(0xdb04) +#define GEN12_OAG_OATAILPTR_MASK 0xffffffc0 + +#define GEN12_OAG_OABUFFER _MMIO(0xdb08) +#define GEN12_OAG_OABUFFER_BUFFER_SIZE_MASK (0x7) +#define GEN12_OAG_OABUFFER_BUFFER_SIZE_SHIFT (3) +#define GEN12_OAG_OABUFFER_MEMORY_SELECT (1 << 0) /* 0: PPGTT, 1: GGTT */ + +#define GEN12_OAG_OAGLBCTXCTRL _MMIO(0x2b28) +#define GEN12_OAG_OAGLBCTXCTRL_TIMER_PERIOD_SHIFT 2 +#define GEN12_OAG_OAGLBCTXCTRL_TIMER_ENABLE (1 << 1) +#define GEN12_OAG_OAGLBCTXCTRL_COUNTER_RESUME (1 << 0) + +#define GEN12_OAG_OACONTROL _MMIO(0xdaf4) +#define GEN12_OAG_OACONTROL_OA_COUNTER_FORMAT_SHIFT 2 +#define GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE (1 << 0) + +#define GEN12_OAG_OA_DEBUG _MMIO(0xdaf8) +#define GEN12_OAG_OA_DEBUG_INCLUDE_CLK_RATIO (1 << 6) +#define GEN12_OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS (1 << 5) +#define GEN12_OAG_OA_DEBUG_DISABLE_GO_1_0_REPORTS (1 << 2) +#define GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS (1 << 1) + +#define GEN12_OAG_OASTATUS _MMIO(0xdafc) +#define GEN12_OAG_OASTATUS_COUNTER_OVERFLOW (1 << 2) +#define GEN12_OAG_OASTATUS_BUFFER_OVERFLOW (1 << 1) +#define GEN12_OAG_OASTATUS_REPORT_LOST (1 << 0) + +#define GDT_CHICKEN_BITS _MMIO(0x9840) +#define GT_NOA_ENABLE 0x00000080 + +#endif /* __INTEL_PERF_OA_REGS__ */ diff --git a/drivers/gpu/drm/i915/i915_perf_types.h b/drivers/gpu/drm/i915/i915_perf_types.h index aa14354a5120..473a3c0544bb 100644 --- a/drivers/gpu/drm/i915/i915_perf_types.h +++ b/drivers/gpu/drm/i915/i915_perf_types.h @@ -18,7 +18,7 @@ #include <uapi/drm/i915_drm.h> #include "gt/intel_sseu.h" -#include "i915_reg.h" +#include "i915_reg_defs.h" #include "intel_wakeref.h" struct drm_i915_private; diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index ea655161793e..cfc21042499d 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -8,8 +8,10 @@ #include "gt/intel_engine.h" #include "gt/intel_engine_pm.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_engine_user.h" #include "gt/intel_gt_pm.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_rc6.h" #include "gt/intel_rps.h" diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 902e4c802a12..3c87d77d2cf6 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -25,8 +25,7 @@ #ifndef _I915_REG_H_ #define _I915_REG_H_ -#include <linux/bitfield.h> -#include <linux/bits.h> +#include "i915_reg_defs.h" /** * DOC: The i915 register macro definition style guide @@ -116,95 +115,6 @@ * #define GEN8_BAR _MMIO(0xb888) */ -/** - * REG_BIT() - Prepare a u32 bit value - * @__n: 0-based bit number - * - * Local wrapper for BIT() to force u32, with compile time checks. - * - * @return: Value with bit @__n set. - */ -#define REG_BIT(__n) \ - ((u32)(BIT(__n) + \ - BUILD_BUG_ON_ZERO(__is_constexpr(__n) && \ - ((__n) < 0 || (__n) > 31)))) - -/** - * REG_GENMASK() - Prepare a continuous u32 bitmask - * @__high: 0-based high bit - * @__low: 0-based low bit - * - * Local wrapper for GENMASK() to force u32, with compile time checks. - * - * @return: Continuous bitmask from @__high to @__low, inclusive. - */ -#define REG_GENMASK(__high, __low) \ - ((u32)(GENMASK(__high, __low) + \ - BUILD_BUG_ON_ZERO(__is_constexpr(__high) && \ - __is_constexpr(__low) && \ - ((__low) < 0 || (__high) > 31 || (__low) > (__high))))) - -/* - * Local integer constant expression version of is_power_of_2(). - */ -#define IS_POWER_OF_2(__x) ((__x) && (((__x) & ((__x) - 1)) == 0)) - -/** - * REG_FIELD_PREP() - Prepare a u32 bitfield value - * @__mask: shifted mask defining the field's length and position - * @__val: value to put in the field - * - * Local copy of FIELD_PREP() to generate an integer constant expression, force - * u32 and for consistency with REG_FIELD_GET(), REG_BIT() and REG_GENMASK(). - * - * @return: @__val masked and shifted into the field defined by @__mask. - */ -#define REG_FIELD_PREP(__mask, __val) \ - ((u32)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) + \ - BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) + \ - BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U32_MAX) + \ - BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \ - BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0)))) - -/** - * REG_FIELD_GET() - Extract a u32 bitfield value - * @__mask: shifted mask defining the field's length and position - * @__val: value to extract the bitfield value from - * - * Local wrapper for FIELD_GET() to force u32 and for consistency with - * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK(). - * - * @return: Masked and shifted value of the field defined by @__mask in @__val. - */ -#define REG_FIELD_GET(__mask, __val) ((u32)FIELD_GET(__mask, __val)) - -typedef struct { - u32 reg; -} i915_reg_t; - -#define _MMIO(r) ((const i915_reg_t){ .reg = (r) }) - -#define INVALID_MMIO_REG _MMIO(0) - -static __always_inline u32 i915_mmio_reg_offset(i915_reg_t reg) -{ - return reg.reg; -} - -static inline bool i915_mmio_reg_equal(i915_reg_t a, i915_reg_t b) -{ - return i915_mmio_reg_offset(a) == i915_mmio_reg_offset(b); -} - -static inline bool i915_mmio_reg_valid(i915_reg_t reg) -{ - return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG); -} - -#define VLV_DISPLAY_BASE 0x180000 -#define VLV_MIPI_BASE VLV_DISPLAY_BASE -#define BXT_MIPI_BASE 0x60000 - #define DISPLAY_MMIO_BASE(dev_priv) (INTEL_INFO(dev_priv)->display_mmio_offset) /* @@ -275,247 +185,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define _MASKED_BIT_ENABLE(a) ({ typeof(a) _a = (a); _MASKED_FIELD(_a, _a); }) #define _MASKED_BIT_DISABLE(a) (_MASKED_FIELD((a), 0)) -/* PCI config space */ - -#define MCHBAR_I915 0x44 -#define MCHBAR_I965 0x48 -#define MCHBAR_SIZE (4 * 4096) - -#define DEVEN 0x54 -#define DEVEN_MCHBAR_EN (1 << 28) - -/* BSM in include/drm/i915_drm.h */ - -#define HPLLCC 0xc0 /* 85x only */ -#define GC_CLOCK_CONTROL_MASK (0x7 << 0) -#define GC_CLOCK_133_200 (0 << 0) -#define GC_CLOCK_100_200 (1 << 0) -#define GC_CLOCK_100_133 (2 << 0) -#define GC_CLOCK_133_266 (3 << 0) -#define GC_CLOCK_133_200_2 (4 << 0) -#define GC_CLOCK_133_266_2 (5 << 0) -#define GC_CLOCK_166_266 (6 << 0) -#define GC_CLOCK_166_250 (7 << 0) - -#define I915_GDRST 0xc0 /* PCI config register */ -#define GRDOM_FULL (0 << 2) -#define GRDOM_RENDER (1 << 2) -#define GRDOM_MEDIA (3 << 2) -#define GRDOM_MASK (3 << 2) -#define GRDOM_RESET_STATUS (1 << 1) -#define GRDOM_RESET_ENABLE (1 << 0) - -/* BSpec only has register offset, PCI device and bit found empirically */ -#define I830_CLOCK_GATE 0xc8 /* device 0 */ -#define I830_L2_CACHE_CLOCK_GATE_DISABLE (1 << 2) - -#define GCDGMBUS 0xcc - -#define GCFGC2 0xda -#define GCFGC 0xf0 /* 915+ only */ -#define GC_LOW_FREQUENCY_ENABLE (1 << 7) -#define GC_DISPLAY_CLOCK_190_200_MHZ (0 << 4) -#define GC_DISPLAY_CLOCK_333_320_MHZ (4 << 4) -#define GC_DISPLAY_CLOCK_267_MHZ_PNV (0 << 4) -#define GC_DISPLAY_CLOCK_333_MHZ_PNV (1 << 4) -#define GC_DISPLAY_CLOCK_444_MHZ_PNV (2 << 4) -#define GC_DISPLAY_CLOCK_200_MHZ_PNV (5 << 4) -#define GC_DISPLAY_CLOCK_133_MHZ_PNV (6 << 4) -#define GC_DISPLAY_CLOCK_167_MHZ_PNV (7 << 4) -#define GC_DISPLAY_CLOCK_MASK (7 << 4) -#define GM45_GC_RENDER_CLOCK_MASK (0xf << 0) -#define GM45_GC_RENDER_CLOCK_266_MHZ (8 << 0) -#define GM45_GC_RENDER_CLOCK_320_MHZ (9 << 0) -#define GM45_GC_RENDER_CLOCK_400_MHZ (0xb << 0) -#define GM45_GC_RENDER_CLOCK_533_MHZ (0xc << 0) -#define I965_GC_RENDER_CLOCK_MASK (0xf << 0) -#define I965_GC_RENDER_CLOCK_267_MHZ (2 << 0) -#define I965_GC_RENDER_CLOCK_333_MHZ (3 << 0) -#define I965_GC_RENDER_CLOCK_444_MHZ (4 << 0) -#define I965_GC_RENDER_CLOCK_533_MHZ (5 << 0) -#define I945_GC_RENDER_CLOCK_MASK (7 << 0) -#define I945_GC_RENDER_CLOCK_166_MHZ (0 << 0) -#define I945_GC_RENDER_CLOCK_200_MHZ (1 << 0) -#define I945_GC_RENDER_CLOCK_250_MHZ (3 << 0) -#define I945_GC_RENDER_CLOCK_400_MHZ (5 << 0) -#define I915_GC_RENDER_CLOCK_MASK (7 << 0) -#define I915_GC_RENDER_CLOCK_166_MHZ (0 << 0) -#define I915_GC_RENDER_CLOCK_200_MHZ (1 << 0) -#define I915_GC_RENDER_CLOCK_333_MHZ (4 << 0) - -#define ASLE 0xe4 -#define ASLS 0xfc - -#define SWSCI 0xe8 -#define SWSCI_SCISEL (1 << 15) -#define SWSCI_GSSCIE (1 << 0) - -#define LBPC 0xf4 /* legacy/combination backlight modes, also called LBB */ - - -#define ILK_GDSR _MMIO(MCHBAR_MIRROR_BASE + 0x2ca4) -#define ILK_GRDOM_FULL (0 << 1) -#define ILK_GRDOM_RENDER (1 << 1) -#define ILK_GRDOM_MEDIA (3 << 1) -#define ILK_GRDOM_MASK (3 << 1) -#define ILK_GRDOM_RESET_ENABLE (1 << 0) - -#define GEN6_MBCUNIT_SNPCR _MMIO(0x900c) /* for LLC config */ -#define GEN6_MBC_SNPCR_SHIFT 21 -#define GEN6_MBC_SNPCR_MASK (3 << 21) -#define GEN6_MBC_SNPCR_MAX (0 << 21) -#define GEN6_MBC_SNPCR_MED (1 << 21) -#define GEN6_MBC_SNPCR_LOW (2 << 21) -#define GEN6_MBC_SNPCR_MIN (3 << 21) /* only 1/16th of the cache is shared */ - -#define VLV_G3DCTL _MMIO(0x9024) -#define VLV_GSCKGCTL _MMIO(0x9028) - -#define FBC_LLC_READ_CTRL _MMIO(0x9044) -#define FBC_LLC_FULLY_OPEN REG_BIT(30) - -#define GEN6_MBCTL _MMIO(0x0907c) -#define GEN6_MBCTL_ENABLE_BOOT_FETCH (1 << 4) -#define GEN6_MBCTL_CTX_FETCH_NEEDED (1 << 3) -#define GEN6_MBCTL_BME_UPDATE_ENABLE (1 << 2) -#define GEN6_MBCTL_MAE_UPDATE_ENABLE (1 << 1) -#define GEN6_MBCTL_BOOT_FETCH_MECH (1 << 0) - -#define GEN6_GDRST _MMIO(0x941c) -#define GEN6_GRDOM_FULL (1 << 0) -#define GEN6_GRDOM_RENDER (1 << 1) -#define GEN6_GRDOM_MEDIA (1 << 2) -#define GEN6_GRDOM_BLT (1 << 3) -#define GEN6_GRDOM_VECS (1 << 4) -#define GEN9_GRDOM_GUC (1 << 5) -#define GEN8_GRDOM_MEDIA2 (1 << 7) -/* GEN11 changed all bit defs except for FULL & RENDER */ -#define GEN11_GRDOM_FULL GEN6_GRDOM_FULL -#define GEN11_GRDOM_RENDER GEN6_GRDOM_RENDER -#define GEN11_GRDOM_BLT (1 << 2) -#define GEN11_GRDOM_GUC (1 << 3) -#define GEN11_GRDOM_MEDIA (1 << 5) -#define GEN11_GRDOM_MEDIA2 (1 << 6) -#define GEN11_GRDOM_MEDIA3 (1 << 7) -#define GEN11_GRDOM_MEDIA4 (1 << 8) -#define GEN11_GRDOM_MEDIA5 (1 << 9) -#define GEN11_GRDOM_MEDIA6 (1 << 10) -#define GEN11_GRDOM_MEDIA7 (1 << 11) -#define GEN11_GRDOM_MEDIA8 (1 << 12) -#define GEN11_GRDOM_VECS (1 << 13) -#define GEN11_GRDOM_VECS2 (1 << 14) -#define GEN11_GRDOM_VECS3 (1 << 15) -#define GEN11_GRDOM_VECS4 (1 << 16) -#define GEN11_GRDOM_SFC0 (1 << 17) -#define GEN11_GRDOM_SFC1 (1 << 18) -#define GEN11_GRDOM_SFC2 (1 << 19) -#define GEN11_GRDOM_SFC3 (1 << 20) - -#define GEN11_VCS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << ((instance) >> 1)) -#define GEN11_VECS_SFC_RESET_BIT(instance) (GEN11_GRDOM_SFC0 << (instance)) - -#define GEN11_VCS_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x88C) -#define GEN11_VCS_SFC_FORCED_LOCK_BIT (1 << 0) -#define GEN11_VCS_SFC_LOCK_STATUS(engine) _MMIO((engine)->mmio_base + 0x890) -#define GEN11_VCS_SFC_USAGE_BIT (1 << 0) -#define GEN11_VCS_SFC_LOCK_ACK_BIT (1 << 1) - -#define GEN11_VECS_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x201C) -#define GEN11_VECS_SFC_FORCED_LOCK_BIT (1 << 0) -#define GEN11_VECS_SFC_LOCK_ACK(engine) _MMIO((engine)->mmio_base + 0x2018) -#define GEN11_VECS_SFC_LOCK_ACK_BIT (1 << 0) -#define GEN11_VECS_SFC_USAGE(engine) _MMIO((engine)->mmio_base + 0x2014) -#define GEN11_VECS_SFC_USAGE_BIT (1 << 0) - -#define GEN12_HCP_SFC_FORCED_LOCK(engine) _MMIO((engine)->mmio_base + 0x2910) -#define GEN12_HCP_SFC_FORCED_LOCK_BIT REG_BIT(0) -#define GEN12_HCP_SFC_LOCK_STATUS(engine) _MMIO((engine)->mmio_base + 0x2914) -#define GEN12_HCP_SFC_LOCK_ACK_BIT REG_BIT(1) -#define GEN12_HCP_SFC_USAGE_BIT REG_BIT(0) - -#define GEN12_SFC_DONE(n) _MMIO(0x1cc000 + (n) * 0x1000) -#define GEN12_SFC_DONE_MAX 4 - -#define RING_PP_DIR_BASE(base) _MMIO((base) + 0x228) -#define RING_PP_DIR_BASE_READ(base) _MMIO((base) + 0x518) -#define RING_PP_DIR_DCLV(base) _MMIO((base) + 0x220) -#define PP_DIR_DCLV_2G 0xffffffff - -#define GEN8_RING_PDP_UDW(base, n) _MMIO((base) + 0x270 + (n) * 8 + 4) -#define GEN8_RING_PDP_LDW(base, n) _MMIO((base) + 0x270 + (n) * 8) - -#define GEN8_R_PWR_CLK_STATE _MMIO(0x20C8) -#define GEN8_RPCS_ENABLE (1 << 31) -#define GEN8_RPCS_S_CNT_ENABLE (1 << 18) -#define GEN8_RPCS_S_CNT_SHIFT 15 -#define GEN8_RPCS_S_CNT_MASK (0x7 << GEN8_RPCS_S_CNT_SHIFT) -#define GEN11_RPCS_S_CNT_SHIFT 12 -#define GEN11_RPCS_S_CNT_MASK (0x3f << GEN11_RPCS_S_CNT_SHIFT) -#define GEN8_RPCS_SS_CNT_ENABLE (1 << 11) -#define GEN8_RPCS_SS_CNT_SHIFT 8 -#define GEN8_RPCS_SS_CNT_MASK (0x7 << GEN8_RPCS_SS_CNT_SHIFT) -#define GEN8_RPCS_EU_MAX_SHIFT 4 -#define GEN8_RPCS_EU_MAX_MASK (0xf << GEN8_RPCS_EU_MAX_SHIFT) -#define GEN8_RPCS_EU_MIN_SHIFT 0 -#define GEN8_RPCS_EU_MIN_MASK (0xf << GEN8_RPCS_EU_MIN_SHIFT) - -#define WAIT_FOR_RC6_EXIT _MMIO(0x20CC) -/* HSW only */ -#define HSW_SELECTIVE_READ_ADDRESSING_SHIFT 2 -#define HSW_SELECTIVE_READ_ADDRESSING_MASK (0x3 << HSW_SLECTIVE_READ_ADDRESSING_SHIFT) -#define HSW_SELECTIVE_WRITE_ADDRESS_SHIFT 4 -#define HSW_SELECTIVE_WRITE_ADDRESS_MASK (0x7 << HSW_SELECTIVE_WRITE_ADDRESS_SHIFT) -/* HSW+ */ -#define HSW_WAIT_FOR_RC6_EXIT_ENABLE (1 << 0) -#define HSW_RCS_CONTEXT_ENABLE (1 << 7) -#define HSW_RCS_INHIBIT (1 << 8) -/* Gen8 */ -#define GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT 4 -#define GEN8_SELECTIVE_WRITE_ADDRESS_MASK (0x3 << GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT) -#define GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT 4 -#define GEN8_SELECTIVE_WRITE_ADDRESS_MASK (0x3 << GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT) -#define GEN8_SELECTIVE_WRITE_ADDRESSING_ENABLE (1 << 6) -#define GEN8_SELECTIVE_READ_SUBSLICE_SELECT_SHIFT 9 -#define GEN8_SELECTIVE_READ_SUBSLICE_SELECT_MASK (0x3 << GEN8_SELECTIVE_READ_SUBSLICE_SELECT_SHIFT) -#define GEN8_SELECTIVE_READ_SLICE_SELECT_SHIFT 11 -#define GEN8_SELECTIVE_READ_SLICE_SELECT_MASK (0x3 << GEN8_SELECTIVE_READ_SLICE_SELECT_SHIFT) -#define GEN8_SELECTIVE_READ_ADDRESSING_ENABLE (1 << 13) - -#define GAM_ECOCHK _MMIO(0x4090) -#define BDW_DISABLE_HDC_INVALIDATION (1 << 25) -#define ECOCHK_SNB_BIT (1 << 10) -#define ECOCHK_DIS_TLB (1 << 8) -#define HSW_ECOCHK_ARB_PRIO_SOL (1 << 6) -#define ECOCHK_PPGTT_CACHE64B (0x3 << 3) -#define ECOCHK_PPGTT_CACHE4B (0x0 << 3) -#define ECOCHK_PPGTT_GFDT_IVB (0x1 << 4) -#define ECOCHK_PPGTT_LLC_IVB (0x1 << 3) -#define ECOCHK_PPGTT_UC_HSW (0x1 << 3) -#define ECOCHK_PPGTT_WT_HSW (0x2 << 3) -#define ECOCHK_PPGTT_WB_HSW (0x3 << 3) - -#define GEN8_RC6_CTX_INFO _MMIO(0x8504) - -#define GAC_ECO_BITS _MMIO(0x14090) -#define ECOBITS_SNB_BIT (1 << 13) -#define ECOBITS_PPGTT_CACHE64B (3 << 8) -#define ECOBITS_PPGTT_CACHE4B (0 << 8) - -#define GEN12_GAMCNTRL_CTRL _MMIO(0xcf54) -#define INVALIDATION_BROADCAST_MODE_DIS REG_BIT(12) -#define GLOBAL_INVALIDATION_MODE REG_BIT(2) - -#define GEN12_GAMSTLB_CTRL _MMIO(0xcf4c) -#define CONTROL_BLOCK_CLKGATE_DIS REG_BIT(12) -#define EGRESS_BLOCK_CLKGATE_DIS REG_BIT(11) -#define TAG_BLOCK_CLKGATE_DIS REG_BIT(7) - -#define GEN12_MERT_MOD_CTRL _MMIO(0xcf28) -#define FORCE_MISS_FTLB REG_BIT(3) - -#define GAB_CTL _MMIO(0x24000) -#define GAB_CTL_CONT_AFTER_PAGEFAULT (1 << 8) - #define GU_CNTL _MMIO(0x101010) #define LMEM_INIT REG_BIT(7) @@ -538,671 +207,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN6_STOLEN_RESERVED_ENABLE (1 << 0) #define GEN11_STOLEN_RESERVED_ADDR_MASK (0xFFFFFFFFFFFULL << 20) -/* VGA stuff */ - -#define VGA_ST01_MDA 0x3ba -#define VGA_ST01_CGA 0x3da - #define _VGA_MSR_WRITE _MMIO(0x3c2) -#define VGA_MSR_WRITE 0x3c2 -#define VGA_MSR_READ 0x3cc -#define VGA_MSR_MEM_EN (1 << 1) -#define VGA_MSR_CGA_MODE (1 << 0) - -#define VGA_SR_INDEX 0x3c4 -#define SR01 1 -#define VGA_SR_DATA 0x3c5 - -#define VGA_AR_INDEX 0x3c0 -#define VGA_AR_VID_EN (1 << 5) -#define VGA_AR_DATA_WRITE 0x3c0 -#define VGA_AR_DATA_READ 0x3c1 - -#define VGA_GR_INDEX 0x3ce -#define VGA_GR_DATA 0x3cf -/* GR05 */ -#define VGA_GR_MEM_READ_MODE_SHIFT 3 -#define VGA_GR_MEM_READ_MODE_PLANE 1 -/* GR06 */ -#define VGA_GR_MEM_MODE_MASK 0xc -#define VGA_GR_MEM_MODE_SHIFT 2 -#define VGA_GR_MEM_A0000_AFFFF 0 -#define VGA_GR_MEM_A0000_BFFFF 1 -#define VGA_GR_MEM_B0000_B7FFF 2 -#define VGA_GR_MEM_B0000_BFFFF 3 - -#define VGA_DACMASK 0x3c6 -#define VGA_DACRX 0x3c7 -#define VGA_DACWX 0x3c8 -#define VGA_DACDATA 0x3c9 - -#define VGA_CR_INDEX_MDA 0x3b4 -#define VGA_CR_DATA_MDA 0x3b5 -#define VGA_CR_INDEX_CGA 0x3d4 -#define VGA_CR_DATA_CGA 0x3d5 - -#define MI_PREDICATE_SRC0 _MMIO(0x2400) -#define MI_PREDICATE_SRC0_UDW _MMIO(0x2400 + 4) -#define MI_PREDICATE_SRC1 _MMIO(0x2408) -#define MI_PREDICATE_SRC1_UDW _MMIO(0x2408 + 4) -#define MI_PREDICATE_DATA _MMIO(0x2410) -#define MI_PREDICATE_RESULT _MMIO(0x2418) -#define MI_PREDICATE_RESULT_1 _MMIO(0x241c) -#define MI_PREDICATE_RESULT_2 _MMIO(0x2214) -#define LOWER_SLICE_ENABLED (1 << 0) -#define LOWER_SLICE_DISABLED (0 << 0) - -/* - * Registers used only by the command parser - */ -#define BCS_SWCTRL _MMIO(0x22200) -#define BCS_SRC_Y REG_BIT(0) -#define BCS_DST_Y REG_BIT(1) - -/* There are 16 GPR registers */ -#define BCS_GPR(n) _MMIO(0x22600 + (n) * 8) -#define BCS_GPR_UDW(n) _MMIO(0x22600 + (n) * 8 + 4) - -#define GPGPU_THREADS_DISPATCHED _MMIO(0x2290) -#define GPGPU_THREADS_DISPATCHED_UDW _MMIO(0x2290 + 4) -#define HS_INVOCATION_COUNT _MMIO(0x2300) -#define HS_INVOCATION_COUNT_UDW _MMIO(0x2300 + 4) -#define DS_INVOCATION_COUNT _MMIO(0x2308) -#define DS_INVOCATION_COUNT_UDW _MMIO(0x2308 + 4) -#define IA_VERTICES_COUNT _MMIO(0x2310) -#define IA_VERTICES_COUNT_UDW _MMIO(0x2310 + 4) -#define IA_PRIMITIVES_COUNT _MMIO(0x2318) -#define IA_PRIMITIVES_COUNT_UDW _MMIO(0x2318 + 4) -#define VS_INVOCATION_COUNT _MMIO(0x2320) -#define VS_INVOCATION_COUNT_UDW _MMIO(0x2320 + 4) -#define GS_INVOCATION_COUNT _MMIO(0x2328) -#define GS_INVOCATION_COUNT_UDW _MMIO(0x2328 + 4) -#define GS_PRIMITIVES_COUNT _MMIO(0x2330) -#define GS_PRIMITIVES_COUNT_UDW _MMIO(0x2330 + 4) -#define CL_INVOCATION_COUNT _MMIO(0x2338) -#define CL_INVOCATION_COUNT_UDW _MMIO(0x2338 + 4) -#define CL_PRIMITIVES_COUNT _MMIO(0x2340) -#define CL_PRIMITIVES_COUNT_UDW _MMIO(0x2340 + 4) -#define PS_INVOCATION_COUNT _MMIO(0x2348) -#define PS_INVOCATION_COUNT_UDW _MMIO(0x2348 + 4) -#define PS_DEPTH_COUNT _MMIO(0x2350) -#define PS_DEPTH_COUNT_UDW _MMIO(0x2350 + 4) - -/* There are the 4 64-bit counter registers, one for each stream output */ -#define GEN7_SO_NUM_PRIMS_WRITTEN(n) _MMIO(0x5200 + (n) * 8) -#define GEN7_SO_NUM_PRIMS_WRITTEN_UDW(n) _MMIO(0x5200 + (n) * 8 + 4) - -#define GEN7_SO_PRIM_STORAGE_NEEDED(n) _MMIO(0x5240 + (n) * 8) -#define GEN7_SO_PRIM_STORAGE_NEEDED_UDW(n) _MMIO(0x5240 + (n) * 8 + 4) - -#define GEN7_3DPRIM_END_OFFSET _MMIO(0x2420) -#define GEN7_3DPRIM_START_VERTEX _MMIO(0x2430) -#define GEN7_3DPRIM_VERTEX_COUNT _MMIO(0x2434) -#define GEN7_3DPRIM_INSTANCE_COUNT _MMIO(0x2438) -#define GEN7_3DPRIM_START_INSTANCE _MMIO(0x243C) -#define GEN7_3DPRIM_BASE_VERTEX _MMIO(0x2440) - -#define GEN7_GPGPU_DISPATCHDIMX _MMIO(0x2500) -#define GEN7_GPGPU_DISPATCHDIMY _MMIO(0x2504) -#define GEN7_GPGPU_DISPATCHDIMZ _MMIO(0x2508) - -/* There are the 16 64-bit CS General Purpose Registers */ -#define HSW_CS_GPR(n) _MMIO(0x2600 + (n) * 8) -#define HSW_CS_GPR_UDW(n) _MMIO(0x2600 + (n) * 8 + 4) - -#define GEN7_OACONTROL _MMIO(0x2360) -#define GEN7_OACONTROL_CTX_MASK 0xFFFFF000 -#define GEN7_OACONTROL_TIMER_PERIOD_MASK 0x3F -#define GEN7_OACONTROL_TIMER_PERIOD_SHIFT 6 -#define GEN7_OACONTROL_TIMER_ENABLE (1 << 5) -#define GEN7_OACONTROL_FORMAT_A13 (0 << 2) -#define GEN7_OACONTROL_FORMAT_A29 (1 << 2) -#define GEN7_OACONTROL_FORMAT_A13_B8_C8 (2 << 2) -#define GEN7_OACONTROL_FORMAT_A29_B8_C8 (3 << 2) -#define GEN7_OACONTROL_FORMAT_B4_C8 (4 << 2) -#define GEN7_OACONTROL_FORMAT_A45_B8_C8 (5 << 2) -#define GEN7_OACONTROL_FORMAT_B4_C8_A16 (6 << 2) -#define GEN7_OACONTROL_FORMAT_C4_B8 (7 << 2) -#define GEN7_OACONTROL_FORMAT_SHIFT 2 -#define GEN7_OACONTROL_PER_CTX_ENABLE (1 << 1) -#define GEN7_OACONTROL_ENABLE (1 << 0) - -#define GEN8_OACTXID _MMIO(0x2364) - -#define GEN8_OA_DEBUG _MMIO(0x2B04) -#define GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS (1 << 5) -#define GEN9_OA_DEBUG_INCLUDE_CLK_RATIO (1 << 6) -#define GEN9_OA_DEBUG_DISABLE_GO_1_0_REPORTS (1 << 2) -#define GEN9_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS (1 << 1) - -#define GEN8_OACONTROL _MMIO(0x2B00) -#define GEN8_OA_REPORT_FORMAT_A12 (0 << 2) -#define GEN8_OA_REPORT_FORMAT_A12_B8_C8 (2 << 2) -#define GEN8_OA_REPORT_FORMAT_A36_B8_C8 (5 << 2) -#define GEN8_OA_REPORT_FORMAT_C4_B8 (7 << 2) -#define GEN8_OA_REPORT_FORMAT_SHIFT 2 -#define GEN8_OA_SPECIFIC_CONTEXT_ENABLE (1 << 1) -#define GEN8_OA_COUNTER_ENABLE (1 << 0) - -#define GEN8_OACTXCONTROL _MMIO(0x2360) -#define GEN8_OA_TIMER_PERIOD_MASK 0x3F -#define GEN8_OA_TIMER_PERIOD_SHIFT 2 -#define GEN8_OA_TIMER_ENABLE (1 << 1) -#define GEN8_OA_COUNTER_RESUME (1 << 0) - -#define GEN7_OABUFFER _MMIO(0x23B0) /* R/W */ -#define GEN7_OABUFFER_OVERRUN_DISABLE (1 << 3) -#define GEN7_OABUFFER_EDGE_TRIGGER (1 << 2) -#define GEN7_OABUFFER_STOP_RESUME_ENABLE (1 << 1) -#define GEN7_OABUFFER_RESUME (1 << 0) - -#define GEN8_OABUFFER_UDW _MMIO(0x23b4) -#define GEN8_OABUFFER _MMIO(0x2b14) -#define GEN8_OABUFFER_MEM_SELECT_GGTT (1 << 0) /* 0: PPGTT, 1: GGTT */ - -#define GEN7_OASTATUS1 _MMIO(0x2364) -#define GEN7_OASTATUS1_TAIL_MASK 0xffffffc0 -#define GEN7_OASTATUS1_COUNTER_OVERFLOW (1 << 2) -#define GEN7_OASTATUS1_OABUFFER_OVERFLOW (1 << 1) -#define GEN7_OASTATUS1_REPORT_LOST (1 << 0) - -#define GEN7_OASTATUS2 _MMIO(0x2368) -#define GEN7_OASTATUS2_HEAD_MASK 0xffffffc0 -#define GEN7_OASTATUS2_MEM_SELECT_GGTT (1 << 0) /* 0: PPGTT, 1: GGTT */ - -#define GEN8_OASTATUS _MMIO(0x2b08) -#define GEN8_OASTATUS_TAIL_POINTER_WRAP (1 << 17) -#define GEN8_OASTATUS_HEAD_POINTER_WRAP (1 << 16) -#define GEN8_OASTATUS_OVERRUN_STATUS (1 << 3) -#define GEN8_OASTATUS_COUNTER_OVERFLOW (1 << 2) -#define GEN8_OASTATUS_OABUFFER_OVERFLOW (1 << 1) -#define GEN8_OASTATUS_REPORT_LOST (1 << 0) - -#define GEN8_OAHEADPTR _MMIO(0x2B0C) -#define GEN8_OAHEADPTR_MASK 0xffffffc0 -#define GEN8_OATAILPTR _MMIO(0x2B10) -#define GEN8_OATAILPTR_MASK 0xffffffc0 - -#define OABUFFER_SIZE_128K (0 << 3) -#define OABUFFER_SIZE_256K (1 << 3) -#define OABUFFER_SIZE_512K (2 << 3) -#define OABUFFER_SIZE_1M (3 << 3) -#define OABUFFER_SIZE_2M (4 << 3) -#define OABUFFER_SIZE_4M (5 << 3) -#define OABUFFER_SIZE_8M (6 << 3) -#define OABUFFER_SIZE_16M (7 << 3) - -#define GEN12_OA_TLB_INV_CR _MMIO(0xceec) - -#define GEN12_SQCM _MMIO(0x8724) -#define EN_32B_ACCESS REG_BIT(30) - -/* Gen12 OAR unit */ -#define GEN12_OAR_OACONTROL _MMIO(0x2960) -#define GEN12_OAR_OACONTROL_COUNTER_FORMAT_SHIFT 1 -#define GEN12_OAR_OACONTROL_COUNTER_ENABLE (1 << 0) - -#define GEN12_OACTXCONTROL _MMIO(0x2360) -#define GEN12_OAR_OASTATUS _MMIO(0x2968) - -/* Gen12 OAG unit */ -#define GEN12_OAG_OAHEADPTR _MMIO(0xdb00) -#define GEN12_OAG_OAHEADPTR_MASK 0xffffffc0 -#define GEN12_OAG_OATAILPTR _MMIO(0xdb04) -#define GEN12_OAG_OATAILPTR_MASK 0xffffffc0 - -#define GEN12_OAG_OABUFFER _MMIO(0xdb08) -#define GEN12_OAG_OABUFFER_BUFFER_SIZE_MASK (0x7) -#define GEN12_OAG_OABUFFER_BUFFER_SIZE_SHIFT (3) -#define GEN12_OAG_OABUFFER_MEMORY_SELECT (1 << 0) /* 0: PPGTT, 1: GGTT */ - -#define GEN12_OAG_OAGLBCTXCTRL _MMIO(0x2b28) -#define GEN12_OAG_OAGLBCTXCTRL_TIMER_PERIOD_SHIFT 2 -#define GEN12_OAG_OAGLBCTXCTRL_TIMER_ENABLE (1 << 1) -#define GEN12_OAG_OAGLBCTXCTRL_COUNTER_RESUME (1 << 0) - -#define GEN12_OAG_OACONTROL _MMIO(0xdaf4) -#define GEN12_OAG_OACONTROL_OA_COUNTER_FORMAT_SHIFT 2 -#define GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE (1 << 0) - -#define GEN12_OAG_OA_DEBUG _MMIO(0xdaf8) -#define GEN12_OAG_OA_DEBUG_INCLUDE_CLK_RATIO (1 << 6) -#define GEN12_OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS (1 << 5) -#define GEN12_OAG_OA_DEBUG_DISABLE_GO_1_0_REPORTS (1 << 2) -#define GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS (1 << 1) - -#define GEN12_OAG_OASTATUS _MMIO(0xdafc) -#define GEN12_OAG_OASTATUS_COUNTER_OVERFLOW (1 << 2) -#define GEN12_OAG_OASTATUS_BUFFER_OVERFLOW (1 << 1) -#define GEN12_OAG_OASTATUS_REPORT_LOST (1 << 0) - -/* - * Flexible, Aggregate EU Counter Registers. - * Note: these aren't contiguous - */ -#define EU_PERF_CNTL0 _MMIO(0xe458) -#define EU_PERF_CNTL1 _MMIO(0xe558) -#define EU_PERF_CNTL2 _MMIO(0xe658) -#define EU_PERF_CNTL3 _MMIO(0xe758) -#define EU_PERF_CNTL4 _MMIO(0xe45c) -#define EU_PERF_CNTL5 _MMIO(0xe55c) -#define EU_PERF_CNTL6 _MMIO(0xe65c) - -#define RT_CTRL _MMIO(0xe530) -#define DIS_NULL_QUERY REG_BIT(10) - -/* - * OA Boolean state - */ - -#define OASTARTTRIG1 _MMIO(0x2710) -#define OASTARTTRIG1_THRESHOLD_COUNT_MASK_MBZ 0xffff0000 -#define OASTARTTRIG1_THRESHOLD_MASK 0xffff - -#define OASTARTTRIG2 _MMIO(0x2714) -#define OASTARTTRIG2_INVERT_A_0 (1 << 0) -#define OASTARTTRIG2_INVERT_A_1 (1 << 1) -#define OASTARTTRIG2_INVERT_A_2 (1 << 2) -#define OASTARTTRIG2_INVERT_A_3 (1 << 3) -#define OASTARTTRIG2_INVERT_A_4 (1 << 4) -#define OASTARTTRIG2_INVERT_A_5 (1 << 5) -#define OASTARTTRIG2_INVERT_A_6 (1 << 6) -#define OASTARTTRIG2_INVERT_A_7 (1 << 7) -#define OASTARTTRIG2_INVERT_A_8 (1 << 8) -#define OASTARTTRIG2_INVERT_A_9 (1 << 9) -#define OASTARTTRIG2_INVERT_A_10 (1 << 10) -#define OASTARTTRIG2_INVERT_A_11 (1 << 11) -#define OASTARTTRIG2_INVERT_A_12 (1 << 12) -#define OASTARTTRIG2_INVERT_A_13 (1 << 13) -#define OASTARTTRIG2_INVERT_A_14 (1 << 14) -#define OASTARTTRIG2_INVERT_A_15 (1 << 15) -#define OASTARTTRIG2_INVERT_B_0 (1 << 16) -#define OASTARTTRIG2_INVERT_B_1 (1 << 17) -#define OASTARTTRIG2_INVERT_B_2 (1 << 18) -#define OASTARTTRIG2_INVERT_B_3 (1 << 19) -#define OASTARTTRIG2_INVERT_C_0 (1 << 20) -#define OASTARTTRIG2_INVERT_C_1 (1 << 21) -#define OASTARTTRIG2_INVERT_D_0 (1 << 22) -#define OASTARTTRIG2_THRESHOLD_ENABLE (1 << 23) -#define OASTARTTRIG2_START_TRIG_FLAG_MBZ (1 << 24) -#define OASTARTTRIG2_EVENT_SELECT_0 (1 << 28) -#define OASTARTTRIG2_EVENT_SELECT_1 (1 << 29) -#define OASTARTTRIG2_EVENT_SELECT_2 (1 << 30) -#define OASTARTTRIG2_EVENT_SELECT_3 (1 << 31) - -#define OASTARTTRIG3 _MMIO(0x2718) -#define OASTARTTRIG3_NOA_SELECT_MASK 0xf -#define OASTARTTRIG3_NOA_SELECT_8_SHIFT 0 -#define OASTARTTRIG3_NOA_SELECT_9_SHIFT 4 -#define OASTARTTRIG3_NOA_SELECT_10_SHIFT 8 -#define OASTARTTRIG3_NOA_SELECT_11_SHIFT 12 -#define OASTARTTRIG3_NOA_SELECT_12_SHIFT 16 -#define OASTARTTRIG3_NOA_SELECT_13_SHIFT 20 -#define OASTARTTRIG3_NOA_SELECT_14_SHIFT 24 -#define OASTARTTRIG3_NOA_SELECT_15_SHIFT 28 - -#define OASTARTTRIG4 _MMIO(0x271c) -#define OASTARTTRIG4_NOA_SELECT_MASK 0xf -#define OASTARTTRIG4_NOA_SELECT_0_SHIFT 0 -#define OASTARTTRIG4_NOA_SELECT_1_SHIFT 4 -#define OASTARTTRIG4_NOA_SELECT_2_SHIFT 8 -#define OASTARTTRIG4_NOA_SELECT_3_SHIFT 12 -#define OASTARTTRIG4_NOA_SELECT_4_SHIFT 16 -#define OASTARTTRIG4_NOA_SELECT_5_SHIFT 20 -#define OASTARTTRIG4_NOA_SELECT_6_SHIFT 24 -#define OASTARTTRIG4_NOA_SELECT_7_SHIFT 28 - -#define OASTARTTRIG5 _MMIO(0x2720) -#define OASTARTTRIG5_THRESHOLD_COUNT_MASK_MBZ 0xffff0000 -#define OASTARTTRIG5_THRESHOLD_MASK 0xffff - -#define OASTARTTRIG6 _MMIO(0x2724) -#define OASTARTTRIG6_INVERT_A_0 (1 << 0) -#define OASTARTTRIG6_INVERT_A_1 (1 << 1) -#define OASTARTTRIG6_INVERT_A_2 (1 << 2) -#define OASTARTTRIG6_INVERT_A_3 (1 << 3) -#define OASTARTTRIG6_INVERT_A_4 (1 << 4) -#define OASTARTTRIG6_INVERT_A_5 (1 << 5) -#define OASTARTTRIG6_INVERT_A_6 (1 << 6) -#define OASTARTTRIG6_INVERT_A_7 (1 << 7) -#define OASTARTTRIG6_INVERT_A_8 (1 << 8) -#define OASTARTTRIG6_INVERT_A_9 (1 << 9) -#define OASTARTTRIG6_INVERT_A_10 (1 << 10) -#define OASTARTTRIG6_INVERT_A_11 (1 << 11) -#define OASTARTTRIG6_INVERT_A_12 (1 << 12) -#define OASTARTTRIG6_INVERT_A_13 (1 << 13) -#define OASTARTTRIG6_INVERT_A_14 (1 << 14) -#define OASTARTTRIG6_INVERT_A_15 (1 << 15) -#define OASTARTTRIG6_INVERT_B_0 (1 << 16) -#define OASTARTTRIG6_INVERT_B_1 (1 << 17) -#define OASTARTTRIG6_INVERT_B_2 (1 << 18) -#define OASTARTTRIG6_INVERT_B_3 (1 << 19) -#define OASTARTTRIG6_INVERT_C_0 (1 << 20) -#define OASTARTTRIG6_INVERT_C_1 (1 << 21) -#define OASTARTTRIG6_INVERT_D_0 (1 << 22) -#define OASTARTTRIG6_THRESHOLD_ENABLE (1 << 23) -#define OASTARTTRIG6_START_TRIG_FLAG_MBZ (1 << 24) -#define OASTARTTRIG6_EVENT_SELECT_4 (1 << 28) -#define OASTARTTRIG6_EVENT_SELECT_5 (1 << 29) -#define OASTARTTRIG6_EVENT_SELECT_6 (1 << 30) -#define OASTARTTRIG6_EVENT_SELECT_7 (1 << 31) - -#define OASTARTTRIG7 _MMIO(0x2728) -#define OASTARTTRIG7_NOA_SELECT_MASK 0xf -#define OASTARTTRIG7_NOA_SELECT_8_SHIFT 0 -#define OASTARTTRIG7_NOA_SELECT_9_SHIFT 4 -#define OASTARTTRIG7_NOA_SELECT_10_SHIFT 8 -#define OASTARTTRIG7_NOA_SELECT_11_SHIFT 12 -#define OASTARTTRIG7_NOA_SELECT_12_SHIFT 16 -#define OASTARTTRIG7_NOA_SELECT_13_SHIFT 20 -#define OASTARTTRIG7_NOA_SELECT_14_SHIFT 24 -#define OASTARTTRIG7_NOA_SELECT_15_SHIFT 28 - -#define OASTARTTRIG8 _MMIO(0x272c) -#define OASTARTTRIG8_NOA_SELECT_MASK 0xf -#define OASTARTTRIG8_NOA_SELECT_0_SHIFT 0 -#define OASTARTTRIG8_NOA_SELECT_1_SHIFT 4 -#define OASTARTTRIG8_NOA_SELECT_2_SHIFT 8 -#define OASTARTTRIG8_NOA_SELECT_3_SHIFT 12 -#define OASTARTTRIG8_NOA_SELECT_4_SHIFT 16 -#define OASTARTTRIG8_NOA_SELECT_5_SHIFT 20 -#define OASTARTTRIG8_NOA_SELECT_6_SHIFT 24 -#define OASTARTTRIG8_NOA_SELECT_7_SHIFT 28 - -#define OAREPORTTRIG1 _MMIO(0x2740) -#define OAREPORTTRIG1_THRESHOLD_MASK 0xffff -#define OAREPORTTRIG1_EDGE_LEVEL_TRIGGER_SELECT_MASK 0xffff0000 /* 0=level */ - -#define OAREPORTTRIG2 _MMIO(0x2744) -#define OAREPORTTRIG2_INVERT_A_0 (1 << 0) -#define OAREPORTTRIG2_INVERT_A_1 (1 << 1) -#define OAREPORTTRIG2_INVERT_A_2 (1 << 2) -#define OAREPORTTRIG2_INVERT_A_3 (1 << 3) -#define OAREPORTTRIG2_INVERT_A_4 (1 << 4) -#define OAREPORTTRIG2_INVERT_A_5 (1 << 5) -#define OAREPORTTRIG2_INVERT_A_6 (1 << 6) -#define OAREPORTTRIG2_INVERT_A_7 (1 << 7) -#define OAREPORTTRIG2_INVERT_A_8 (1 << 8) -#define OAREPORTTRIG2_INVERT_A_9 (1 << 9) -#define OAREPORTTRIG2_INVERT_A_10 (1 << 10) -#define OAREPORTTRIG2_INVERT_A_11 (1 << 11) -#define OAREPORTTRIG2_INVERT_A_12 (1 << 12) -#define OAREPORTTRIG2_INVERT_A_13 (1 << 13) -#define OAREPORTTRIG2_INVERT_A_14 (1 << 14) -#define OAREPORTTRIG2_INVERT_A_15 (1 << 15) -#define OAREPORTTRIG2_INVERT_B_0 (1 << 16) -#define OAREPORTTRIG2_INVERT_B_1 (1 << 17) -#define OAREPORTTRIG2_INVERT_B_2 (1 << 18) -#define OAREPORTTRIG2_INVERT_B_3 (1 << 19) -#define OAREPORTTRIG2_INVERT_C_0 (1 << 20) -#define OAREPORTTRIG2_INVERT_C_1 (1 << 21) -#define OAREPORTTRIG2_INVERT_D_0 (1 << 22) -#define OAREPORTTRIG2_THRESHOLD_ENABLE (1 << 23) -#define OAREPORTTRIG2_REPORT_TRIGGER_ENABLE (1 << 31) - -#define OAREPORTTRIG3 _MMIO(0x2748) -#define OAREPORTTRIG3_NOA_SELECT_MASK 0xf -#define OAREPORTTRIG3_NOA_SELECT_8_SHIFT 0 -#define OAREPORTTRIG3_NOA_SELECT_9_SHIFT 4 -#define OAREPORTTRIG3_NOA_SELECT_10_SHIFT 8 -#define OAREPORTTRIG3_NOA_SELECT_11_SHIFT 12 -#define OAREPORTTRIG3_NOA_SELECT_12_SHIFT 16 -#define OAREPORTTRIG3_NOA_SELECT_13_SHIFT 20 -#define OAREPORTTRIG3_NOA_SELECT_14_SHIFT 24 -#define OAREPORTTRIG3_NOA_SELECT_15_SHIFT 28 - -#define OAREPORTTRIG4 _MMIO(0x274c) -#define OAREPORTTRIG4_NOA_SELECT_MASK 0xf -#define OAREPORTTRIG4_NOA_SELECT_0_SHIFT 0 -#define OAREPORTTRIG4_NOA_SELECT_1_SHIFT 4 -#define OAREPORTTRIG4_NOA_SELECT_2_SHIFT 8 -#define OAREPORTTRIG4_NOA_SELECT_3_SHIFT 12 -#define OAREPORTTRIG4_NOA_SELECT_4_SHIFT 16 -#define OAREPORTTRIG4_NOA_SELECT_5_SHIFT 20 -#define OAREPORTTRIG4_NOA_SELECT_6_SHIFT 24 -#define OAREPORTTRIG4_NOA_SELECT_7_SHIFT 28 - -#define OAREPORTTRIG5 _MMIO(0x2750) -#define OAREPORTTRIG5_THRESHOLD_MASK 0xffff -#define OAREPORTTRIG5_EDGE_LEVEL_TRIGGER_SELECT_MASK 0xffff0000 /* 0=level */ - -#define OAREPORTTRIG6 _MMIO(0x2754) -#define OAREPORTTRIG6_INVERT_A_0 (1 << 0) -#define OAREPORTTRIG6_INVERT_A_1 (1 << 1) -#define OAREPORTTRIG6_INVERT_A_2 (1 << 2) -#define OAREPORTTRIG6_INVERT_A_3 (1 << 3) -#define OAREPORTTRIG6_INVERT_A_4 (1 << 4) -#define OAREPORTTRIG6_INVERT_A_5 (1 << 5) -#define OAREPORTTRIG6_INVERT_A_6 (1 << 6) -#define OAREPORTTRIG6_INVERT_A_7 (1 << 7) -#define OAREPORTTRIG6_INVERT_A_8 (1 << 8) -#define OAREPORTTRIG6_INVERT_A_9 (1 << 9) -#define OAREPORTTRIG6_INVERT_A_10 (1 << 10) -#define OAREPORTTRIG6_INVERT_A_11 (1 << 11) -#define OAREPORTTRIG6_INVERT_A_12 (1 << 12) -#define OAREPORTTRIG6_INVERT_A_13 (1 << 13) -#define OAREPORTTRIG6_INVERT_A_14 (1 << 14) -#define OAREPORTTRIG6_INVERT_A_15 (1 << 15) -#define OAREPORTTRIG6_INVERT_B_0 (1 << 16) -#define OAREPORTTRIG6_INVERT_B_1 (1 << 17) -#define OAREPORTTRIG6_INVERT_B_2 (1 << 18) -#define OAREPORTTRIG6_INVERT_B_3 (1 << 19) -#define OAREPORTTRIG6_INVERT_C_0 (1 << 20) -#define OAREPORTTRIG6_INVERT_C_1 (1 << 21) -#define OAREPORTTRIG6_INVERT_D_0 (1 << 22) -#define OAREPORTTRIG6_THRESHOLD_ENABLE (1 << 23) -#define OAREPORTTRIG6_REPORT_TRIGGER_ENABLE (1 << 31) - -#define OAREPORTTRIG7 _MMIO(0x2758) -#define OAREPORTTRIG7_NOA_SELECT_MASK 0xf -#define OAREPORTTRIG7_NOA_SELECT_8_SHIFT 0 -#define OAREPORTTRIG7_NOA_SELECT_9_SHIFT 4 -#define OAREPORTTRIG7_NOA_SELECT_10_SHIFT 8 -#define OAREPORTTRIG7_NOA_SELECT_11_SHIFT 12 -#define OAREPORTTRIG7_NOA_SELECT_12_SHIFT 16 -#define OAREPORTTRIG7_NOA_SELECT_13_SHIFT 20 -#define OAREPORTTRIG7_NOA_SELECT_14_SHIFT 24 -#define OAREPORTTRIG7_NOA_SELECT_15_SHIFT 28 - -#define OAREPORTTRIG8 _MMIO(0x275c) -#define OAREPORTTRIG8_NOA_SELECT_MASK 0xf -#define OAREPORTTRIG8_NOA_SELECT_0_SHIFT 0 -#define OAREPORTTRIG8_NOA_SELECT_1_SHIFT 4 -#define OAREPORTTRIG8_NOA_SELECT_2_SHIFT 8 -#define OAREPORTTRIG8_NOA_SELECT_3_SHIFT 12 -#define OAREPORTTRIG8_NOA_SELECT_4_SHIFT 16 -#define OAREPORTTRIG8_NOA_SELECT_5_SHIFT 20 -#define OAREPORTTRIG8_NOA_SELECT_6_SHIFT 24 -#define OAREPORTTRIG8_NOA_SELECT_7_SHIFT 28 - -/* Same layout as OASTARTTRIGX */ -#define GEN12_OAG_OASTARTTRIG1 _MMIO(0xd900) -#define GEN12_OAG_OASTARTTRIG2 _MMIO(0xd904) -#define GEN12_OAG_OASTARTTRIG3 _MMIO(0xd908) -#define GEN12_OAG_OASTARTTRIG4 _MMIO(0xd90c) -#define GEN12_OAG_OASTARTTRIG5 _MMIO(0xd910) -#define GEN12_OAG_OASTARTTRIG6 _MMIO(0xd914) -#define GEN12_OAG_OASTARTTRIG7 _MMIO(0xd918) -#define GEN12_OAG_OASTARTTRIG8 _MMIO(0xd91c) - -/* Same layout as OAREPORTTRIGX */ -#define GEN12_OAG_OAREPORTTRIG1 _MMIO(0xd920) -#define GEN12_OAG_OAREPORTTRIG2 _MMIO(0xd924) -#define GEN12_OAG_OAREPORTTRIG3 _MMIO(0xd928) -#define GEN12_OAG_OAREPORTTRIG4 _MMIO(0xd92c) -#define GEN12_OAG_OAREPORTTRIG5 _MMIO(0xd930) -#define GEN12_OAG_OAREPORTTRIG6 _MMIO(0xd934) -#define GEN12_OAG_OAREPORTTRIG7 _MMIO(0xd938) -#define GEN12_OAG_OAREPORTTRIG8 _MMIO(0xd93c) - -/* CECX_0 */ -#define OACEC_COMPARE_LESS_OR_EQUAL 6 -#define OACEC_COMPARE_NOT_EQUAL 5 -#define OACEC_COMPARE_LESS_THAN 4 -#define OACEC_COMPARE_GREATER_OR_EQUAL 3 -#define OACEC_COMPARE_EQUAL 2 -#define OACEC_COMPARE_GREATER_THAN 1 -#define OACEC_COMPARE_ANY_EQUAL 0 - -#define OACEC_COMPARE_VALUE_MASK 0xffff -#define OACEC_COMPARE_VALUE_SHIFT 3 - -#define OACEC_SELECT_NOA (0 << 19) -#define OACEC_SELECT_PREV (1 << 19) -#define OACEC_SELECT_BOOLEAN (2 << 19) - -/* 11-bit array 0: pass-through, 1: negated */ -#define GEN12_OASCEC_NEGATE_MASK 0x7ff -#define GEN12_OASCEC_NEGATE_SHIFT 21 - -/* CECX_1 */ -#define OACEC_MASK_MASK 0xffff -#define OACEC_CONSIDERATIONS_MASK 0xffff -#define OACEC_CONSIDERATIONS_SHIFT 16 - -#define OACEC0_0 _MMIO(0x2770) -#define OACEC0_1 _MMIO(0x2774) -#define OACEC1_0 _MMIO(0x2778) -#define OACEC1_1 _MMIO(0x277c) -#define OACEC2_0 _MMIO(0x2780) -#define OACEC2_1 _MMIO(0x2784) -#define OACEC3_0 _MMIO(0x2788) -#define OACEC3_1 _MMIO(0x278c) -#define OACEC4_0 _MMIO(0x2790) -#define OACEC4_1 _MMIO(0x2794) -#define OACEC5_0 _MMIO(0x2798) -#define OACEC5_1 _MMIO(0x279c) -#define OACEC6_0 _MMIO(0x27a0) -#define OACEC6_1 _MMIO(0x27a4) -#define OACEC7_0 _MMIO(0x27a8) -#define OACEC7_1 _MMIO(0x27ac) - -/* Same layout as CECX_Y */ -#define GEN12_OAG_CEC0_0 _MMIO(0xd940) -#define GEN12_OAG_CEC0_1 _MMIO(0xd944) -#define GEN12_OAG_CEC1_0 _MMIO(0xd948) -#define GEN12_OAG_CEC1_1 _MMIO(0xd94c) -#define GEN12_OAG_CEC2_0 _MMIO(0xd950) -#define GEN12_OAG_CEC2_1 _MMIO(0xd954) -#define GEN12_OAG_CEC3_0 _MMIO(0xd958) -#define GEN12_OAG_CEC3_1 _MMIO(0xd95c) -#define GEN12_OAG_CEC4_0 _MMIO(0xd960) -#define GEN12_OAG_CEC4_1 _MMIO(0xd964) -#define GEN12_OAG_CEC5_0 _MMIO(0xd968) -#define GEN12_OAG_CEC5_1 _MMIO(0xd96c) -#define GEN12_OAG_CEC6_0 _MMIO(0xd970) -#define GEN12_OAG_CEC6_1 _MMIO(0xd974) -#define GEN12_OAG_CEC7_0 _MMIO(0xd978) -#define GEN12_OAG_CEC7_1 _MMIO(0xd97c) - -/* Same layout as CECX_Y + negate 11-bit array */ -#define GEN12_OAG_SCEC0_0 _MMIO(0xdc00) -#define GEN12_OAG_SCEC0_1 _MMIO(0xdc04) -#define GEN12_OAG_SCEC1_0 _MMIO(0xdc08) -#define GEN12_OAG_SCEC1_1 _MMIO(0xdc0c) -#define GEN12_OAG_SCEC2_0 _MMIO(0xdc10) -#define GEN12_OAG_SCEC2_1 _MMIO(0xdc14) -#define GEN12_OAG_SCEC3_0 _MMIO(0xdc18) -#define GEN12_OAG_SCEC3_1 _MMIO(0xdc1c) -#define GEN12_OAG_SCEC4_0 _MMIO(0xdc20) -#define GEN12_OAG_SCEC4_1 _MMIO(0xdc24) -#define GEN12_OAG_SCEC5_0 _MMIO(0xdc28) -#define GEN12_OAG_SCEC5_1 _MMIO(0xdc2c) -#define GEN12_OAG_SCEC6_0 _MMIO(0xdc30) -#define GEN12_OAG_SCEC6_1 _MMIO(0xdc34) -#define GEN12_OAG_SCEC7_0 _MMIO(0xdc38) -#define GEN12_OAG_SCEC7_1 _MMIO(0xdc3c) - -/* OA perf counters */ -#define OA_PERFCNT1_LO _MMIO(0x91B8) -#define OA_PERFCNT1_HI _MMIO(0x91BC) -#define OA_PERFCNT2_LO _MMIO(0x91C0) -#define OA_PERFCNT2_HI _MMIO(0x91C4) -#define OA_PERFCNT3_LO _MMIO(0x91C8) -#define OA_PERFCNT3_HI _MMIO(0x91CC) -#define OA_PERFCNT4_LO _MMIO(0x91D8) -#define OA_PERFCNT4_HI _MMIO(0x91DC) - -#define OA_PERFMATRIX_LO _MMIO(0x91C8) -#define OA_PERFMATRIX_HI _MMIO(0x91CC) - -/* RPM unit config (Gen8+) */ -#define RPM_CONFIG0 _MMIO(0x0D00) -#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT 3 -#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK (1 << GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT) -#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ 0 -#define GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ 1 -#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT 3 -#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK (0x7 << GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT) -#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ 0 -#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ 1 -#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ 2 -#define GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ 3 -#define GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT 1 -#define GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK (0x3 << GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT) - -#define RPM_CONFIG1 _MMIO(0x0D04) -#define GEN10_GT_NOA_ENABLE (1 << 9) - -/* GPM unit config (Gen9+) */ -#define CTC_MODE _MMIO(0xA26C) -#define CTC_SOURCE_PARAMETER_MASK 1 -#define CTC_SOURCE_CRYSTAL_CLOCK 0 -#define CTC_SOURCE_DIVIDE_LOGIC 1 -#define CTC_SHIFT_PARAMETER_SHIFT 1 -#define CTC_SHIFT_PARAMETER_MASK (0x3 << CTC_SHIFT_PARAMETER_SHIFT) - -/* RCP unit config (Gen8+) */ -#define RCP_CONFIG _MMIO(0x0D08) - -/* NOA (HSW) */ -#define HSW_MBVID2_NOA0 _MMIO(0x9E80) -#define HSW_MBVID2_NOA1 _MMIO(0x9E84) -#define HSW_MBVID2_NOA2 _MMIO(0x9E88) -#define HSW_MBVID2_NOA3 _MMIO(0x9E8C) -#define HSW_MBVID2_NOA4 _MMIO(0x9E90) -#define HSW_MBVID2_NOA5 _MMIO(0x9E94) -#define HSW_MBVID2_NOA6 _MMIO(0x9E98) -#define HSW_MBVID2_NOA7 _MMIO(0x9E9C) -#define HSW_MBVID2_NOA8 _MMIO(0x9EA0) -#define HSW_MBVID2_NOA9 _MMIO(0x9EA4) - -#define HSW_MBVID2_MISR0 _MMIO(0x9EC0) - -/* NOA (Gen8+) */ -#define NOA_CONFIG(i) _MMIO(0x0D0C + (i) * 4) - -#define MICRO_BP0_0 _MMIO(0x9800) -#define MICRO_BP0_2 _MMIO(0x9804) -#define MICRO_BP0_1 _MMIO(0x9808) - -#define MICRO_BP1_0 _MMIO(0x980C) -#define MICRO_BP1_2 _MMIO(0x9810) -#define MICRO_BP1_1 _MMIO(0x9814) - -#define MICRO_BP2_0 _MMIO(0x9818) -#define MICRO_BP2_2 _MMIO(0x981C) -#define MICRO_BP2_1 _MMIO(0x9820) - -#define MICRO_BP3_0 _MMIO(0x9824) -#define MICRO_BP3_2 _MMIO(0x9828) -#define MICRO_BP3_1 _MMIO(0x982C) - -#define MICRO_BP_TRIGGER _MMIO(0x9830) -#define MICRO_BP3_COUNT_STATUS01 _MMIO(0x9834) -#define MICRO_BP3_COUNT_STATUS23 _MMIO(0x9838) -#define MICRO_BP_FIRED_ARMED _MMIO(0x983C) - -#define GEN12_OAA_DBG_REG _MMIO(0xdc44) -#define GEN12_OAG_OA_PESS _MMIO(0x2b2c) -#define GEN12_OAG_SPCTR_CNF _MMIO(0xdc40) - -#define GDT_CHICKEN_BITS _MMIO(0x9840) -#define GT_NOA_ENABLE 0x00000080 - -#define NOA_DATA _MMIO(0x986C) -#define NOA_WRITE _MMIO(0x9888) -#define GEN10_NOA_WRITE_HIGH _MMIO(0x9884) #define _GEN7_PIPEA_DE_LOAD_SL 0x70068 #define _GEN7_PIPEB_DE_LOAD_SL 0x71068 @@ -1244,177 +249,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define VLV_IOSF_DATA _MMIO(VLV_DISPLAY_BASE + 0x2104) #define VLV_IOSF_ADDR _MMIO(VLV_DISPLAY_BASE + 0x2108) -/* See configdb bunit SB addr map */ -#define BUNIT_REG_BISOC 0x11 - -/* PUNIT_REG_*SSPM0 */ -#define _SSPM0_SSC(val) ((val) << 0) -#define SSPM0_SSC_MASK _SSPM0_SSC(0x3) -#define SSPM0_SSC_PWR_ON _SSPM0_SSC(0x0) -#define SSPM0_SSC_CLK_GATE _SSPM0_SSC(0x1) -#define SSPM0_SSC_RESET _SSPM0_SSC(0x2) -#define SSPM0_SSC_PWR_GATE _SSPM0_SSC(0x3) -#define _SSPM0_SSS(val) ((val) << 24) -#define SSPM0_SSS_MASK _SSPM0_SSS(0x3) -#define SSPM0_SSS_PWR_ON _SSPM0_SSS(0x0) -#define SSPM0_SSS_CLK_GATE _SSPM0_SSS(0x1) -#define SSPM0_SSS_RESET _SSPM0_SSS(0x2) -#define SSPM0_SSS_PWR_GATE _SSPM0_SSS(0x3) - -/* PUNIT_REG_*SSPM1 */ -#define SSPM1_FREQSTAT_SHIFT 24 -#define SSPM1_FREQSTAT_MASK (0x1f << SSPM1_FREQSTAT_SHIFT) -#define SSPM1_FREQGUAR_SHIFT 8 -#define SSPM1_FREQGUAR_MASK (0x1f << SSPM1_FREQGUAR_SHIFT) -#define SSPM1_FREQ_SHIFT 0 -#define SSPM1_FREQ_MASK (0x1f << SSPM1_FREQ_SHIFT) - -#define PUNIT_REG_VEDSSPM0 0x32 -#define PUNIT_REG_VEDSSPM1 0x33 - -#define PUNIT_REG_DSPSSPM 0x36 -#define DSPFREQSTAT_SHIFT_CHV 24 -#define DSPFREQSTAT_MASK_CHV (0x1f << DSPFREQSTAT_SHIFT_CHV) -#define DSPFREQGUAR_SHIFT_CHV 8 -#define DSPFREQGUAR_MASK_CHV (0x1f << DSPFREQGUAR_SHIFT_CHV) -#define DSPFREQSTAT_SHIFT 30 -#define DSPFREQSTAT_MASK (0x3 << DSPFREQSTAT_SHIFT) -#define DSPFREQGUAR_SHIFT 14 -#define DSPFREQGUAR_MASK (0x3 << DSPFREQGUAR_SHIFT) -#define DSP_MAXFIFO_PM5_STATUS (1 << 22) /* chv */ -#define DSP_AUTO_CDCLK_GATE_DISABLE (1 << 7) /* chv */ -#define DSP_MAXFIFO_PM5_ENABLE (1 << 6) /* chv */ -#define _DP_SSC(val, pipe) ((val) << (2 * (pipe))) -#define DP_SSC_MASK(pipe) _DP_SSC(0x3, (pipe)) -#define DP_SSC_PWR_ON(pipe) _DP_SSC(0x0, (pipe)) -#define DP_SSC_CLK_GATE(pipe) _DP_SSC(0x1, (pipe)) -#define DP_SSC_RESET(pipe) _DP_SSC(0x2, (pipe)) -#define DP_SSC_PWR_GATE(pipe) _DP_SSC(0x3, (pipe)) -#define _DP_SSS(val, pipe) ((val) << (2 * (pipe) + 16)) -#define DP_SSS_MASK(pipe) _DP_SSS(0x3, (pipe)) -#define DP_SSS_PWR_ON(pipe) _DP_SSS(0x0, (pipe)) -#define DP_SSS_CLK_GATE(pipe) _DP_SSS(0x1, (pipe)) -#define DP_SSS_RESET(pipe) _DP_SSS(0x2, (pipe)) -#define DP_SSS_PWR_GATE(pipe) _DP_SSS(0x3, (pipe)) - -#define PUNIT_REG_ISPSSPM0 0x39 -#define PUNIT_REG_ISPSSPM1 0x3a - -#define PUNIT_REG_PWRGT_CTRL 0x60 -#define PUNIT_REG_PWRGT_STATUS 0x61 -#define PUNIT_PWRGT_MASK(pw_idx) (3 << ((pw_idx) * 2)) -#define PUNIT_PWRGT_PWR_ON(pw_idx) (0 << ((pw_idx) * 2)) -#define PUNIT_PWRGT_CLK_GATE(pw_idx) (1 << ((pw_idx) * 2)) -#define PUNIT_PWRGT_RESET(pw_idx) (2 << ((pw_idx) * 2)) -#define PUNIT_PWRGT_PWR_GATE(pw_idx) (3 << ((pw_idx) * 2)) - -#define PUNIT_PWGT_IDX_RENDER 0 -#define PUNIT_PWGT_IDX_MEDIA 1 -#define PUNIT_PWGT_IDX_DISP2D 3 -#define PUNIT_PWGT_IDX_DPIO_CMN_BC 5 -#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01 6 -#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23 7 -#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01 8 -#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23 9 -#define PUNIT_PWGT_IDX_DPIO_RX0 10 -#define PUNIT_PWGT_IDX_DPIO_RX1 11 -#define PUNIT_PWGT_IDX_DPIO_CMN_D 12 - -#define PUNIT_REG_GPU_LFM 0xd3 -#define PUNIT_REG_GPU_FREQ_REQ 0xd4 -#define PUNIT_REG_GPU_FREQ_STS 0xd8 -#define GPLLENABLE (1 << 4) -#define GENFREQSTATUS (1 << 0) -#define PUNIT_REG_MEDIA_TURBO_FREQ_REQ 0xdc -#define PUNIT_REG_CZ_TIMESTAMP 0xce - -#define PUNIT_FUSE_BUS2 0xf6 /* bits 47:40 */ -#define PUNIT_FUSE_BUS1 0xf5 /* bits 55:48 */ - -#define FB_GFX_FMAX_AT_VMAX_FUSE 0x136 -#define FB_GFX_FREQ_FUSE_MASK 0xff -#define FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT 24 -#define FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT 16 -#define FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT 8 - -#define FB_GFX_FMIN_AT_VMIN_FUSE 0x137 -#define FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT 8 - -#define PUNIT_REG_DDR_SETUP2 0x139 -#define FORCE_DDR_FREQ_REQ_ACK (1 << 8) -#define FORCE_DDR_LOW_FREQ (1 << 1) -#define FORCE_DDR_HIGH_FREQ (1 << 0) - -#define PUNIT_GPU_STATUS_REG 0xdb -#define PUNIT_GPU_STATUS_MAX_FREQ_SHIFT 16 -#define PUNIT_GPU_STATUS_MAX_FREQ_MASK 0xff -#define PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT 8 -#define PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK 0xff - -#define PUNIT_GPU_DUTYCYCLE_REG 0xdf -#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT 8 -#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK 0xff - -#define IOSF_NC_FB_GFX_FREQ_FUSE 0x1c -#define FB_GFX_MAX_FREQ_FUSE_SHIFT 3 -#define FB_GFX_MAX_FREQ_FUSE_MASK 0x000007f8 -#define FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT 11 -#define FB_GFX_FGUARANTEED_FREQ_FUSE_MASK 0x0007f800 -#define IOSF_NC_FB_GFX_FMAX_FUSE_HI 0x34 -#define FB_FMAX_VMIN_FREQ_HI_MASK 0x00000007 -#define IOSF_NC_FB_GFX_FMAX_FUSE_LO 0x30 -#define FB_FMAX_VMIN_FREQ_LO_SHIFT 27 -#define FB_FMAX_VMIN_FREQ_LO_MASK 0xf8000000 - -#define VLV_TURBO_SOC_OVERRIDE 0x04 -#define VLV_OVERRIDE_EN 1 -#define VLV_SOC_TDP_EN (1 << 1) -#define VLV_BIAS_CPU_125_SOC_875 (6 << 2) -#define CHV_BIAS_CPU_50_SOC_50 (3 << 2) - -/* vlv2 north clock has */ -#define CCK_FUSE_REG 0x8 -#define CCK_FUSE_HPLL_FREQ_MASK 0x3 -#define CCK_REG_DSI_PLL_FUSE 0x44 -#define CCK_REG_DSI_PLL_CONTROL 0x48 -#define DSI_PLL_VCO_EN (1 << 31) -#define DSI_PLL_LDO_GATE (1 << 30) -#define DSI_PLL_P1_POST_DIV_SHIFT 17 -#define DSI_PLL_P1_POST_DIV_MASK (0x1ff << 17) -#define DSI_PLL_P2_MUX_DSI0_DIV2 (1 << 13) -#define DSI_PLL_P3_MUX_DSI1_DIV2 (1 << 12) -#define DSI_PLL_MUX_MASK (3 << 9) -#define DSI_PLL_MUX_DSI0_DSIPLL (0 << 10) -#define DSI_PLL_MUX_DSI0_CCK (1 << 10) -#define DSI_PLL_MUX_DSI1_DSIPLL (0 << 9) -#define DSI_PLL_MUX_DSI1_CCK (1 << 9) -#define DSI_PLL_CLK_GATE_MASK (0xf << 5) -#define DSI_PLL_CLK_GATE_DSI0_DSIPLL (1 << 8) -#define DSI_PLL_CLK_GATE_DSI1_DSIPLL (1 << 7) -#define DSI_PLL_CLK_GATE_DSI0_CCK (1 << 6) -#define DSI_PLL_CLK_GATE_DSI1_CCK (1 << 5) -#define DSI_PLL_LOCK (1 << 0) -#define CCK_REG_DSI_PLL_DIVIDER 0x4c -#define DSI_PLL_LFSR (1 << 31) -#define DSI_PLL_FRACTION_EN (1 << 30) -#define DSI_PLL_FRAC_COUNTER_SHIFT 27 -#define DSI_PLL_FRAC_COUNTER_MASK (7 << 27) -#define DSI_PLL_USYNC_CNT_SHIFT 18 -#define DSI_PLL_USYNC_CNT_MASK (0x1ff << 18) -#define DSI_PLL_N1_DIV_SHIFT 16 -#define DSI_PLL_N1_DIV_MASK (3 << 16) -#define DSI_PLL_M1_DIV_SHIFT 0 -#define DSI_PLL_M1_DIV_MASK (0x1ff << 0) -#define CCK_CZ_CLOCK_CONTROL 0x62 -#define CCK_GPLL_CLOCK_CONTROL 0x67 -#define CCK_DISPLAY_CLOCK_CONTROL 0x6b -#define CCK_DISPLAY_REF_CLOCK_CONTROL 0x6c -#define CCK_TRUNK_FORCE_ON (1 << 17) -#define CCK_TRUNK_FORCE_OFF (1 << 16) -#define CCK_FREQUENCY_STATUS (0x1f << 8) -#define CCK_FREQUENCY_STATUS_SHIFT 8 -#define CCK_FREQUENCY_VALUES (0x1f << 0) - /* DPIO registers */ #define DPIO_DEVFN 0 @@ -1905,402 +739,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define OCL2_LDOFUSE_PWR_DIS (1 << 6) #define BXT_PORT_CL1CM_DW30(phy) _BXT_PHY((phy), _PORT_CL1CM_DW30_BC) -/* - * ICL Port/COMBO-PHY Registers - */ -#define _ICL_COMBOPHY_A 0x162000 -#define _ICL_COMBOPHY_B 0x6C000 -#define _EHL_COMBOPHY_C 0x160000 -#define _RKL_COMBOPHY_D 0x161000 -#define _ADL_COMBOPHY_E 0x16B000 - -#define _ICL_COMBOPHY(phy) _PICK(phy, _ICL_COMBOPHY_A, \ - _ICL_COMBOPHY_B, \ - _EHL_COMBOPHY_C, \ - _RKL_COMBOPHY_D, \ - _ADL_COMBOPHY_E) - -/* ICL Port CL_DW registers */ -#define _ICL_PORT_CL_DW(dw, phy) (_ICL_COMBOPHY(phy) + \ - 4 * (dw)) - -#define ICL_PORT_CL_DW5(phy) _MMIO(_ICL_PORT_CL_DW(5, phy)) -#define CL_POWER_DOWN_ENABLE (1 << 4) -#define SUS_CLOCK_CONFIG (3 << 0) - -#define ICL_PORT_CL_DW10(phy) _MMIO(_ICL_PORT_CL_DW(10, phy)) -#define PG_SEQ_DELAY_OVERRIDE_MASK (3 << 25) -#define PG_SEQ_DELAY_OVERRIDE_SHIFT 25 -#define PG_SEQ_DELAY_OVERRIDE_ENABLE (1 << 24) -#define PWR_UP_ALL_LANES (0x0 << 4) -#define PWR_DOWN_LN_3_2_1 (0xe << 4) -#define PWR_DOWN_LN_3_2 (0xc << 4) -#define PWR_DOWN_LN_3 (0x8 << 4) -#define PWR_DOWN_LN_2_1_0 (0x7 << 4) -#define PWR_DOWN_LN_1_0 (0x3 << 4) -#define PWR_DOWN_LN_3_1 (0xa << 4) -#define PWR_DOWN_LN_3_1_0 (0xb << 4) -#define PWR_DOWN_LN_MASK (0xf << 4) -#define PWR_DOWN_LN_SHIFT 4 -#define EDP4K2K_MODE_OVRD_EN (1 << 3) -#define EDP4K2K_MODE_OVRD_OPTIMIZED (1 << 2) - -#define ICL_PORT_CL_DW12(phy) _MMIO(_ICL_PORT_CL_DW(12, phy)) -#define ICL_LANE_ENABLE_AUX (1 << 0) - -/* ICL Port COMP_DW registers */ -#define _ICL_PORT_COMP 0x100 -#define _ICL_PORT_COMP_DW(dw, phy) (_ICL_COMBOPHY(phy) + \ - _ICL_PORT_COMP + 4 * (dw)) - -#define ICL_PORT_COMP_DW0(phy) _MMIO(_ICL_PORT_COMP_DW(0, phy)) -#define COMP_INIT (1 << 31) - -#define ICL_PORT_COMP_DW1(phy) _MMIO(_ICL_PORT_COMP_DW(1, phy)) - -#define ICL_PORT_COMP_DW3(phy) _MMIO(_ICL_PORT_COMP_DW(3, phy)) -#define PROCESS_INFO_DOT_0 (0 << 26) -#define PROCESS_INFO_DOT_1 (1 << 26) -#define PROCESS_INFO_DOT_4 (2 << 26) -#define PROCESS_INFO_MASK (7 << 26) -#define PROCESS_INFO_SHIFT 26 -#define VOLTAGE_INFO_0_85V (0 << 24) -#define VOLTAGE_INFO_0_95V (1 << 24) -#define VOLTAGE_INFO_1_05V (2 << 24) -#define VOLTAGE_INFO_MASK (3 << 24) -#define VOLTAGE_INFO_SHIFT 24 - -#define ICL_PORT_COMP_DW8(phy) _MMIO(_ICL_PORT_COMP_DW(8, phy)) -#define IREFGEN (1 << 24) - -#define ICL_PORT_COMP_DW9(phy) _MMIO(_ICL_PORT_COMP_DW(9, phy)) - -#define ICL_PORT_COMP_DW10(phy) _MMIO(_ICL_PORT_COMP_DW(10, phy)) - -/* ICL Port PCS registers */ -#define _ICL_PORT_PCS_AUX 0x300 -#define _ICL_PORT_PCS_GRP 0x600 -#define _ICL_PORT_PCS_LN(ln) (0x800 + (ln) * 0x100) -#define _ICL_PORT_PCS_DW_AUX(dw, phy) (_ICL_COMBOPHY(phy) + \ - _ICL_PORT_PCS_AUX + 4 * (dw)) -#define _ICL_PORT_PCS_DW_GRP(dw, phy) (_ICL_COMBOPHY(phy) + \ - _ICL_PORT_PCS_GRP + 4 * (dw)) -#define _ICL_PORT_PCS_DW_LN(dw, ln, phy) (_ICL_COMBOPHY(phy) + \ - _ICL_PORT_PCS_LN(ln) + 4 * (dw)) -#define ICL_PORT_PCS_DW1_AUX(phy) _MMIO(_ICL_PORT_PCS_DW_AUX(1, phy)) -#define ICL_PORT_PCS_DW1_GRP(phy) _MMIO(_ICL_PORT_PCS_DW_GRP(1, phy)) -#define ICL_PORT_PCS_DW1_LN(ln, phy) _MMIO(_ICL_PORT_PCS_DW_LN(1, ln, phy)) -#define DCC_MODE_SELECT_MASK (0x3 << 20) -#define DCC_MODE_SELECT_CONTINUOSLY (0x3 << 20) -#define COMMON_KEEPER_EN (1 << 26) -#define LATENCY_OPTIM_MASK (0x3 << 2) -#define LATENCY_OPTIM_VAL(x) ((x) << 2) - -/* ICL Port TX registers */ -#define _ICL_PORT_TX_AUX 0x380 -#define _ICL_PORT_TX_GRP 0x680 -#define _ICL_PORT_TX_LN(ln) (0x880 + (ln) * 0x100) - -#define _ICL_PORT_TX_DW_AUX(dw, phy) (_ICL_COMBOPHY(phy) + \ - _ICL_PORT_TX_AUX + 4 * (dw)) -#define _ICL_PORT_TX_DW_GRP(dw, phy) (_ICL_COMBOPHY(phy) + \ - _ICL_PORT_TX_GRP + 4 * (dw)) -#define _ICL_PORT_TX_DW_LN(dw, ln, phy) (_ICL_COMBOPHY(phy) + \ - _ICL_PORT_TX_LN(ln) + 4 * (dw)) - -#define ICL_PORT_TX_DW2_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(2, phy)) -#define ICL_PORT_TX_DW2_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(2, phy)) -#define ICL_PORT_TX_DW2_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(2, ln, phy)) -#define SWING_SEL_UPPER(x) (((x) >> 3) << 15) -#define SWING_SEL_UPPER_MASK (1 << 15) -#define SWING_SEL_LOWER(x) (((x) & 0x7) << 11) -#define SWING_SEL_LOWER_MASK (0x7 << 11) -#define FRC_LATENCY_OPTIM_MASK (0x7 << 8) -#define FRC_LATENCY_OPTIM_VAL(x) ((x) << 8) -#define RCOMP_SCALAR(x) ((x) << 0) -#define RCOMP_SCALAR_MASK (0xFF << 0) - -#define ICL_PORT_TX_DW4_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(4, phy)) -#define ICL_PORT_TX_DW4_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(4, phy)) -#define ICL_PORT_TX_DW4_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(4, ln, phy)) -#define LOADGEN_SELECT (1 << 31) -#define POST_CURSOR_1(x) ((x) << 12) -#define POST_CURSOR_1_MASK (0x3F << 12) -#define POST_CURSOR_2(x) ((x) << 6) -#define POST_CURSOR_2_MASK (0x3F << 6) -#define CURSOR_COEFF(x) ((x) << 0) -#define CURSOR_COEFF_MASK (0x3F << 0) - -#define ICL_PORT_TX_DW5_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(5, phy)) -#define ICL_PORT_TX_DW5_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(5, phy)) -#define ICL_PORT_TX_DW5_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(5, ln, phy)) -#define TX_TRAINING_EN (1 << 31) -#define TAP2_DISABLE (1 << 30) -#define TAP3_DISABLE (1 << 29) -#define SCALING_MODE_SEL(x) ((x) << 18) -#define SCALING_MODE_SEL_MASK (0x7 << 18) -#define RTERM_SELECT(x) ((x) << 3) -#define RTERM_SELECT_MASK (0x7 << 3) - -#define ICL_PORT_TX_DW7_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(7, phy)) -#define ICL_PORT_TX_DW7_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(7, phy)) -#define ICL_PORT_TX_DW7_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(7, ln, phy)) -#define N_SCALAR(x) ((x) << 24) -#define N_SCALAR_MASK (0x7F << 24) - -#define ICL_PORT_TX_DW8_AUX(phy) _MMIO(_ICL_PORT_TX_DW_AUX(8, phy)) -#define ICL_PORT_TX_DW8_GRP(phy) _MMIO(_ICL_PORT_TX_DW_GRP(8, phy)) -#define ICL_PORT_TX_DW8_LN(ln, phy) _MMIO(_ICL_PORT_TX_DW_LN(8, ln, phy)) -#define ICL_PORT_TX_DW8_ODCC_CLK_SEL REG_BIT(31) -#define ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_MASK REG_GENMASK(30, 29) -#define ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_DIV2 REG_FIELD_PREP(ICL_PORT_TX_DW8_ODCC_CLK_DIV_SEL_MASK, 0x1) - -#define _ICL_DPHY_CHKN_REG 0x194 -#define ICL_DPHY_CHKN(port) _MMIO(_ICL_COMBOPHY(port) + _ICL_DPHY_CHKN_REG) -#define ICL_DPHY_CHKN_AFE_OVER_PPI_STRAP REG_BIT(7) - -#define MG_PHY_PORT_LN(ln, tc_port, ln0p1, ln0p2, ln1p1) \ - _MMIO(_PORT(tc_port, ln0p1, ln0p2) + (ln) * ((ln1p1) - (ln0p1))) - -#define MG_TX_LINK_PARAMS_TX1LN0_PORT1 0x16812C -#define MG_TX_LINK_PARAMS_TX1LN1_PORT1 0x16852C -#define MG_TX_LINK_PARAMS_TX1LN0_PORT2 0x16912C -#define MG_TX_LINK_PARAMS_TX1LN1_PORT2 0x16952C -#define MG_TX_LINK_PARAMS_TX1LN0_PORT3 0x16A12C -#define MG_TX_LINK_PARAMS_TX1LN1_PORT3 0x16A52C -#define MG_TX_LINK_PARAMS_TX1LN0_PORT4 0x16B12C -#define MG_TX_LINK_PARAMS_TX1LN1_PORT4 0x16B52C -#define MG_TX1_LINK_PARAMS(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_LINK_PARAMS_TX1LN0_PORT1, \ - MG_TX_LINK_PARAMS_TX1LN0_PORT2, \ - MG_TX_LINK_PARAMS_TX1LN1_PORT1) - -#define MG_TX_LINK_PARAMS_TX2LN0_PORT1 0x1680AC -#define MG_TX_LINK_PARAMS_TX2LN1_PORT1 0x1684AC -#define MG_TX_LINK_PARAMS_TX2LN0_PORT2 0x1690AC -#define MG_TX_LINK_PARAMS_TX2LN1_PORT2 0x1694AC -#define MG_TX_LINK_PARAMS_TX2LN0_PORT3 0x16A0AC -#define MG_TX_LINK_PARAMS_TX2LN1_PORT3 0x16A4AC -#define MG_TX_LINK_PARAMS_TX2LN0_PORT4 0x16B0AC -#define MG_TX_LINK_PARAMS_TX2LN1_PORT4 0x16B4AC -#define MG_TX2_LINK_PARAMS(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_LINK_PARAMS_TX2LN0_PORT1, \ - MG_TX_LINK_PARAMS_TX2LN0_PORT2, \ - MG_TX_LINK_PARAMS_TX2LN1_PORT1) -#define CRI_USE_FS32 (1 << 5) - -#define MG_TX_PISO_READLOAD_TX1LN0_PORT1 0x16814C -#define MG_TX_PISO_READLOAD_TX1LN1_PORT1 0x16854C -#define MG_TX_PISO_READLOAD_TX1LN0_PORT2 0x16914C -#define MG_TX_PISO_READLOAD_TX1LN1_PORT2 0x16954C -#define MG_TX_PISO_READLOAD_TX1LN0_PORT3 0x16A14C -#define MG_TX_PISO_READLOAD_TX1LN1_PORT3 0x16A54C -#define MG_TX_PISO_READLOAD_TX1LN0_PORT4 0x16B14C -#define MG_TX_PISO_READLOAD_TX1LN1_PORT4 0x16B54C -#define MG_TX1_PISO_READLOAD(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_PISO_READLOAD_TX1LN0_PORT1, \ - MG_TX_PISO_READLOAD_TX1LN0_PORT2, \ - MG_TX_PISO_READLOAD_TX1LN1_PORT1) - -#define MG_TX_PISO_READLOAD_TX2LN0_PORT1 0x1680CC -#define MG_TX_PISO_READLOAD_TX2LN1_PORT1 0x1684CC -#define MG_TX_PISO_READLOAD_TX2LN0_PORT2 0x1690CC -#define MG_TX_PISO_READLOAD_TX2LN1_PORT2 0x1694CC -#define MG_TX_PISO_READLOAD_TX2LN0_PORT3 0x16A0CC -#define MG_TX_PISO_READLOAD_TX2LN1_PORT3 0x16A4CC -#define MG_TX_PISO_READLOAD_TX2LN0_PORT4 0x16B0CC -#define MG_TX_PISO_READLOAD_TX2LN1_PORT4 0x16B4CC -#define MG_TX2_PISO_READLOAD(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_PISO_READLOAD_TX2LN0_PORT1, \ - MG_TX_PISO_READLOAD_TX2LN0_PORT2, \ - MG_TX_PISO_READLOAD_TX2LN1_PORT1) -#define CRI_CALCINIT (1 << 1) - -#define MG_TX_SWINGCTRL_TX1LN0_PORT1 0x168148 -#define MG_TX_SWINGCTRL_TX1LN1_PORT1 0x168548 -#define MG_TX_SWINGCTRL_TX1LN0_PORT2 0x169148 -#define MG_TX_SWINGCTRL_TX1LN1_PORT2 0x169548 -#define MG_TX_SWINGCTRL_TX1LN0_PORT3 0x16A148 -#define MG_TX_SWINGCTRL_TX1LN1_PORT3 0x16A548 -#define MG_TX_SWINGCTRL_TX1LN0_PORT4 0x16B148 -#define MG_TX_SWINGCTRL_TX1LN1_PORT4 0x16B548 -#define MG_TX1_SWINGCTRL(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_SWINGCTRL_TX1LN0_PORT1, \ - MG_TX_SWINGCTRL_TX1LN0_PORT2, \ - MG_TX_SWINGCTRL_TX1LN1_PORT1) - -#define MG_TX_SWINGCTRL_TX2LN0_PORT1 0x1680C8 -#define MG_TX_SWINGCTRL_TX2LN1_PORT1 0x1684C8 -#define MG_TX_SWINGCTRL_TX2LN0_PORT2 0x1690C8 -#define MG_TX_SWINGCTRL_TX2LN1_PORT2 0x1694C8 -#define MG_TX_SWINGCTRL_TX2LN0_PORT3 0x16A0C8 -#define MG_TX_SWINGCTRL_TX2LN1_PORT3 0x16A4C8 -#define MG_TX_SWINGCTRL_TX2LN0_PORT4 0x16B0C8 -#define MG_TX_SWINGCTRL_TX2LN1_PORT4 0x16B4C8 -#define MG_TX2_SWINGCTRL(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_SWINGCTRL_TX2LN0_PORT1, \ - MG_TX_SWINGCTRL_TX2LN0_PORT2, \ - MG_TX_SWINGCTRL_TX2LN1_PORT1) -#define CRI_TXDEEMPH_OVERRIDE_17_12(x) ((x) << 0) -#define CRI_TXDEEMPH_OVERRIDE_17_12_MASK (0x3F << 0) - -#define MG_TX_DRVCTRL_TX1LN0_TXPORT1 0x168144 -#define MG_TX_DRVCTRL_TX1LN1_TXPORT1 0x168544 -#define MG_TX_DRVCTRL_TX1LN0_TXPORT2 0x169144 -#define MG_TX_DRVCTRL_TX1LN1_TXPORT2 0x169544 -#define MG_TX_DRVCTRL_TX1LN0_TXPORT3 0x16A144 -#define MG_TX_DRVCTRL_TX1LN1_TXPORT3 0x16A544 -#define MG_TX_DRVCTRL_TX1LN0_TXPORT4 0x16B144 -#define MG_TX_DRVCTRL_TX1LN1_TXPORT4 0x16B544 -#define MG_TX1_DRVCTRL(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_DRVCTRL_TX1LN0_TXPORT1, \ - MG_TX_DRVCTRL_TX1LN0_TXPORT2, \ - MG_TX_DRVCTRL_TX1LN1_TXPORT1) - -#define MG_TX_DRVCTRL_TX2LN0_PORT1 0x1680C4 -#define MG_TX_DRVCTRL_TX2LN1_PORT1 0x1684C4 -#define MG_TX_DRVCTRL_TX2LN0_PORT2 0x1690C4 -#define MG_TX_DRVCTRL_TX2LN1_PORT2 0x1694C4 -#define MG_TX_DRVCTRL_TX2LN0_PORT3 0x16A0C4 -#define MG_TX_DRVCTRL_TX2LN1_PORT3 0x16A4C4 -#define MG_TX_DRVCTRL_TX2LN0_PORT4 0x16B0C4 -#define MG_TX_DRVCTRL_TX2LN1_PORT4 0x16B4C4 -#define MG_TX2_DRVCTRL(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_DRVCTRL_TX2LN0_PORT1, \ - MG_TX_DRVCTRL_TX2LN0_PORT2, \ - MG_TX_DRVCTRL_TX2LN1_PORT1) -#define CRI_TXDEEMPH_OVERRIDE_11_6(x) ((x) << 24) -#define CRI_TXDEEMPH_OVERRIDE_11_6_MASK (0x3F << 24) -#define CRI_TXDEEMPH_OVERRIDE_EN (1 << 22) -#define CRI_TXDEEMPH_OVERRIDE_5_0(x) ((x) << 16) -#define CRI_TXDEEMPH_OVERRIDE_5_0_MASK (0x3F << 16) -#define CRI_LOADGEN_SEL(x) ((x) << 12) -#define CRI_LOADGEN_SEL_MASK (0x3 << 12) - -#define MG_CLKHUB_LN0_PORT1 0x16839C -#define MG_CLKHUB_LN1_PORT1 0x16879C -#define MG_CLKHUB_LN0_PORT2 0x16939C -#define MG_CLKHUB_LN1_PORT2 0x16979C -#define MG_CLKHUB_LN0_PORT3 0x16A39C -#define MG_CLKHUB_LN1_PORT3 0x16A79C -#define MG_CLKHUB_LN0_PORT4 0x16B39C -#define MG_CLKHUB_LN1_PORT4 0x16B79C -#define MG_CLKHUB(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_CLKHUB_LN0_PORT1, \ - MG_CLKHUB_LN0_PORT2, \ - MG_CLKHUB_LN1_PORT1) -#define CFG_LOW_RATE_LKREN_EN (1 << 11) - -#define MG_TX_DCC_TX1LN0_PORT1 0x168110 -#define MG_TX_DCC_TX1LN1_PORT1 0x168510 -#define MG_TX_DCC_TX1LN0_PORT2 0x169110 -#define MG_TX_DCC_TX1LN1_PORT2 0x169510 -#define MG_TX_DCC_TX1LN0_PORT3 0x16A110 -#define MG_TX_DCC_TX1LN1_PORT3 0x16A510 -#define MG_TX_DCC_TX1LN0_PORT4 0x16B110 -#define MG_TX_DCC_TX1LN1_PORT4 0x16B510 -#define MG_TX1_DCC(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_DCC_TX1LN0_PORT1, \ - MG_TX_DCC_TX1LN0_PORT2, \ - MG_TX_DCC_TX1LN1_PORT1) -#define MG_TX_DCC_TX2LN0_PORT1 0x168090 -#define MG_TX_DCC_TX2LN1_PORT1 0x168490 -#define MG_TX_DCC_TX2LN0_PORT2 0x169090 -#define MG_TX_DCC_TX2LN1_PORT2 0x169490 -#define MG_TX_DCC_TX2LN0_PORT3 0x16A090 -#define MG_TX_DCC_TX2LN1_PORT3 0x16A490 -#define MG_TX_DCC_TX2LN0_PORT4 0x16B090 -#define MG_TX_DCC_TX2LN1_PORT4 0x16B490 -#define MG_TX2_DCC(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_TX_DCC_TX2LN0_PORT1, \ - MG_TX_DCC_TX2LN0_PORT2, \ - MG_TX_DCC_TX2LN1_PORT1) -#define CFG_AMI_CK_DIV_OVERRIDE_VAL(x) ((x) << 25) -#define CFG_AMI_CK_DIV_OVERRIDE_VAL_MASK (0x3 << 25) -#define CFG_AMI_CK_DIV_OVERRIDE_EN (1 << 24) - -#define MG_DP_MODE_LN0_ACU_PORT1 0x1683A0 -#define MG_DP_MODE_LN1_ACU_PORT1 0x1687A0 -#define MG_DP_MODE_LN0_ACU_PORT2 0x1693A0 -#define MG_DP_MODE_LN1_ACU_PORT2 0x1697A0 -#define MG_DP_MODE_LN0_ACU_PORT3 0x16A3A0 -#define MG_DP_MODE_LN1_ACU_PORT3 0x16A7A0 -#define MG_DP_MODE_LN0_ACU_PORT4 0x16B3A0 -#define MG_DP_MODE_LN1_ACU_PORT4 0x16B7A0 -#define MG_DP_MODE(ln, tc_port) \ - MG_PHY_PORT_LN(ln, tc_port, MG_DP_MODE_LN0_ACU_PORT1, \ - MG_DP_MODE_LN0_ACU_PORT2, \ - MG_DP_MODE_LN1_ACU_PORT1) -#define MG_DP_MODE_CFG_DP_X2_MODE (1 << 7) -#define MG_DP_MODE_CFG_DP_X1_MODE (1 << 6) - -/* - * DG2 SNPS PHY registers (TC1 = PHY_E) - */ -#define _SNPS_PHY_A_BASE 0x168000 -#define _SNPS_PHY_B_BASE 0x169000 -#define _SNPS_PHY(phy) _PHY(phy, \ - _SNPS_PHY_A_BASE, \ - _SNPS_PHY_B_BASE) -#define _SNPS2(phy, reg) (_SNPS_PHY(phy) - \ - _SNPS_PHY_A_BASE + (reg)) -#define _MMIO_SNPS(phy, reg) _MMIO(_SNPS2(phy, reg)) -#define _MMIO_SNPS_LN(ln, phy, reg) _MMIO(_SNPS2(phy, \ - (reg) + (ln) * 0x10)) - -#define SNPS_PHY_MPLLB_CP(phy) _MMIO_SNPS(phy, 0x168000) -#define SNPS_PHY_MPLLB_CP_INT REG_GENMASK(31, 25) -#define SNPS_PHY_MPLLB_CP_INT_GS REG_GENMASK(23, 17) -#define SNPS_PHY_MPLLB_CP_PROP REG_GENMASK(15, 9) -#define SNPS_PHY_MPLLB_CP_PROP_GS REG_GENMASK(7, 1) - -#define SNPS_PHY_MPLLB_DIV(phy) _MMIO_SNPS(phy, 0x168004) -#define SNPS_PHY_MPLLB_FORCE_EN REG_BIT(31) -#define SNPS_PHY_MPLLB_DIV_CLK_EN REG_BIT(30) -#define SNPS_PHY_MPLLB_DIV5_CLK_EN REG_BIT(29) -#define SNPS_PHY_MPLLB_V2I REG_GENMASK(27, 26) -#define SNPS_PHY_MPLLB_FREQ_VCO REG_GENMASK(25, 24) -#define SNPS_PHY_MPLLB_DIV_MULTIPLIER REG_GENMASK(23, 16) -#define SNPS_PHY_MPLLB_PMIX_EN REG_BIT(10) -#define SNPS_PHY_MPLLB_DP2_MODE REG_BIT(9) -#define SNPS_PHY_MPLLB_WORD_DIV2_EN REG_BIT(8) -#define SNPS_PHY_MPLLB_TX_CLK_DIV REG_GENMASK(7, 5) -#define SNPS_PHY_MPLLB_SHIM_DIV32_CLK_SEL REG_BIT(0) - -#define SNPS_PHY_MPLLB_FRACN1(phy) _MMIO_SNPS(phy, 0x168008) -#define SNPS_PHY_MPLLB_FRACN_EN REG_BIT(31) -#define SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN REG_BIT(30) -#define SNPS_PHY_MPLLB_FRACN_DEN REG_GENMASK(15, 0) - -#define SNPS_PHY_MPLLB_FRACN2(phy) _MMIO_SNPS(phy, 0x16800C) -#define SNPS_PHY_MPLLB_FRACN_REM REG_GENMASK(31, 16) -#define SNPS_PHY_MPLLB_FRACN_QUOT REG_GENMASK(15, 0) - -#define SNPS_PHY_MPLLB_SSCEN(phy) _MMIO_SNPS(phy, 0x168014) -#define SNPS_PHY_MPLLB_SSC_EN REG_BIT(31) -#define SNPS_PHY_MPLLB_SSC_UP_SPREAD REG_BIT(30) -#define SNPS_PHY_MPLLB_SSC_PEAK REG_GENMASK(29, 10) - -#define SNPS_PHY_MPLLB_SSCSTEP(phy) _MMIO_SNPS(phy, 0x168018) -#define SNPS_PHY_MPLLB_SSC_STEPSIZE REG_GENMASK(31, 11) - -#define SNPS_PHY_MPLLB_DIV2(phy) _MMIO_SNPS(phy, 0x16801C) -#define SNPS_PHY_MPLLB_HDMI_PIXEL_CLK_DIV REG_GENMASK(19, 18) -#define SNPS_PHY_MPLLB_HDMI_DIV REG_GENMASK(17, 15) -#define SNPS_PHY_MPLLB_REF_CLK_DIV REG_GENMASK(14, 12) -#define SNPS_PHY_MPLLB_MULTIPLIER REG_GENMASK(11, 0) - -#define SNPS_PHY_REF_CONTROL(phy) _MMIO_SNPS(phy, 0x168188) -#define SNPS_PHY_REF_CONTROL_REF_RANGE REG_GENMASK(31, 27) - -#define SNPS_PHY_TX_REQ(phy) _MMIO_SNPS(phy, 0x168200) -#define SNPS_PHY_TX_REQ_LN_DIS_PWR_STATE_PSR REG_GENMASK(31, 30) - -#define SNPS_PHY_TX_EQ(ln, phy) _MMIO_SNPS_LN(ln, phy, 0x168300) -#define SNPS_PHY_TX_EQ_MAIN REG_GENMASK(23, 18) -#define SNPS_PHY_TX_EQ_POST REG_GENMASK(15, 10) -#define SNPS_PHY_TX_EQ_PRE REG_GENMASK(7, 2) - /* The spec defines this only for BXT PHY0, but lets assume that this * would exist for PHY1 too if it had a second channel. */ @@ -2309,21 +747,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define BXT_PORT_CL2CM_DW6(phy) _BXT_PHY((phy), _PORT_CL2CM_DW6_BC) #define DW6_OLDO_DYN_PWR_DOWN_EN (1 << 28) -#define FIA1_BASE 0x163000 -#define FIA2_BASE 0x16E000 -#define FIA3_BASE 0x16F000 -#define _FIA(fia) _PICK((fia), FIA1_BASE, FIA2_BASE, FIA3_BASE) -#define _MMIO_FIA(fia, off) _MMIO(_FIA(fia) + (off)) - -/* ICL PHY DFLEX registers */ -#define PORT_TX_DFLEXDPMLE1(fia) _MMIO_FIA((fia), 0x008C0) -#define DFLEXDPMLE1_DPMLETC_MASK(idx) (0xf << (4 * (idx))) -#define DFLEXDPMLE1_DPMLETC_ML0(idx) (1 << (4 * (idx))) -#define DFLEXDPMLE1_DPMLETC_ML1_0(idx) (3 << (4 * (idx))) -#define DFLEXDPMLE1_DPMLETC_ML3(idx) (8 << (4 * (idx))) -#define DFLEXDPMLE1_DPMLETC_ML3_2(idx) (12 << (4 * (idx))) -#define DFLEXDPMLE1_DPMLETC_ML3_0(idx) (15 << (4 * (idx))) - /* BXT PHY Ref registers */ #define _PORT_REF_DW3_A 0x16218C #define _PORT_REF_DW3_BC 0x6C18C @@ -2548,65 +971,13 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN11_VEBOX2_RING_BASE 0x1d8000 #define XEHP_VEBOX3_RING_BASE 0x1e8000 #define XEHP_VEBOX4_RING_BASE 0x1f8000 +#define GEN12_COMPUTE0_RING_BASE 0x1a000 +#define GEN12_COMPUTE1_RING_BASE 0x1c000 +#define GEN12_COMPUTE2_RING_BASE 0x1e000 +#define GEN12_COMPUTE3_RING_BASE 0x26000 #define BLT_RING_BASE 0x22000 -#define RING_TAIL(base) _MMIO((base) + 0x30) -#define RING_HEAD(base) _MMIO((base) + 0x34) -#define RING_START(base) _MMIO((base) + 0x38) -#define RING_CTL(base) _MMIO((base) + 0x3c) -#define RING_CTL_SIZE(size) ((size) - PAGE_SIZE) /* in bytes -> pages */ -#define RING_SYNC_0(base) _MMIO((base) + 0x40) -#define RING_SYNC_1(base) _MMIO((base) + 0x44) -#define RING_SYNC_2(base) _MMIO((base) + 0x48) -#define GEN6_RVSYNC (RING_SYNC_0(RENDER_RING_BASE)) -#define GEN6_RBSYNC (RING_SYNC_1(RENDER_RING_BASE)) -#define GEN6_RVESYNC (RING_SYNC_2(RENDER_RING_BASE)) -#define GEN6_VBSYNC (RING_SYNC_0(GEN6_BSD_RING_BASE)) -#define GEN6_VRSYNC (RING_SYNC_1(GEN6_BSD_RING_BASE)) -#define GEN6_VVESYNC (RING_SYNC_2(GEN6_BSD_RING_BASE)) -#define GEN6_BRSYNC (RING_SYNC_0(BLT_RING_BASE)) -#define GEN6_BVSYNC (RING_SYNC_1(BLT_RING_BASE)) -#define GEN6_BVESYNC (RING_SYNC_2(BLT_RING_BASE)) -#define GEN6_VEBSYNC (RING_SYNC_0(VEBOX_RING_BASE)) -#define GEN6_VERSYNC (RING_SYNC_1(VEBOX_RING_BASE)) -#define GEN6_VEVSYNC (RING_SYNC_2(VEBOX_RING_BASE)) -#define GEN6_NOSYNC INVALID_MMIO_REG -#define RING_PSMI_CTL(base) _MMIO((base) + 0x50) -#define RING_MAX_IDLE(base) _MMIO((base) + 0x54) -#define RING_HWS_PGA(base) _MMIO((base) + 0x80) -#define RING_ID(base) _MMIO((base) + 0x8c) -#define RING_HWS_PGA_GEN6(base) _MMIO((base) + 0x2080) - -#define RING_CMD_CCTL(base) _MMIO((base) + 0xc4) -/* - * CMD_CCTL read/write fields take a MOCS value and _not_ a table index. - * The lsb of each can be considered a separate enabling bit for encryption. - * 6:0 == default MOCS value for reads => 6:1 == table index for reads. - * 13:7 == default MOCS value for writes => 13:8 == table index for writes. - * 15:14 == Reserved => 31:30 are set to 0. - */ -#define CMD_CCTL_WRITE_OVERRIDE_MASK REG_GENMASK(13, 7) -#define CMD_CCTL_READ_OVERRIDE_MASK REG_GENMASK(6, 0) -#define CMD_CCTL_MOCS_MASK (CMD_CCTL_WRITE_OVERRIDE_MASK | \ - CMD_CCTL_READ_OVERRIDE_MASK) -#define CMD_CCTL_MOCS_OVERRIDE(write, read) \ - (REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, (write) << 1) | \ - REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, (read) << 1)) - -#define BLIT_CCTL(base) _MMIO((base) + 0x204) -#define BLIT_CCTL_DST_MOCS_MASK REG_GENMASK(14, 8) -#define BLIT_CCTL_SRC_MOCS_MASK REG_GENMASK(6, 0) -#define BLIT_CCTL_MASK (BLIT_CCTL_DST_MOCS_MASK | \ - BLIT_CCTL_SRC_MOCS_MASK) -#define BLIT_CCTL_MOCS(dst, src) \ - (REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, (dst) << 1) | \ - REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, (src) << 1)) - -#define RING_RESET_CTL(base) _MMIO((base) + 0xd0) -#define RESET_CTL_CAT_ERROR REG_BIT(2) -#define RESET_CTL_READY_TO_RESET REG_BIT(1) -#define RESET_CTL_REQUEST_RESET REG_BIT(0) - -#define RING_SEMA_WAIT_POLL(base) _MMIO((base) + 0x24c) + + #define HSW_GTT_CACHE_EN _MMIO(0x4024) #define GTT_CACHE_EN_ALL 0xF0007FFF @@ -2623,190 +994,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN7_MEDIA_MAX_REQ_COUNT _MMIO(0x4070) #define GEN7_GFX_MAX_REQ_COUNT _MMIO(0x4074) -#define GAMTARBMODE _MMIO(0x04a08) -#define ARB_MODE_BWGTLB_DISABLE (1 << 9) -#define ARB_MODE_SWIZZLE_BDW (1 << 1) -#define RENDER_HWS_PGA_GEN7 _MMIO(0x04080) - -#define _RING_FAULT_REG_RCS 0x4094 -#define _RING_FAULT_REG_VCS 0x4194 -#define _RING_FAULT_REG_BCS 0x4294 -#define _RING_FAULT_REG_VECS 0x4394 -#define RING_FAULT_REG(engine) _MMIO(_PICK((engine)->class, \ - _RING_FAULT_REG_RCS, \ - _RING_FAULT_REG_VCS, \ - _RING_FAULT_REG_VECS, \ - _RING_FAULT_REG_BCS)) -#define GEN8_RING_FAULT_REG _MMIO(0x4094) -#define GEN12_RING_FAULT_REG _MMIO(0xcec4) -#define GEN8_RING_FAULT_ENGINE_ID(x) (((x) >> 12) & 0x7) -#define RING_FAULT_GTTSEL_MASK (1 << 11) -#define RING_FAULT_SRCID(x) (((x) >> 3) & 0xff) -#define RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3) -#define RING_FAULT_VALID (1 << 0) -#define DONE_REG _MMIO(0x40b0) -#define GEN12_GAM_DONE _MMIO(0xcf68) -#define GEN8_PRIVATE_PAT_LO _MMIO(0x40e0) -#define GEN8_PRIVATE_PAT_HI _MMIO(0x40e0 + 4) -#define GEN10_PAT_INDEX(index) _MMIO(0x40e0 + (index) * 4) -#define GEN12_PAT_INDEX(index) _MMIO(0x4800 + (index) * 4) -#define BSD_HWS_PGA_GEN7 _MMIO(0x04180) -#define GEN12_GFX_CCS_AUX_NV _MMIO(0x4208) -#define GEN12_VD0_AUX_NV _MMIO(0x4218) -#define GEN12_VD1_AUX_NV _MMIO(0x4228) -#define GEN12_VD2_AUX_NV _MMIO(0x4298) -#define GEN12_VD3_AUX_NV _MMIO(0x42A8) -#define GEN12_VE0_AUX_NV _MMIO(0x4238) -#define GEN12_VE1_AUX_NV _MMIO(0x42B8) -#define AUX_INV REG_BIT(0) -#define BLT_HWS_PGA_GEN7 _MMIO(0x04280) -#define VEBOX_HWS_PGA_GEN7 _MMIO(0x04380) -#define RING_ACTHD(base) _MMIO((base) + 0x74) -#define RING_ACTHD_UDW(base) _MMIO((base) + 0x5c) -#define RING_NOPID(base) _MMIO((base) + 0x94) -#define RING_IMR(base) _MMIO((base) + 0xa8) -#define RING_HWSTAM(base) _MMIO((base) + 0x98) -#define RING_TIMESTAMP(base) _MMIO((base) + 0x358) -#define RING_TIMESTAMP_UDW(base) _MMIO((base) + 0x358 + 4) -#define TAIL_ADDR 0x001FFFF8 -#define HEAD_WRAP_COUNT 0xFFE00000 -#define HEAD_WRAP_ONE 0x00200000 -#define HEAD_ADDR 0x001FFFFC -#define RING_NR_PAGES 0x001FF000 -#define RING_REPORT_MASK 0x00000006 -#define RING_REPORT_64K 0x00000002 -#define RING_REPORT_128K 0x00000004 -#define RING_NO_REPORT 0x00000000 -#define RING_VALID_MASK 0x00000001 -#define RING_VALID 0x00000001 -#define RING_INVALID 0x00000000 -#define RING_WAIT_I8XX (1 << 0) /* gen2, PRBx_HEAD */ -#define RING_WAIT (1 << 11) /* gen3+, PRBx_CTL */ -#define RING_WAIT_SEMAPHORE (1 << 10) /* gen6+ */ - -#define MISC_STATUS0 _MMIO(0xA500) -#define MISC_STATUS1 _MMIO(0xA504) - -/* There are 16 64-bit CS General Purpose Registers per-engine on Gen8+ */ -#define GEN8_RING_CS_GPR(base, n) _MMIO((base) + 0x600 + (n) * 8) -#define GEN8_RING_CS_GPR_UDW(base, n) _MMIO((base) + 0x600 + (n) * 8 + 4) - -#define RING_FORCE_TO_NONPRIV(base, i) _MMIO(((base) + 0x4D0) + (i) * 4) -#define RING_FORCE_TO_NONPRIV_ADDRESS_MASK REG_GENMASK(25, 2) -#define RING_FORCE_TO_NONPRIV_ACCESS_RW (0 << 28) /* CFL+ & Gen11+ */ -#define RING_FORCE_TO_NONPRIV_ACCESS_RD (1 << 28) -#define RING_FORCE_TO_NONPRIV_ACCESS_WR (2 << 28) -#define RING_FORCE_TO_NONPRIV_ACCESS_INVALID (3 << 28) -#define RING_FORCE_TO_NONPRIV_ACCESS_MASK (3 << 28) -#define RING_FORCE_TO_NONPRIV_RANGE_1 (0 << 0) /* CFL+ & Gen11+ */ -#define RING_FORCE_TO_NONPRIV_RANGE_4 (1 << 0) -#define RING_FORCE_TO_NONPRIV_RANGE_16 (2 << 0) -#define RING_FORCE_TO_NONPRIV_RANGE_64 (3 << 0) -#define RING_FORCE_TO_NONPRIV_RANGE_MASK (3 << 0) -#define RING_FORCE_TO_NONPRIV_MASK_VALID \ - (RING_FORCE_TO_NONPRIV_RANGE_MASK \ - | RING_FORCE_TO_NONPRIV_ACCESS_MASK) -#define RING_MAX_NONPRIV_SLOTS 12 - -#define GEN7_TLB_RD_ADDR _MMIO(0x4700) - -#define GEN9_GAMT_ECO_REG_RW_IA _MMIO(0x4ab0) -#define GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS (1 << 18) - -#define GEN8_GAMW_ECO_DEV_RW_IA _MMIO(0x4080) -#define GAMW_ECO_ENABLE_64K_IPS_FIELD 0xF -#define GAMW_ECO_DEV_CTX_RELOAD_DISABLE (1 << 7) - -#define GAMT_CHKN_BIT_REG _MMIO(0x4ab8) -#define GAMT_CHKN_DISABLE_L3_COH_PIPE (1 << 31) -#define GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING (1 << 28) -#define GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT (1 << 24) - -#define GEN8_RTCR _MMIO(0x4260) -#define GEN8_M1TCR _MMIO(0x4264) -#define GEN8_M2TCR _MMIO(0x4268) -#define GEN8_BTCR _MMIO(0x426c) -#define GEN8_VTCR _MMIO(0x4270) - -#if 0 -#define PRB0_TAIL _MMIO(0x2030) -#define PRB0_HEAD _MMIO(0x2034) -#define PRB0_START _MMIO(0x2038) -#define PRB0_CTL _MMIO(0x203c) -#define PRB1_TAIL _MMIO(0x2040) /* 915+ only */ -#define PRB1_HEAD _MMIO(0x2044) /* 915+ only */ -#define PRB1_START _MMIO(0x2048) /* 915+ only */ -#define PRB1_CTL _MMIO(0x204c) /* 915+ only */ -#endif -#define IPEIR_I965 _MMIO(0x2064) -#define IPEHR_I965 _MMIO(0x2068) -#define GEN7_SC_INSTDONE _MMIO(0x7100) -#define GEN12_SC_INSTDONE_EXTRA _MMIO(0x7104) -#define GEN12_SC_INSTDONE_EXTRA2 _MMIO(0x7108) -#define GEN7_SAMPLER_INSTDONE _MMIO(0xe160) -#define GEN7_ROW_INSTDONE _MMIO(0xe164) -#define XEHPG_INSTDONE_GEOM_SVG _MMIO(0x666c) -#define MCFG_MCR_SELECTOR _MMIO(0xfd0) -#define SF_MCR_SELECTOR _MMIO(0xfd8) -#define GEN8_MCR_SELECTOR _MMIO(0xfdc) -#define GEN8_MCR_SLICE(slice) (((slice) & 3) << 26) -#define GEN8_MCR_SLICE_MASK GEN8_MCR_SLICE(3) -#define GEN8_MCR_SUBSLICE(subslice) (((subslice) & 3) << 24) -#define GEN8_MCR_SUBSLICE_MASK GEN8_MCR_SUBSLICE(3) -#define GEN11_MCR_SLICE(slice) (((slice) & 0xf) << 27) -#define GEN11_MCR_SLICE_MASK GEN11_MCR_SLICE(0xf) -#define GEN11_MCR_SUBSLICE(subslice) (((subslice) & 0x7) << 24) -#define GEN11_MCR_SUBSLICE_MASK GEN11_MCR_SUBSLICE(0x7) -#define RING_IPEIR(base) _MMIO((base) + 0x64) -#define RING_IPEHR(base) _MMIO((base) + 0x68) -#define RING_EIR(base) _MMIO((base) + 0xb0) -#define RING_EMR(base) _MMIO((base) + 0xb4) -#define RING_ESR(base) _MMIO((base) + 0xb8) -/* - * On GEN4, only the render ring INSTDONE exists and has a different - * layout than the GEN7+ version. - * The GEN2 counterpart of this register is GEN2_INSTDONE. - */ -#define RING_INSTDONE(base) _MMIO((base) + 0x6c) -#define RING_INSTPS(base) _MMIO((base) + 0x70) -#define RING_DMA_FADD(base) _MMIO((base) + 0x78) -#define RING_DMA_FADD_UDW(base) _MMIO((base) + 0x60) /* gen8+ */ -#define RING_INSTPM(base) _MMIO((base) + 0xc0) -#define RING_MI_MODE(base) _MMIO((base) + 0x9c) -#define RING_CMD_BUF_CCTL(base) _MMIO((base) + 0x84) -#define INSTPS _MMIO(0x2070) /* 965+ only */ -#define GEN4_INSTDONE1 _MMIO(0x207c) /* 965+ only, aka INSTDONE_2 on SNB */ -#define ACTHD_I965 _MMIO(0x2074) -#define HWS_PGA _MMIO(0x2080) -#define HWS_ADDRESS_MASK 0xfffff000 -#define HWS_START_ADDRESS_SHIFT 4 -#define PWRCTXA _MMIO(0x2088) /* 965GM+ only */ -#define PWRCTX_EN (1 << 0) -#define IPEIR(base) _MMIO((base) + 0x88) -#define IPEHR(base) _MMIO((base) + 0x8c) -#define GEN2_INSTDONE _MMIO(0x2090) -#define NOPID _MMIO(0x2094) -#define HWSTAM _MMIO(0x2098) -#define DMA_FADD_I8XX(base) _MMIO((base) + 0xd0) -#define RING_BBSTATE(base) _MMIO((base) + 0x110) -#define RING_BB_PPGTT (1 << 5) -#define RING_SBBADDR(base) _MMIO((base) + 0x114) /* hsw+ */ -#define RING_SBBSTATE(base) _MMIO((base) + 0x118) /* hsw+ */ -#define RING_SBBADDR_UDW(base) _MMIO((base) + 0x11c) /* gen8+ */ -#define RING_BBADDR(base) _MMIO((base) + 0x140) -#define RING_BBADDR_UDW(base) _MMIO((base) + 0x168) /* gen8+ */ -#define RING_BB_PER_CTX_PTR(base) _MMIO((base) + 0x1c0) /* gen8+ */ -#define RING_INDIRECT_CTX(base) _MMIO((base) + 0x1c4) /* gen8+ */ -#define RING_INDIRECT_CTX_OFFSET(base) _MMIO((base) + 0x1c8) /* gen8+ */ -#define RING_CTX_TIMESTAMP(base) _MMIO((base) + 0x3a8) /* gen8+ */ - -#define VDBOX_CGCTL3F10(base) _MMIO((base) + 0x3f10) -#define IECPUNIT_CLKGATE_DIS REG_BIT(22) - -#define VDBOX_CGCTL3F18(base) _MMIO((base) + 0x3f18) -#define ALNUNIT_CLKGATE_DIS REG_BIT(13) - -#define ERROR_GEN6 _MMIO(0x40a0) #define GEN7_ERR_INT _MMIO(0x44040) #define ERR_INT_POISON (1 << 31) #define ERR_INT_MMIO_UNCLAIMED (1 << 13) @@ -2819,20 +1006,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define ERR_INT_FIFO_UNDERRUN_A (1 << 0) #define ERR_INT_FIFO_UNDERRUN(pipe) (1 << ((pipe) * 3)) -#define GEN8_FAULT_TLB_DATA0 _MMIO(0x4b10) -#define GEN8_FAULT_TLB_DATA1 _MMIO(0x4b14) -#define GEN12_FAULT_TLB_DATA0 _MMIO(0xceb8) -#define GEN12_FAULT_TLB_DATA1 _MMIO(0xcebc) -#define FAULT_VA_HIGH_BITS (0xf << 0) -#define FAULT_GTT_SEL (1 << 4) - -#define GEN12_GFX_TLB_INV_CR _MMIO(0xced8) -#define GEN12_VD_TLB_INV_CR _MMIO(0xcedc) -#define GEN12_VE_TLB_INV_CR _MMIO(0xcee0) -#define GEN12_BLT_TLB_INV_CR _MMIO(0xcee4) - -#define GEN12_AUX_ERR_DBG _MMIO(0x43f4) - #define FPGA_DBG _MMIO(0x42300) #define FPGA_DBG_RM_NOCLAIM REG_BIT(31) @@ -2860,95 +1033,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define DERRMR_PIPEC_VBLANK (1 << 21) #define DERRMR_PIPEC_HBLANK (1 << 22) - -/* GM45+ chicken bits -- debug workaround bits that may be required - * for various sorts of correct behavior. The top 16 bits of each are - * the enables for writing to the corresponding low bit. - */ -#define _3D_CHICKEN _MMIO(0x2084) -#define _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB (1 << 10) -#define _3D_CHICKEN2 _MMIO(0x208c) - -#define FF_SLICE_CHICKEN _MMIO(0x2088) -#define FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX (1 << 1) - -/* Disables pipelining of read flushes past the SF-WIZ interface. - * Required on all Ironlake steppings according to the B-Spec, but the - * particular danger of not doing so is not specified. - */ -# define _3D_CHICKEN2_WM_READ_PIPELINED (1 << 14) -#define _3D_CHICKEN3 _MMIO(0x2090) -#define _3D_CHICKEN_SF_PROVOKING_VERTEX_FIX (1 << 12) -#define _3D_CHICKEN_SF_DISABLE_OBJEND_CULL (1 << 10) -#define _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE (1 << 5) -#define _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL (1 << 5) -#define _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x) ((x) << 1) /* gen8+ */ -#define _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH (1 << 1) /* gen6 */ - -#define MI_MODE _MMIO(0x209c) -# define VS_TIMER_DISPATCH (1 << 6) -# define MI_FLUSH_ENABLE (1 << 12) -# define TGL_NESTED_BB_EN (1 << 12) -# define ASYNC_FLIP_PERF_DISABLE (1 << 14) -# define MODE_IDLE (1 << 9) -# define STOP_RING (1 << 8) - -#define GEN6_GT_MODE _MMIO(0x20d0) -#define GEN7_GT_MODE _MMIO(0x7008) -#define GEN6_WIZ_HASHING(hi, lo) (((hi) << 9) | ((lo) << 7)) -#define GEN6_WIZ_HASHING_8x8 GEN6_WIZ_HASHING(0, 0) -#define GEN6_WIZ_HASHING_8x4 GEN6_WIZ_HASHING(0, 1) -#define GEN6_WIZ_HASHING_16x4 GEN6_WIZ_HASHING(1, 0) -#define GEN6_WIZ_HASHING_MASK GEN6_WIZ_HASHING(1, 1) -#define GEN6_TD_FOUR_ROW_DISPATCH_DISABLE (1 << 5) -#define GEN9_IZ_HASHING_MASK(slice) (0x3 << ((slice) * 2)) -#define GEN9_IZ_HASHING(slice, val) ((val) << ((slice) * 2)) - -/* chicken reg for WaConextSwitchWithConcurrentTLBInvalidate */ -#define GEN9_CSFE_CHICKEN1_RCS _MMIO(0x20D4) -#define GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE (1 << 2) -#define GEN11_ENABLE_32_PLANE_MODE (1 << 7) - -#define SCCGCTL94DC _MMIO(0x94dc) -#define CG3DDISURB REG_BIT(14) - -#define MLTICTXCTL _MMIO(0xb170) -#define TDONRENDER REG_BIT(2) - -#define L3SQCREG1_CCS0 _MMIO(0xb200) -#define FLUSHALLNONCOH REG_BIT(5) - -/* WaClearTdlStateAckDirtyBits */ -#define GEN8_STATE_ACK _MMIO(0x20F0) -#define GEN9_STATE_ACK_SLICE1 _MMIO(0x20F8) -#define GEN9_STATE_ACK_SLICE2 _MMIO(0x2100) -#define GEN9_STATE_ACK_TDL0 (1 << 12) -#define GEN9_STATE_ACK_TDL1 (1 << 13) -#define GEN9_STATE_ACK_TDL2 (1 << 14) -#define GEN9_STATE_ACK_TDL3 (1 << 15) -#define GEN9_SUBSLICE_TDL_ACK_BITS \ - (GEN9_STATE_ACK_TDL3 | GEN9_STATE_ACK_TDL2 | \ - GEN9_STATE_ACK_TDL1 | GEN9_STATE_ACK_TDL0) - -#define GFX_MODE _MMIO(0x2520) -#define GFX_MODE_GEN7 _MMIO(0x229c) -#define RING_MODE_GEN7(base) _MMIO((base) + 0x29c) -#define GFX_RUN_LIST_ENABLE (1 << 15) -#define GFX_INTERRUPT_STEERING (1 << 14) -#define GFX_TLB_INVALIDATE_EXPLICIT (1 << 13) -#define GFX_SURFACE_FAULT_ENABLE (1 << 12) -#define GFX_REPLAY_MODE (1 << 11) -#define GFX_PSMI_GRANULARITY (1 << 10) -#define GFX_PPGTT_ENABLE (1 << 9) -#define GEN8_GFX_PPGTT_48B (1 << 7) - -#define GFX_FORWARD_VBLANK_MASK (3 << 5) -#define GFX_FORWARD_VBLANK_NEVER (0 << 5) -#define GFX_FORWARD_VBLANK_ALWAYS (1 << 5) -#define GFX_FORWARD_VBLANK_COND (2 << 5) - -#define GEN11_GFX_DISABLE_LEGACY_MODE (1 << 3) - #define VLV_GU_CTL0 _MMIO(VLV_DISPLAY_BASE + 0x2030) #define VLV_GU_CTL1 _MMIO(VLV_DISPLAY_BASE + 0x2034) #define SCPD0 _MMIO(0x209c) /* 915+ only */ @@ -2988,7 +1072,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define INSTPM_FORCE_ORDERING (1 << 7) /* GEN6+ */ #define INSTPM_TLB_INVALIDATE (1 << 9) #define INSTPM_SYNC_FLUSH (1 << 5) -#define ACTHD(base) _MMIO((base) + 0xc8) #define MEM_MODE _MMIO(0x20cc) #define MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1 << 3) /* 830 only */ #define MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1 << 2) /* 830/845 only */ @@ -3115,132 +1198,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define MI_AGPBUSY_INT_EN (1 << 1) /* 85x only */ #define MI_AGPBUSY_830_MODE (1 << 0) /* 85x only */ -#define CACHE_MODE_0 _MMIO(0x2120) /* 915+ only */ -#define CM0_PIPELINED_RENDER_FLUSH_DISABLE (1 << 8) -#define CM0_IZ_OPT_DISABLE (1 << 6) -#define CM0_ZR_OPT_DISABLE (1 << 5) -#define CM0_STC_EVICT_DISABLE_LRA_SNB (1 << 5) -#define CM0_DEPTH_EVICT_DISABLE (1 << 4) -#define CM0_COLOR_EVICT_DISABLE (1 << 3) -#define CM0_DEPTH_WRITE_DISABLE (1 << 1) -#define CM0_RC_OP_FLUSH_DISABLE (1 << 0) -#define GFX_FLSH_CNTL _MMIO(0x2170) /* 915+ only */ -#define GFX_FLSH_CNTL_GEN6 _MMIO(0x101008) -#define GFX_FLSH_CNTL_EN (1 << 0) -#define ECOSKPD _MMIO(0x21d0) -#define ECO_CONSTANT_BUFFER_SR_DISABLE REG_BIT(4) -#define ECO_GATING_CX_ONLY (1 << 3) -#define ECO_FLIP_DONE (1 << 0) - -#define CACHE_MODE_0_GEN7 _MMIO(0x7000) /* IVB+ */ -#define RC_OP_FLUSH_ENABLE (1 << 0) -#define HIZ_RAW_STALL_OPT_DISABLE (1 << 2) -#define CACHE_MODE_1 _MMIO(0x7004) /* IVB+ */ -#define PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1 << 6) -#define GEN8_4x4_STC_OPTIMIZATION_DISABLE (1 << 6) -#define GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE (1 << 1) - -#define GEN6_BLITTER_ECOSKPD _MMIO(0x221d0) -#define GEN6_BLITTER_LOCK_SHIFT 16 -#define GEN6_BLITTER_FBC_NOTIFY (1 << 3) - -#define GEN6_RC_SLEEP_PSMI_CONTROL _MMIO(0x2050) -#define GEN6_PSMI_SLEEP_MSG_DISABLE (1 << 0) -#define GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE REG_BIT(7) -#define GEN8_RC_SEMA_IDLE_MSG_DISABLE (1 << 12) -#define GEN8_FF_DOP_CLOCK_GATE_DISABLE (1 << 10) - -#define GEN6_RCS_PWR_FSM _MMIO(0x22ac) -#define GEN9_RCS_FE_FSM2 _MMIO(0x22a4) - -#define GEN10_CACHE_MODE_SS _MMIO(0xe420) -#define ENABLE_PREFETCH_INTO_IC REG_BIT(3) -#define FLOAT_BLEND_OPTIMIZATION_ENABLE REG_BIT(4) - -/* Fuse readout registers for GT */ -#define HSW_PAVP_FUSE1 _MMIO(0x911C) -#define XEHP_SFC_ENABLE_MASK REG_GENMASK(27, 24) -#define HSW_F1_EU_DIS_MASK REG_GENMASK(17, 16) -#define HSW_F1_EU_DIS_10EUS 0 -#define HSW_F1_EU_DIS_8EUS 1 -#define HSW_F1_EU_DIS_6EUS 2 - -#define CHV_FUSE_GT _MMIO(VLV_DISPLAY_BASE + 0x2168) -#define CHV_FGT_DISABLE_SS0 (1 << 10) -#define CHV_FGT_DISABLE_SS1 (1 << 11) -#define CHV_FGT_EU_DIS_SS0_R0_SHIFT 16 -#define CHV_FGT_EU_DIS_SS0_R0_MASK (0xf << CHV_FGT_EU_DIS_SS0_R0_SHIFT) -#define CHV_FGT_EU_DIS_SS0_R1_SHIFT 20 -#define CHV_FGT_EU_DIS_SS0_R1_MASK (0xf << CHV_FGT_EU_DIS_SS0_R1_SHIFT) -#define CHV_FGT_EU_DIS_SS1_R0_SHIFT 24 -#define CHV_FGT_EU_DIS_SS1_R0_MASK (0xf << CHV_FGT_EU_DIS_SS1_R0_SHIFT) -#define CHV_FGT_EU_DIS_SS1_R1_SHIFT 28 -#define CHV_FGT_EU_DIS_SS1_R1_MASK (0xf << CHV_FGT_EU_DIS_SS1_R1_SHIFT) - -#define GEN8_FUSE2 _MMIO(0x9120) -#define GEN8_F2_SS_DIS_SHIFT 21 -#define GEN8_F2_SS_DIS_MASK (0x7 << GEN8_F2_SS_DIS_SHIFT) -#define GEN8_F2_S_ENA_SHIFT 25 -#define GEN8_F2_S_ENA_MASK (0x7 << GEN8_F2_S_ENA_SHIFT) - -#define GEN9_F2_SS_DIS_SHIFT 20 -#define GEN9_F2_SS_DIS_MASK (0xf << GEN9_F2_SS_DIS_SHIFT) - -#define GEN10_F2_S_ENA_SHIFT 22 -#define GEN10_F2_S_ENA_MASK (0x3f << GEN10_F2_S_ENA_SHIFT) -#define GEN10_F2_SS_DIS_SHIFT 18 -#define GEN10_F2_SS_DIS_MASK (0xf << GEN10_F2_SS_DIS_SHIFT) - -#define GEN10_MIRROR_FUSE3 _MMIO(0x9118) -#define GEN10_L3BANK_PAIR_COUNT 4 -#define GEN10_L3BANK_MASK 0x0F -/* on Xe_HP the same fuses indicates mslices instead of L3 banks */ -#define GEN12_MAX_MSLICES 4 -#define GEN12_MEML3_EN_MASK 0x0F - -#define GEN8_EU_DISABLE0 _MMIO(0x9134) -#define GEN8_EU_DIS0_S0_MASK 0xffffff -#define GEN8_EU_DIS0_S1_SHIFT 24 -#define GEN8_EU_DIS0_S1_MASK (0xff << GEN8_EU_DIS0_S1_SHIFT) - -#define GEN8_EU_DISABLE1 _MMIO(0x9138) -#define GEN8_EU_DIS1_S1_MASK 0xffff -#define GEN8_EU_DIS1_S2_SHIFT 16 -#define GEN8_EU_DIS1_S2_MASK (0xffff << GEN8_EU_DIS1_S2_SHIFT) - -#define GEN8_EU_DISABLE2 _MMIO(0x913c) -#define GEN8_EU_DIS2_S2_MASK 0xff - -#define GEN9_EU_DISABLE(slice) _MMIO(0x9134 + (slice) * 0x4) - -#define GEN10_EU_DISABLE3 _MMIO(0x9140) -#define GEN10_EU_DIS_SS_MASK 0xff - -#define GEN11_GT_VEBOX_VDBOX_DISABLE _MMIO(0x9140) -#define GEN11_GT_VDBOX_DISABLE_MASK 0xff -#define GEN11_GT_VEBOX_DISABLE_SHIFT 16 -#define GEN11_GT_VEBOX_DISABLE_MASK (0x0f << GEN11_GT_VEBOX_DISABLE_SHIFT) - -#define GEN11_EU_DISABLE _MMIO(0x9134) -#define GEN11_EU_DIS_MASK 0xFF - -#define GEN11_GT_SLICE_ENABLE _MMIO(0x9138) -#define GEN11_GT_S_ENA_MASK 0xFF - -#define GEN11_GT_SUBSLICE_DISABLE _MMIO(0x913C) - -#define GEN12_GT_GEOMETRY_DSS_ENABLE _MMIO(0x913C) -#define GEN12_GT_COMPUTE_DSS_ENABLE _MMIO(0x9144) - -#define XEHP_EU_ENABLE _MMIO(0x9134) -#define XEHP_EU_ENA_MASK 0xFF - -#define GEN6_BSD_SLEEP_PSMI_CONTROL _MMIO(0x12050) -#define GEN6_BSD_SLEEP_MSG_DISABLE (1 << 0) -#define GEN6_BSD_SLEEP_FLUSH_DISABLE (1 << 2) -#define GEN6_BSD_SLEEP_INDICATOR (1 << 3) -#define GEN6_BSD_GO_INDICATOR (1 << 4) - /* On modern GEN architectures interrupt control consists of two sets * of registers. The first set pertains to the ring generating the * interrupt. The second control is for the functional block generating the @@ -3398,10 +1355,10 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define FBC_LL_SIZE (1536) /* Framebuffer compression for GM45+ */ -#define DPFC_CB_BASE _MMIO(0x3200) -#define ILK_DPFC_CB_BASE _MMIO(0x43200) -#define DPFC_CONTROL _MMIO(0x3208) -#define ILK_DPFC_CONTROL _MMIO(0x43208) +#define DPFC_CB_BASE _MMIO(0x3200) +#define ILK_DPFC_CB_BASE(fbc_id) _MMIO_PIPE((fbc_id), 0x43200, 0x43240) +#define DPFC_CONTROL _MMIO(0x3208) +#define ILK_DPFC_CONTROL(fbc_id) _MMIO_PIPE((fbc_id), 0x43208, 0x43248) #define DPFC_CTL_EN REG_BIT(31) #define DPFC_CTL_PLANE_MASK_G4X REG_BIT(30) /* g4x-snb */ #define DPFC_CTL_PLANE_G4X(i9xx_plane) REG_FIELD_PREP(DPFC_CTL_PLANE_MASK_G4X, (i9xx_plane)) @@ -3419,28 +1376,28 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define DPFC_CTL_LIMIT_4X REG_FIELD_PREP(DPFC_CTL_LIMIT_MASK, 2) #define DPFC_CTL_FENCENO_MASK REG_GENMASK(3, 0) #define DPFC_CTL_FENCENO(fence) REG_FIELD_PREP(DPFC_CTL_FENCENO_MASK, (fence)) -#define DPFC_RECOMP_CTL _MMIO(0x320c) -#define ILK_DPFC_RECOMP_CTL _MMIO(0x4320c) +#define DPFC_RECOMP_CTL _MMIO(0x320c) +#define ILK_DPFC_RECOMP_CTL(fbc_id) _MMIO_PIPE((fbc_id), 0x4320c, 0x4324c) #define DPFC_RECOMP_STALL_EN REG_BIT(27) #define DPFC_RECOMP_STALL_WM_MASK REG_GENMASK(26, 16) #define DPFC_RECOMP_TIMER_COUNT_MASK REG_GENMASK(5, 0) -#define DPFC_STATUS _MMIO(0x3210) -#define ILK_DPFC_STATUS _MMIO(0x43210) +#define DPFC_STATUS _MMIO(0x3210) +#define ILK_DPFC_STATUS(fbc_id) _MMIO_PIPE((fbc_id), 0x43210, 0x43250) #define DPFC_INVAL_SEG_MASK REG_GENMASK(26, 16) #define DPFC_COMP_SEG_MASK REG_GENMASK(10, 0) -#define DPFC_STATUS2 _MMIO(0x3214) -#define ILK_DPFC_STATUS2 _MMIO(0x43214) +#define DPFC_STATUS2 _MMIO(0x3214) +#define ILK_DPFC_STATUS2(fbc_id) _MMIO_PIPE((fbc_id), 0x43214, 0x43254) #define DPFC_COMP_SEG_MASK_IVB REG_GENMASK(11, 0) -#define DPFC_FENCE_YOFF _MMIO(0x3218) -#define ILK_DPFC_FENCE_YOFF _MMIO(0x43218) -#define DPFC_CHICKEN _MMIO(0x3224) -#define ILK_DPFC_CHICKEN _MMIO(0x43224) +#define DPFC_FENCE_YOFF _MMIO(0x3218) +#define ILK_DPFC_FENCE_YOFF(fbc_id) _MMIO_PIPE((fbc_id), 0x43218, 0x43258) +#define DPFC_CHICKEN _MMIO(0x3224) +#define ILK_DPFC_CHICKEN(fbc_id) _MMIO_PIPE((fbc_id), 0x43224, 0x43264) #define DPFC_HT_MODIFY REG_BIT(31) /* pre-ivb */ #define DPFC_NUKE_ON_ANY_MODIFICATION REG_BIT(23) /* bdw+ */ #define DPFC_CHICKEN_COMP_DUMMY_PIXEL REG_BIT(14) /* glk+ */ #define DPFC_DISABLE_DUMMY0 REG_BIT(8) /* ivb+ */ -#define GLK_FBC_STRIDE _MMIO(0x43228) +#define GLK_FBC_STRIDE(fbc_id) _MMIO_PIPE((fbc_id), 0x43228, 0x43268) #define FBC_STRIDE_OVERRIDE REG_BIT(15) #define FBC_STRIDE_MASK REG_GENMASK(14, 0) #define FBC_STRIDE(x) REG_FIELD_PREP(FBC_STRIDE_MASK, (x)) @@ -3483,9 +1440,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define IPS_CTL _MMIO(0x43408) #define IPS_ENABLE (1 << 31) -#define MSG_FBC_REND_STATE _MMIO(0x50380) +#define MSG_FBC_REND_STATE(fbc_id) _MMIO_PIPE((fbc_id), 0x50380, 0x50384) #define FBC_REND_NUKE REG_BIT(2) -#define FBC_REND_CACHE_CLEAN REG_BIT(1) +#define FBC_REND_CACHE_CLEAN REG_BIT(1) /* * GPIO regs @@ -3874,413 +1831,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) _PALETTE_B, _CHV_PALETTE_C) + \ (i) * 4) -/* MCH MMIO space */ - -/* - * MCHBAR mirror. - * - * This mirrors the MCHBAR MMIO space whose location is determined by - * device 0 function 0's pci config register 0x44 or 0x48 and matches it in - * every way. It is not accessible from the CP register read instructions. - * - * Starting from Haswell, you can't write registers using the MCHBAR mirror, - * just read. - */ -#define MCHBAR_MIRROR_BASE 0x10000 - -#define MCHBAR_MIRROR_BASE_SNB 0x140000 - -#define CTG_STOLEN_RESERVED _MMIO(MCHBAR_MIRROR_BASE + 0x34) -#define ELK_STOLEN_RESERVED _MMIO(MCHBAR_MIRROR_BASE + 0x48) -#define G4X_STOLEN_RESERVED_ADDR1_MASK (0xFFFF << 16) -#define G4X_STOLEN_RESERVED_ADDR2_MASK (0xFFF << 4) -#define G4X_STOLEN_RESERVED_ENABLE (1 << 0) - -/* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */ -#define DCLK _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04) - -/* 915-945 and GM965 MCH register controlling DRAM channel access */ -#define DCC _MMIO(MCHBAR_MIRROR_BASE + 0x200) -#define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0) -#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC (1 << 0) -#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0) -#define DCC_ADDRESSING_MODE_MASK (3 << 0) -#define DCC_CHANNEL_XOR_DISABLE (1 << 10) -#define DCC_CHANNEL_XOR_BIT_17 (1 << 9) -#define DCC2 _MMIO(MCHBAR_MIRROR_BASE + 0x204) -#define DCC2_MODIFIED_ENHANCED_DISABLE (1 << 20) - -/* Pineview MCH register contains DDR3 setting */ -#define CSHRDDR3CTL _MMIO(MCHBAR_MIRROR_BASE + 0x1a8) -#define CSHRDDR3CTL_DDR3 (1 << 2) - -/* 965 MCH register controlling DRAM channel configuration */ -#define C0DRB3_BW _MMIO(MCHBAR_MIRROR_BASE + 0x206) -#define C1DRB3_BW _MMIO(MCHBAR_MIRROR_BASE + 0x606) - -/* snb MCH registers for reading the DRAM channel configuration */ -#define MAD_DIMM_C0 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004) -#define MAD_DIMM_C1 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5008) -#define MAD_DIMM_C2 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C) -#define MAD_DIMM_ECC_MASK (0x3 << 24) -#define MAD_DIMM_ECC_OFF (0x0 << 24) -#define MAD_DIMM_ECC_IO_ON_LOGIC_OFF (0x1 << 24) -#define MAD_DIMM_ECC_IO_OFF_LOGIC_ON (0x2 << 24) -#define MAD_DIMM_ECC_ON (0x3 << 24) -#define MAD_DIMM_ENH_INTERLEAVE (0x1 << 22) -#define MAD_DIMM_RANK_INTERLEAVE (0x1 << 21) -#define MAD_DIMM_B_WIDTH_X16 (0x1 << 20) /* X8 chips if unset */ -#define MAD_DIMM_A_WIDTH_X16 (0x1 << 19) /* X8 chips if unset */ -#define MAD_DIMM_B_DUAL_RANK (0x1 << 18) -#define MAD_DIMM_A_DUAL_RANK (0x1 << 17) -#define MAD_DIMM_A_SELECT (0x1 << 16) -/* DIMM sizes are in multiples of 256mb. */ -#define MAD_DIMM_B_SIZE_SHIFT 8 -#define MAD_DIMM_B_SIZE_MASK (0xff << MAD_DIMM_B_SIZE_SHIFT) -#define MAD_DIMM_A_SIZE_SHIFT 0 -#define MAD_DIMM_A_SIZE_MASK (0xff << MAD_DIMM_A_SIZE_SHIFT) - -/* snb MCH registers for priority tuning */ -#define MCH_SSKPD _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5d10) -#define MCH_SSKPD_WM0_MASK 0x3f -#define MCH_SSKPD_WM0_VAL 0xc - -/* Clocking configuration register */ -#define CLKCFG _MMIO(MCHBAR_MIRROR_BASE + 0xc00) -#define CLKCFG_FSB_400 (0 << 0) /* hrawclk 100 */ -#define CLKCFG_FSB_400_ALT (5 << 0) /* hrawclk 100 */ -#define CLKCFG_FSB_533 (1 << 0) /* hrawclk 133 */ -#define CLKCFG_FSB_667 (3 << 0) /* hrawclk 166 */ -#define CLKCFG_FSB_800 (2 << 0) /* hrawclk 200 */ -#define CLKCFG_FSB_1067 (6 << 0) /* hrawclk 266 */ -#define CLKCFG_FSB_1067_ALT (0 << 0) /* hrawclk 266 */ -#define CLKCFG_FSB_1333 (7 << 0) /* hrawclk 333 */ -#define CLKCFG_FSB_1333_ALT (4 << 0) /* hrawclk 333 */ -#define CLKCFG_FSB_1600_ALT (6 << 0) /* hrawclk 400 */ -#define CLKCFG_FSB_MASK (7 << 0) -#define CLKCFG_MEM_533 (1 << 4) -#define CLKCFG_MEM_667 (2 << 4) -#define CLKCFG_MEM_800 (3 << 4) -#define CLKCFG_MEM_MASK (7 << 4) - -#define HPLLVCO _MMIO(MCHBAR_MIRROR_BASE + 0xc38) -#define HPLLVCO_MOBILE _MMIO(MCHBAR_MIRROR_BASE + 0xc0f) - -#define TSC1 _MMIO(0x11001) -#define TSE (1 << 0) -#define TR1 _MMIO(0x11006) -#define TSFS _MMIO(0x11020) -#define TSFS_SLOPE_MASK 0x0000ff00 -#define TSFS_SLOPE_SHIFT 8 -#define TSFS_INTR_MASK 0x000000ff - -#define CRSTANDVID _MMIO(0x11100) -#define PXVFREQ(fstart) _MMIO(0x11110 + (fstart) * 4) /* P[0-15]VIDFREQ (0x1114c) (Ironlake) */ -#define PXVFREQ_PX_MASK 0x7f000000 -#define PXVFREQ_PX_SHIFT 24 -#define VIDFREQ_BASE _MMIO(0x11110) -#define VIDFREQ1 _MMIO(0x11110) /* VIDFREQ1-4 (0x1111c) (Cantiga) */ -#define VIDFREQ2 _MMIO(0x11114) -#define VIDFREQ3 _MMIO(0x11118) -#define VIDFREQ4 _MMIO(0x1111c) -#define VIDFREQ_P0_MASK 0x1f000000 -#define VIDFREQ_P0_SHIFT 24 -#define VIDFREQ_P0_CSCLK_MASK 0x00f00000 -#define VIDFREQ_P0_CSCLK_SHIFT 20 -#define VIDFREQ_P0_CRCLK_MASK 0x000f0000 -#define VIDFREQ_P0_CRCLK_SHIFT 16 -#define VIDFREQ_P1_MASK 0x00001f00 -#define VIDFREQ_P1_SHIFT 8 -#define VIDFREQ_P1_CSCLK_MASK 0x000000f0 -#define VIDFREQ_P1_CSCLK_SHIFT 4 -#define VIDFREQ_P1_CRCLK_MASK 0x0000000f -#define INTTOEXT_BASE_ILK _MMIO(0x11300) -#define INTTOEXT_BASE _MMIO(0x11120) /* INTTOEXT1-8 (0x1113c) */ -#define INTTOEXT_MAP3_SHIFT 24 -#define INTTOEXT_MAP3_MASK (0x1f << INTTOEXT_MAP3_SHIFT) -#define INTTOEXT_MAP2_SHIFT 16 -#define INTTOEXT_MAP2_MASK (0x1f << INTTOEXT_MAP2_SHIFT) -#define INTTOEXT_MAP1_SHIFT 8 -#define INTTOEXT_MAP1_MASK (0x1f << INTTOEXT_MAP1_SHIFT) -#define INTTOEXT_MAP0_SHIFT 0 -#define INTTOEXT_MAP0_MASK (0x1f << INTTOEXT_MAP0_SHIFT) -#define MEMSWCTL _MMIO(0x11170) /* Ironlake only */ -#define MEMCTL_CMD_MASK 0xe000 -#define MEMCTL_CMD_SHIFT 13 -#define MEMCTL_CMD_RCLK_OFF 0 -#define MEMCTL_CMD_RCLK_ON 1 -#define MEMCTL_CMD_CHFREQ 2 -#define MEMCTL_CMD_CHVID 3 -#define MEMCTL_CMD_VMMOFF 4 -#define MEMCTL_CMD_VMMON 5 -#define MEMCTL_CMD_STS (1 << 12) /* write 1 triggers command, clears - when command complete */ -#define MEMCTL_FREQ_MASK 0x0f00 /* jitter, from 0-15 */ -#define MEMCTL_FREQ_SHIFT 8 -#define MEMCTL_SFCAVM (1 << 7) -#define MEMCTL_TGT_VID_MASK 0x007f -#define MEMIHYST _MMIO(0x1117c) -#define MEMINTREN _MMIO(0x11180) /* 16 bits */ -#define MEMINT_RSEXIT_EN (1 << 8) -#define MEMINT_CX_SUPR_EN (1 << 7) -#define MEMINT_CONT_BUSY_EN (1 << 6) -#define MEMINT_AVG_BUSY_EN (1 << 5) -#define MEMINT_EVAL_CHG_EN (1 << 4) -#define MEMINT_MON_IDLE_EN (1 << 3) -#define MEMINT_UP_EVAL_EN (1 << 2) -#define MEMINT_DOWN_EVAL_EN (1 << 1) -#define MEMINT_SW_CMD_EN (1 << 0) -#define MEMINTRSTR _MMIO(0x11182) /* 16 bits */ -#define MEM_RSEXIT_MASK 0xc000 -#define MEM_RSEXIT_SHIFT 14 -#define MEM_CONT_BUSY_MASK 0x3000 -#define MEM_CONT_BUSY_SHIFT 12 -#define MEM_AVG_BUSY_MASK 0x0c00 -#define MEM_AVG_BUSY_SHIFT 10 -#define MEM_EVAL_CHG_MASK 0x0300 -#define MEM_EVAL_BUSY_SHIFT 8 -#define MEM_MON_IDLE_MASK 0x00c0 -#define MEM_MON_IDLE_SHIFT 6 -#define MEM_UP_EVAL_MASK 0x0030 -#define MEM_UP_EVAL_SHIFT 4 -#define MEM_DOWN_EVAL_MASK 0x000c -#define MEM_DOWN_EVAL_SHIFT 2 -#define MEM_SW_CMD_MASK 0x0003 -#define MEM_INT_STEER_GFX 0 -#define MEM_INT_STEER_CMR 1 -#define MEM_INT_STEER_SMI 2 -#define MEM_INT_STEER_SCI 3 -#define MEMINTRSTS _MMIO(0x11184) -#define MEMINT_RSEXIT (1 << 7) -#define MEMINT_CONT_BUSY (1 << 6) -#define MEMINT_AVG_BUSY (1 << 5) -#define MEMINT_EVAL_CHG (1 << 4) -#define MEMINT_MON_IDLE (1 << 3) -#define MEMINT_UP_EVAL (1 << 2) -#define MEMINT_DOWN_EVAL (1 << 1) -#define MEMINT_SW_CMD (1 << 0) -#define MEMMODECTL _MMIO(0x11190) -#define MEMMODE_BOOST_EN (1 << 31) -#define MEMMODE_BOOST_FREQ_MASK 0x0f000000 /* jitter for boost, 0-15 */ -#define MEMMODE_BOOST_FREQ_SHIFT 24 -#define MEMMODE_IDLE_MODE_MASK 0x00030000 -#define MEMMODE_IDLE_MODE_SHIFT 16 -#define MEMMODE_IDLE_MODE_EVAL 0 -#define MEMMODE_IDLE_MODE_CONT 1 -#define MEMMODE_HWIDLE_EN (1 << 15) -#define MEMMODE_SWMODE_EN (1 << 14) -#define MEMMODE_RCLK_GATE (1 << 13) -#define MEMMODE_HW_UPDATE (1 << 12) -#define MEMMODE_FSTART_MASK 0x00000f00 /* starting jitter, 0-15 */ -#define MEMMODE_FSTART_SHIFT 8 -#define MEMMODE_FMAX_MASK 0x000000f0 /* max jitter, 0-15 */ -#define MEMMODE_FMAX_SHIFT 4 -#define MEMMODE_FMIN_MASK 0x0000000f /* min jitter, 0-15 */ -#define RCBMAXAVG _MMIO(0x1119c) -#define MEMSWCTL2 _MMIO(0x1119e) /* Cantiga only */ -#define SWMEMCMD_RENDER_OFF (0 << 13) -#define SWMEMCMD_RENDER_ON (1 << 13) -#define SWMEMCMD_SWFREQ (2 << 13) -#define SWMEMCMD_TARVID (3 << 13) -#define SWMEMCMD_VRM_OFF (4 << 13) -#define SWMEMCMD_VRM_ON (5 << 13) -#define CMDSTS (1 << 12) -#define SFCAVM (1 << 11) -#define SWFREQ_MASK 0x0380 /* P0-7 */ -#define SWFREQ_SHIFT 7 -#define TARVID_MASK 0x001f -#define MEMSTAT_CTG _MMIO(0x111a0) -#define RCBMINAVG _MMIO(0x111a0) -#define RCUPEI _MMIO(0x111b0) -#define RCDNEI _MMIO(0x111b4) -#define RSTDBYCTL _MMIO(0x111b8) -#define RS1EN (1 << 31) -#define RS2EN (1 << 30) -#define RS3EN (1 << 29) -#define D3RS3EN (1 << 28) /* Display D3 imlies RS3 */ -#define SWPROMORSX (1 << 27) /* RSx promotion timers ignored */ -#define RCWAKERW (1 << 26) /* Resetwarn from PCH causes wakeup */ -#define DPRSLPVREN (1 << 25) /* Fast voltage ramp enable */ -#define GFXTGHYST (1 << 24) /* Hysteresis to allow trunk gating */ -#define RCX_SW_EXIT (1 << 23) /* Leave RSx and prevent re-entry */ -#define RSX_STATUS_MASK (7 << 20) -#define RSX_STATUS_ON (0 << 20) -#define RSX_STATUS_RC1 (1 << 20) -#define RSX_STATUS_RC1E (2 << 20) -#define RSX_STATUS_RS1 (3 << 20) -#define RSX_STATUS_RS2 (4 << 20) /* aka rc6 */ -#define RSX_STATUS_RSVD (5 << 20) /* deep rc6 unsupported on ilk */ -#define RSX_STATUS_RS3 (6 << 20) /* rs3 unsupported on ilk */ -#define RSX_STATUS_RSVD2 (7 << 20) -#define UWRCRSXE (1 << 19) /* wake counter limit prevents rsx */ -#define RSCRP (1 << 18) /* rs requests control on rs1/2 reqs */ -#define JRSC (1 << 17) /* rsx coupled to cpu c-state */ -#define RS2INC0 (1 << 16) /* allow rs2 in cpu c0 */ -#define RS1CONTSAV_MASK (3 << 14) -#define RS1CONTSAV_NO_RS1 (0 << 14) /* rs1 doesn't save/restore context */ -#define RS1CONTSAV_RSVD (1 << 14) -#define RS1CONTSAV_SAVE_RS1 (2 << 14) /* rs1 saves context */ -#define RS1CONTSAV_FULL_RS1 (3 << 14) /* rs1 saves and restores context */ -#define NORMSLEXLAT_MASK (3 << 12) -#define SLOW_RS123 (0 << 12) -#define SLOW_RS23 (1 << 12) -#define SLOW_RS3 (2 << 12) -#define NORMAL_RS123 (3 << 12) -#define RCMODE_TIMEOUT (1 << 11) /* 0 is eval interval method */ -#define IMPROMOEN (1 << 10) /* promo is immediate or delayed until next idle interval (only for timeout method above) */ -#define RCENTSYNC (1 << 9) /* rs coupled to cpu c-state (3/6/7) */ -#define STATELOCK (1 << 7) /* locked to rs_cstate if 0 */ -#define RS_CSTATE_MASK (3 << 4) -#define RS_CSTATE_C367_RS1 (0 << 4) -#define RS_CSTATE_C36_RS1_C7_RS2 (1 << 4) -#define RS_CSTATE_RSVD (2 << 4) -#define RS_CSTATE_C367_RS2 (3 << 4) -#define REDSAVES (1 << 3) /* no context save if was idle during rs0 */ -#define REDRESTORES (1 << 2) /* no restore if was idle during rs0 */ -#define VIDCTL _MMIO(0x111c0) -#define VIDSTS _MMIO(0x111c8) -#define VIDSTART _MMIO(0x111cc) /* 8 bits */ -#define MEMSTAT_ILK _MMIO(0x111f8) -#define MEMSTAT_VID_MASK 0x7f00 -#define MEMSTAT_VID_SHIFT 8 -#define MEMSTAT_PSTATE_MASK 0x00f8 -#define MEMSTAT_PSTATE_SHIFT 3 -#define MEMSTAT_MON_ACTV (1 << 2) -#define MEMSTAT_SRC_CTL_MASK 0x0003 -#define MEMSTAT_SRC_CTL_CORE 0 -#define MEMSTAT_SRC_CTL_TRB 1 -#define MEMSTAT_SRC_CTL_THM 2 -#define MEMSTAT_SRC_CTL_STDBY 3 -#define RCPREVBSYTUPAVG _MMIO(0x113b8) -#define RCPREVBSYTDNAVG _MMIO(0x113bc) -#define PMMISC _MMIO(0x11214) -#define MCPPCE_EN (1 << 0) /* enable PM_MSG from PCH->MPC */ -#define SDEW _MMIO(0x1124c) -#define CSIEW0 _MMIO(0x11250) -#define CSIEW1 _MMIO(0x11254) -#define CSIEW2 _MMIO(0x11258) -#define PEW(i) _MMIO(0x1125c + (i) * 4) /* 5 registers */ -#define DEW(i) _MMIO(0x11270 + (i) * 4) /* 3 registers */ -#define MCHAFE _MMIO(0x112c0) -#define CSIEC _MMIO(0x112e0) -#define DMIEC _MMIO(0x112e4) -#define DDREC _MMIO(0x112e8) -#define PEG0EC _MMIO(0x112ec) -#define PEG1EC _MMIO(0x112f0) -#define GFXEC _MMIO(0x112f4) -#define RPPREVBSYTUPAVG _MMIO(0x113b8) -#define RPPREVBSYTDNAVG _MMIO(0x113bc) -#define ECR _MMIO(0x11600) -#define ECR_GPFE (1 << 31) -#define ECR_IMONE (1 << 30) -#define ECR_CAP_MASK 0x0000001f /* Event range, 0-31 */ -#define OGW0 _MMIO(0x11608) -#define OGW1 _MMIO(0x1160c) -#define EG0 _MMIO(0x11610) -#define EG1 _MMIO(0x11614) -#define EG2 _MMIO(0x11618) -#define EG3 _MMIO(0x1161c) -#define EG4 _MMIO(0x11620) -#define EG5 _MMIO(0x11624) -#define EG6 _MMIO(0x11628) -#define EG7 _MMIO(0x1162c) -#define PXW(i) _MMIO(0x11664 + (i) * 4) /* 4 registers */ -#define PXWL(i) _MMIO(0x11680 + (i) * 8) /* 8 registers */ -#define LCFUSE02 _MMIO(0x116c0) -#define LCFUSE_HIV_MASK 0x000000ff -#define CSIPLL0 _MMIO(0x12c10) -#define DDRMPLL1 _MMIO(0X12c20) #define PEG_BAND_GAP_DATA _MMIO(0x14d68) -#define GEN6_GT_THREAD_STATUS_REG _MMIO(0x13805c) -#define GEN6_GT_THREAD_STATUS_CORE_MASK 0x7 - -#define GEN6_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5948) -#define BXT_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7070) -#define GEN6_RP_STATE_LIMITS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994) -#define GEN6_RP_STATE_CAP _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998) -#define RP0_CAP_MASK REG_GENMASK(7, 0) -#define RP1_CAP_MASK REG_GENMASK(15, 8) -#define RPN_CAP_MASK REG_GENMASK(23, 16) #define BXT_RP_STATE_CAP _MMIO(0x138170) #define GEN9_RP_STATE_LIMITS _MMIO(0x138148) #define XEHPSDV_RP_STATE_CAP _MMIO(0x250014) -/* - * Logical Context regs - */ -#define CCID(base) _MMIO((base) + 0x180) -#define CCID_EN BIT(0) -#define CCID_EXTENDED_STATE_RESTORE BIT(2) -#define CCID_EXTENDED_STATE_SAVE BIT(3) -/* - * Notes on SNB/IVB/VLV context size: - * - Power context is saved elsewhere (LLC or stolen) - * - Ring/execlist context is saved on SNB, not on IVB - * - Extended context size already includes render context size - * - We always need to follow the extended context size. - * SNB BSpec has comments indicating that we should use the - * render context size instead if execlists are disabled, but - * based on empirical testing that's just nonsense. - * - Pipelined/VF state is saved on SNB/IVB respectively - * - GT1 size just indicates how much of render context - * doesn't need saving on GT1 - */ -#define CXT_SIZE _MMIO(0x21a0) -#define GEN6_CXT_POWER_SIZE(cxt_reg) (((cxt_reg) >> 24) & 0x3f) -#define GEN6_CXT_RING_SIZE(cxt_reg) (((cxt_reg) >> 18) & 0x3f) -#define GEN6_CXT_RENDER_SIZE(cxt_reg) (((cxt_reg) >> 12) & 0x3f) -#define GEN6_CXT_EXTENDED_SIZE(cxt_reg) (((cxt_reg) >> 6) & 0x3f) -#define GEN6_CXT_PIPELINE_SIZE(cxt_reg) (((cxt_reg) >> 0) & 0x3f) -#define GEN6_CXT_TOTAL_SIZE(cxt_reg) (GEN6_CXT_RING_SIZE(cxt_reg) + \ - GEN6_CXT_EXTENDED_SIZE(cxt_reg) + \ - GEN6_CXT_PIPELINE_SIZE(cxt_reg)) -#define GEN7_CXT_SIZE _MMIO(0x21a8) -#define GEN7_CXT_POWER_SIZE(ctx_reg) (((ctx_reg) >> 25) & 0x7f) -#define GEN7_CXT_RING_SIZE(ctx_reg) (((ctx_reg) >> 22) & 0x7) -#define GEN7_CXT_RENDER_SIZE(ctx_reg) (((ctx_reg) >> 16) & 0x3f) -#define GEN7_CXT_EXTENDED_SIZE(ctx_reg) (((ctx_reg) >> 9) & 0x7f) -#define GEN7_CXT_GT1_SIZE(ctx_reg) (((ctx_reg) >> 6) & 0x7) -#define GEN7_CXT_VFSTATE_SIZE(ctx_reg) (((ctx_reg) >> 0) & 0x3f) -#define GEN7_CXT_TOTAL_SIZE(ctx_reg) (GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \ - GEN7_CXT_VFSTATE_SIZE(ctx_reg)) - -enum { - INTEL_ADVANCED_CONTEXT = 0, - INTEL_LEGACY_32B_CONTEXT, - INTEL_ADVANCED_AD_CONTEXT, - INTEL_LEGACY_64B_CONTEXT -}; - -enum { - FAULT_AND_HANG = 0, - FAULT_AND_HALT, /* Debug only */ - FAULT_AND_STREAM, - FAULT_AND_CONTINUE /* Unsupported */ -}; - -#define CTX_GTT_ADDRESS_MASK GENMASK(31, 12) -#define GEN8_CTX_VALID (1 << 0) -#define GEN8_CTX_FORCE_PD_RESTORE (1 << 1) -#define GEN8_CTX_FORCE_RESTORE (1 << 2) -#define GEN8_CTX_L3LLC_COHERENT (1 << 5) -#define GEN8_CTX_PRIVILEGE (1 << 8) -#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 - -#define GEN8_CTX_ID_SHIFT 32 -#define GEN8_CTX_ID_WIDTH 21 -#define GEN11_SW_CTX_ID_SHIFT 37 -#define GEN11_SW_CTX_ID_WIDTH 11 -#define GEN11_ENGINE_CLASS_SHIFT 61 -#define GEN11_ENGINE_CLASS_WIDTH 3 -#define GEN11_ENGINE_INSTANCE_SHIFT 48 -#define GEN11_ENGINE_INSTANCE_WIDTH 6 - -#define XEHP_SW_CTX_ID_SHIFT 39 -#define XEHP_SW_CTX_ID_WIDTH 16 -#define XEHP_SW_COUNTER_SHIFT 58 -#define XEHP_SW_COUNTER_WIDTH 6 - #define CHV_CLK_CTL1 _MMIO(0x101100) #define VLV_CLK_CTL2 _MMIO(0x101104) #define CLK_CTL2_CZCOUNT_30NS_SHIFT 28 @@ -4332,75 +1888,6 @@ enum { _MMIO_PIPE(pipe, _CLKGATE_DIS_PSL_A, _CLKGATE_DIS_PSL_B) /* - * GEN10 clock gating regs - */ - -#define UNSLCGCTL9440 _MMIO(0x9440) -#define GAMTLBOACS_CLKGATE_DIS REG_BIT(28) -#define GAMTLBVDBOX5_CLKGATE_DIS REG_BIT(27) -#define GAMTLBVDBOX6_CLKGATE_DIS REG_BIT(26) -#define GAMTLBVDBOX3_CLKGATE_DIS REG_BIT(24) -#define GAMTLBVDBOX4_CLKGATE_DIS REG_BIT(23) -#define GAMTLBVDBOX7_CLKGATE_DIS REG_BIT(22) -#define GAMTLBVDBOX2_CLKGATE_DIS REG_BIT(21) -#define GAMTLBVDBOX0_CLKGATE_DIS REG_BIT(17) -#define GAMTLBKCR_CLKGATE_DIS REG_BIT(16) -#define GAMTLBGUC_CLKGATE_DIS REG_BIT(15) -#define GAMTLBBLT_CLKGATE_DIS REG_BIT(14) -#define GAMTLBVDBOX1_CLKGATE_DIS REG_BIT(6) - -#define UNSLCGCTL9444 _MMIO(0x9444) -#define GAMTLBGFXA0_CLKGATE_DIS REG_BIT(30) -#define GAMTLBGFXA1_CLKGATE_DIS REG_BIT(29) -#define GAMTLBCOMPA0_CLKGATE_DIS REG_BIT(28) -#define GAMTLBCOMPA1_CLKGATE_DIS REG_BIT(27) -#define GAMTLBCOMPB0_CLKGATE_DIS REG_BIT(26) -#define GAMTLBCOMPB1_CLKGATE_DIS REG_BIT(25) -#define GAMTLBCOMPC0_CLKGATE_DIS REG_BIT(24) -#define GAMTLBCOMPC1_CLKGATE_DIS REG_BIT(23) -#define GAMTLBCOMPD0_CLKGATE_DIS REG_BIT(22) -#define GAMTLBCOMPD1_CLKGATE_DIS REG_BIT(21) -#define GAMTLBMERT_CLKGATE_DIS REG_BIT(20) -#define GAMTLBVEBOX3_CLKGATE_DIS REG_BIT(19) -#define GAMTLBVEBOX2_CLKGATE_DIS REG_BIT(18) -#define GAMTLBVEBOX1_CLKGATE_DIS REG_BIT(17) -#define GAMTLBVEBOX0_CLKGATE_DIS REG_BIT(16) -#define LTCDD_CLKGATE_DIS REG_BIT(10) - -#define SLICE_UNIT_LEVEL_CLKGATE _MMIO(0x94d4) -#define SARBUNIT_CLKGATE_DIS (1 << 5) -#define RCCUNIT_CLKGATE_DIS (1 << 7) -#define MSCUNIT_CLKGATE_DIS (1 << 10) -#define NODEDSS_CLKGATE_DIS REG_BIT(12) -#define L3_CLKGATE_DIS REG_BIT(16) -#define L3_CR2X_CLKGATE_DIS REG_BIT(17) - -#define SUBSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9524) -#define DSS_ROUTER_CLKGATE_DIS REG_BIT(28) -#define GWUNIT_CLKGATE_DIS REG_BIT(16) - -#define SUBSLICE_UNIT_LEVEL_CLKGATE2 _MMIO(0x9528) -#define CPSSUNIT_CLKGATE_DIS REG_BIT(9) - -#define SSMCGCTL9530 _MMIO(0x9530) -#define RTFUNIT_CLKGATE_DIS REG_BIT(18) - -#define UNSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9434) -#define VFUNIT_CLKGATE_DIS REG_BIT(20) -#define TSGUNIT_CLKGATE_DIS REG_BIT(17) /* XEHPSDV */ -#define CG3DDISCFEG_CLKGATE_DIS REG_BIT(17) /* DG2 */ -#define GAMEDIA_CLKGATE_DIS REG_BIT(11) -#define HSUNIT_CLKGATE_DIS REG_BIT(8) -#define VSUNIT_CLKGATE_DIS REG_BIT(3) - -#define UNSLICE_UNIT_LEVEL_CLKGATE2 _MMIO(0x94e4) -#define VSUNIT_CLKGATE_DIS_TGL REG_BIT(19) -#define PSDUNIT_CLKGATE_DIS REG_BIT(5) - -#define INF_UNIT_LEVEL_CLKGATE _MMIO(0x9560) -#define CGPSF_CLKGATE_DIS (1 << 3) - -/* * Display engine regs */ @@ -4494,6 +1981,10 @@ enum { #define _VSYNC_A 0x60014 #define _EXITLINE_A 0x60018 #define _PIPEASRC 0x6001c +#define PIPESRC_WIDTH_MASK REG_GENMASK(31, 16) +#define PIPESRC_WIDTH(w) REG_FIELD_PREP(PIPESRC_WIDTH_MASK, (w)) +#define PIPESRC_HEIGHT_MASK REG_GENMASK(15, 0) +#define PIPESRC_HEIGHT(h) REG_FIELD_PREP(PIPESRC_HEIGHT_MASK, (h)) #define _BCLRPAT_A 0x60020 #define _VSYNCSHIFT_A 0x60028 #define _PIPE_MULT_A 0x6002c @@ -6145,16 +3636,14 @@ enum { #define _PIPEB_DATA_M_G4X 0x71050 /* Transfer unit size for display port - 1, default is 0x3f (for TU size 64) */ -#define TU_SIZE(x) (((x) - 1) << 25) /* default size 64 */ -#define TU_SIZE_SHIFT 25 -#define TU_SIZE_MASK (0x3f << 25) +#define TU_SIZE_MASK REG_GENMASK(30, 25) +#define TU_SIZE(x) REG_FIELD_PREP(TU_SIZE_MASK, (x) - 1) /* default size 64 */ -#define DATA_LINK_M_N_MASK (0xffffff) +#define DATA_LINK_M_N_MASK REG_GENMASK(23, 0) #define DATA_LINK_N_MAX (0x800000) #define _PIPEA_DATA_N_G4X 0x70054 #define _PIPEB_DATA_N_G4X 0x71054 -#define PIPE_GMCH_DATA_N_MASK (0xffffff) /* * Computing Link M and N values for the Display Port link @@ -6169,11 +3658,8 @@ enum { #define _PIPEA_LINK_M_G4X 0x70060 #define _PIPEB_LINK_M_G4X 0x71060 -#define PIPEA_DP_LINK_M_MASK (0xffffff) - #define _PIPEA_LINK_N_G4X 0x70064 #define _PIPEB_LINK_N_G4X 0x71064 -#define PIPEA_DP_LINK_N_MASK (0xffffff) #define PIPE_DATA_M_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_DATA_M_G4X, _PIPEB_DATA_M_G4X) #define PIPE_DATA_N_G4X(pipe) _MMIO_PIPE(pipe, _PIPEA_DATA_N_G4X, _PIPEB_DATA_N_G4X) @@ -6184,65 +3670,61 @@ enum { /* Pipe A */ #define _PIPEADSL 0x70000 -#define DSL_LINEMASK_GEN2 0x00000fff -#define DSL_LINEMASK_GEN3 0x00001fff +#define PIPEDSL_CURR_FIELD REG_BIT(31) /* ctg+ */ +#define PIPEDSL_LINE_MASK REG_GENMASK(19, 0) #define _PIPEACONF 0x70008 -#define PIPECONF_ENABLE (1 << 31) -#define PIPECONF_DISABLE 0 -#define PIPECONF_DOUBLE_WIDE (1 << 30) -#define I965_PIPECONF_ACTIVE (1 << 30) -#define PIPECONF_DSI_PLL_LOCKED (1 << 29) /* vlv & pipe A only */ -#define PIPECONF_FRAME_START_DELAY_MASK (3 << 27) /* pre-hsw */ -#define PIPECONF_FRAME_START_DELAY(x) ((x) << 27) /* pre-hsw: 0-3 */ -#define PIPECONF_SINGLE_WIDE 0 -#define PIPECONF_PIPE_UNLOCKED 0 -#define PIPECONF_PIPE_LOCKED (1 << 25) -#define PIPECONF_FORCE_BORDER (1 << 25) -#define PIPECONF_GAMMA_MODE_MASK_I9XX (1 << 24) /* gmch */ -#define PIPECONF_GAMMA_MODE_MASK_ILK (3 << 24) /* ilk-ivb */ -#define PIPECONF_GAMMA_MODE_8BIT (0 << 24) /* gmch,ilk-ivb */ -#define PIPECONF_GAMMA_MODE_10BIT (1 << 24) /* gmch,ilk-ivb */ -#define PIPECONF_GAMMA_MODE_12BIT (2 << 24) /* ilk-ivb */ -#define PIPECONF_GAMMA_MODE_SPLIT (3 << 24) /* ivb */ -#define PIPECONF_GAMMA_MODE(x) ((x) << 24) /* pass in GAMMA_MODE_MODE_* */ -#define PIPECONF_GAMMA_MODE_SHIFT 24 -#define PIPECONF_INTERLACE_MASK (7 << 21) -#define PIPECONF_INTERLACE_MASK_HSW (3 << 21) -/* Note that pre-gen3 does not support interlaced display directly. Panel - * fitting must be disabled on pre-ilk for interlaced. */ -#define PIPECONF_PROGRESSIVE (0 << 21) -#define PIPECONF_INTERLACE_W_SYNC_SHIFT_PANEL (4 << 21) /* gen4 only */ -#define PIPECONF_INTERLACE_W_SYNC_SHIFT (5 << 21) /* gen4 only */ -#define PIPECONF_INTERLACE_W_FIELD_INDICATION (6 << 21) -#define PIPECONF_INTERLACE_FIELD_0_ONLY (7 << 21) /* gen3 only */ -/* Ironlake and later have a complete new set of values for interlaced. PFIT - * means panel fitter required, PF means progressive fetch, DBL means power - * saving pixel doubling. */ -#define PIPECONF_PFIT_PF_INTERLACED_ILK (1 << 21) -#define PIPECONF_INTERLACED_ILK (3 << 21) -#define PIPECONF_INTERLACED_DBL_ILK (4 << 21) /* ilk/snb only */ -#define PIPECONF_PFIT_PF_INTERLACED_DBL_ILK (5 << 21) /* ilk/snb only */ -#define PIPECONF_INTERLACE_MODE_MASK (7 << 21) -#define PIPECONF_EDP_RR_MODE_SWITCH (1 << 20) -#define PIPECONF_CXSR_DOWNCLOCK (1 << 16) -#define PIPECONF_EDP_RR_MODE_SWITCH_VLV (1 << 14) -#define PIPECONF_COLOR_RANGE_SELECT (1 << 13) -#define PIPECONF_OUTPUT_COLORSPACE_MASK (3 << 11) /* ilk-ivb */ -#define PIPECONF_OUTPUT_COLORSPACE_RGB (0 << 11) /* ilk-ivb */ -#define PIPECONF_OUTPUT_COLORSPACE_YUV601 (1 << 11) /* ilk-ivb */ -#define PIPECONF_OUTPUT_COLORSPACE_YUV709 (2 << 11) /* ilk-ivb */ -#define PIPECONF_OUTPUT_COLORSPACE_YUV_HSW (1 << 11) /* hsw only */ -#define PIPECONF_BPC_MASK (0x7 << 5) -#define PIPECONF_8BPC (0 << 5) -#define PIPECONF_10BPC (1 << 5) -#define PIPECONF_6BPC (2 << 5) -#define PIPECONF_12BPC (3 << 5) -#define PIPECONF_DITHER_EN (1 << 4) -#define PIPECONF_DITHER_TYPE_MASK (0x0000000c) -#define PIPECONF_DITHER_TYPE_SP (0 << 2) -#define PIPECONF_DITHER_TYPE_ST1 (1 << 2) -#define PIPECONF_DITHER_TYPE_ST2 (2 << 2) -#define PIPECONF_DITHER_TYPE_TEMP (3 << 2) +#define PIPECONF_ENABLE REG_BIT(31) +#define PIPECONF_DOUBLE_WIDE REG_BIT(30) /* pre-i965 */ +#define PIPECONF_STATE_ENABLE REG_BIT(30) /* i965+ */ +#define PIPECONF_DSI_PLL_LOCKED REG_BIT(29) /* vlv & pipe A only */ +#define PIPECONF_FRAME_START_DELAY_MASK REG_GENMASK(28, 27) /* pre-hsw */ +#define PIPECONF_FRAME_START_DELAY(x) REG_FIELD_PREP(PIPECONF_FRAME_START_DELAY_MASK, (x)) /* pre-hsw: 0-3 */ +#define PIPECONF_PIPE_LOCKED REG_BIT(25) +#define PIPECONF_FORCE_BORDER REG_BIT(25) +#define PIPECONF_GAMMA_MODE_MASK_I9XX REG_BIT(24) /* gmch */ +#define PIPECONF_GAMMA_MODE_MASK_ILK REG_GENMASK(25, 24) /* ilk-ivb */ +#define PIPECONF_GAMMA_MODE_8BIT REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK, 0) +#define PIPECONF_GAMMA_MODE_10BIT REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK, 1) +#define PIPECONF_GAMMA_MODE_12BIT REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK_ILK, 2) /* ilk-ivb */ +#define PIPECONF_GAMMA_MODE_SPLIT REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK_ILK, 3) /* ivb */ +#define PIPECONF_GAMMA_MODE(x) REG_FIELD_PREP(PIPECONF_GAMMA_MODE_MASK_ILK, (x)) /* pass in GAMMA_MODE_MODE_* */ +#define PIPECONF_INTERLACE_MASK REG_GENMASK(23, 21) /* gen3+ */ +#define PIPECONF_INTERLACE_PROGRESSIVE REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 0) +#define PIPECONF_INTERLACE_W_SYNC_SHIFT_PANEL REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 4) /* gen4 only */ +#define PIPECONF_INTERLACE_W_SYNC_SHIFT REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 5) /* gen4 only */ +#define PIPECONF_INTERLACE_W_FIELD_INDICATION REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 6) +#define PIPECONF_INTERLACE_FIELD_0_ONLY REG_FIELD_PREP(PIPECONF_INTERLACE_MASK, 7) /* gen3 only */ +/* + * ilk+: PF/D=progressive fetch/display, IF/D=interlaced fetch/display, + * DBL=power saving pixel doubling, PF-ID* requires panel fitter + */ +#define PIPECONF_INTERLACE_MASK_ILK REG_GENMASK(23, 21) /* ilk+ */ +#define PIPECONF_INTERLACE_MASK_HSW REG_GENMASK(22, 21) /* hsw+ */ +#define PIPECONF_INTERLACE_PF_PD_ILK REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 0) +#define PIPECONF_INTERLACE_PF_ID_ILK REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 1) +#define PIPECONF_INTERLACE_IF_ID_ILK REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 3) +#define PIPECONF_INTERLACE_IF_ID_DBL_ILK REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 4) /* ilk/snb only */ +#define PIPECONF_INTERLACE_PF_ID_DBL_ILK REG_FIELD_PREP(PIPECONF_INTERLACE_MASK_ILK, 5) /* ilk/snb only */ +#define PIPECONF_EDP_RR_MODE_SWITCH REG_BIT(20) +#define PIPECONF_CXSR_DOWNCLOCK REG_BIT(16) +#define PIPECONF_EDP_RR_MODE_SWITCH_VLV REG_BIT(14) +#define PIPECONF_COLOR_RANGE_SELECT REG_BIT(13) +#define PIPECONF_OUTPUT_COLORSPACE_MASK REG_GENMASK(12, 11) /* ilk-ivb */ +#define PIPECONF_OUTPUT_COLORSPACE_RGB REG_FIELD_PREP(PIPECONF_OUTPUT_COLORSPACE_MASK, 0) /* ilk-ivb */ +#define PIPECONF_OUTPUT_COLORSPACE_YUV601 REG_FIELD_PREP(PIPECONF_OUTPUT_COLORSPACE_MASK, 1) /* ilk-ivb */ +#define PIPECONF_OUTPUT_COLORSPACE_YUV709 REG_FIELD_PREP(PIPECONF_OUTPUT_COLORSPACE_MASK, 2) /* ilk-ivb */ +#define PIPECONF_OUTPUT_COLORSPACE_YUV_HSW REG_BIT(11) /* hsw only */ +#define PIPECONF_BPC_MASK REG_GENMASK(7, 5) /* ctg-ivb */ +#define PIPECONF_BPC_8 REG_FIELD_PREP(PIPECONF_BPC_MASK, 0) +#define PIPECONF_BPC_10 REG_FIELD_PREP(PIPECONF_BPC_MASK, 1) +#define PIPECONF_BPC_6 REG_FIELD_PREP(PIPECONF_BPC_MASK, 2) +#define PIPECONF_BPC_12 REG_FIELD_PREP(PIPECONF_BPC_MASK, 3) +#define PIPECONF_DITHER_EN REG_BIT(4) +#define PIPECONF_DITHER_TYPE_MASK REG_GENMASK(3, 2) +#define PIPECONF_DITHER_TYPE_SP REG_FIELD_PREP(PIPECONF_DITHER_TYPE_MASK, 0) +#define PIPECONF_DITHER_TYPE_ST1 REG_FIELD_PREP(PIPECONF_DITHER_TYPE_MASK, 1) +#define PIPECONF_DITHER_TYPE_ST2 REG_FIELD_PREP(PIPECONF_DITHER_TYPE_MASK, 2) +#define PIPECONF_DITHER_TYPE_TEMP REG_FIELD_PREP(PIPECONF_DITHER_TYPE_MASK, 3) #define _PIPEASTAT 0x70024 #define PIPE_FIFO_UNDERRUN_STATUS (1UL << 31) #define SPRITE1_FLIP_DONE_INT_EN_VLV (1UL << 30) @@ -6327,38 +3809,41 @@ enum { #define _PIPE_MISC_A 0x70030 #define _PIPE_MISC_B 0x71030 -#define PIPEMISC_YUV420_ENABLE (1 << 27) /* glk+ */ -#define PIPEMISC_YUV420_MODE_FULL_BLEND (1 << 26) /* glk+ */ -#define PIPEMISC_HDR_MODE_PRECISION (1 << 23) /* icl+ */ -#define PIPEMISC_OUTPUT_COLORSPACE_YUV (1 << 11) -#define PIPEMISC_PIXEL_ROUNDING_TRUNC REG_BIT(8) /* tgl+ */ +#define PIPEMISC_YUV420_ENABLE REG_BIT(27) /* glk+ */ +#define PIPEMISC_YUV420_MODE_FULL_BLEND REG_BIT(26) /* glk+ */ +#define PIPEMISC_HDR_MODE_PRECISION REG_BIT(23) /* icl+ */ +#define PIPEMISC_OUTPUT_COLORSPACE_YUV REG_BIT(11) +#define PIPEMISC_PIXEL_ROUNDING_TRUNC REG_BIT(8) /* tgl+ */ /* * For Display < 13, Bits 5-7 of PIPE MISC represent DITHER BPC with * valid values of: 6, 8, 10 BPC. * ADLP+, the bits 5-7 represent PORT OUTPUT BPC with valid values of: * 6, 8, 10, 12 BPC. */ -#define PIPEMISC_BPC_MASK (7 << 5) -#define PIPEMISC_8_BPC (0 << 5) -#define PIPEMISC_10_BPC (1 << 5) -#define PIPEMISC_6_BPC (2 << 5) -#define PIPEMISC_12_BPC_ADLP (4 << 5) /* adlp+ */ -#define PIPEMISC_DITHER_ENABLE (1 << 4) -#define PIPEMISC_DITHER_TYPE_MASK (3 << 2) -#define PIPEMISC_DITHER_TYPE_SP (0 << 2) +#define PIPEMISC_BPC_MASK REG_GENMASK(7, 5) +#define PIPEMISC_BPC_8 REG_FIELD_PREP(PIPEMISC_BPC_MASK, 0) +#define PIPEMISC_BPC_10 REG_FIELD_PREP(PIPEMISC_BPC_MASK, 1) +#define PIPEMISC_BPC_6 REG_FIELD_PREP(PIPEMISC_BPC_MASK, 2) +#define PIPEMISC_BPC_12_ADLP REG_FIELD_PREP(PIPEMISC_BPC_MASK, 4) /* adlp+ */ +#define PIPEMISC_DITHER_ENABLE REG_BIT(4) +#define PIPEMISC_DITHER_TYPE_MASK REG_GENMASK(3, 2) +#define PIPEMISC_DITHER_TYPE_SP REG_FIELD_PREP(PIPEMISC_DITHER_TYPE_MASK, 0) +#define PIPEMISC_DITHER_TYPE_ST1 REG_FIELD_PREP(PIPEMISC_DITHER_TYPE_MASK, 1) +#define PIPEMISC_DITHER_TYPE_ST2 REG_FIELD_PREP(PIPEMISC_DITHER_TYPE_MASK, 2) +#define PIPEMISC_DITHER_TYPE_TEMP REG_FIELD_PREP(PIPEMISC_DITHER_TYPE_MASK, 3) #define PIPEMISC(pipe) _MMIO_PIPE2(pipe, _PIPE_MISC_A) #define _PIPE_MISC2_A 0x7002C #define _PIPE_MISC2_B 0x7102C -#define PIPE_MISC2_BUBBLE_COUNTER_SCALER_EN (0x50 << 24) -#define PIPE_MISC2_BUBBLE_COUNTER_SCALER_DIS (0x14 << 24) -#define PIPE_MISC2_UNDERRUN_BUBBLE_COUNTER_MASK (0xff << 24) +#define PIPE_MISC2_BUBBLE_COUNTER_MASK REG_GENMASK(31, 24) +#define PIPE_MISC2_BUBBLE_COUNTER_SCALER_EN REG_FIELD_PREP(PIPE_MISC2_BUBBLE_COUNTER_MASK, 80) +#define PIPE_MISC2_BUBBLE_COUNTER_SCALER_DIS REG_FIELD_PREP(PIPE_MISC2_BUBBLE_COUNTER_MASK, 20) #define PIPE_MISC2(pipe) _MMIO_PIPE2(pipe, _PIPE_MISC2_A) /* Skylake+ pipe bottom (background) color */ #define _SKL_BOTTOM_COLOR_A 0x70034 -#define SKL_BOTTOM_COLOR_GAMMA_ENABLE (1 << 31) -#define SKL_BOTTOM_COLOR_CSC_ENABLE (1 << 30) +#define SKL_BOTTOM_COLOR_GAMMA_ENABLE REG_BIT(31) +#define SKL_BOTTOM_COLOR_CSC_ENABLE REG_BIT(30) #define SKL_BOTTOM_COLOR(pipe) _MMIO_PIPE2(pipe, _SKL_BOTTOM_COLOR_A) #define _ICL_PIPE_A_STATUS 0x70058 @@ -6699,49 +4184,32 @@ enum { #define _WM0_PIPEC_IVB 0x45200 #define WM0_PIPE_ILK(pipe) _MMIO_PIPE3((pipe), _WM0_PIPEA_ILK, \ _WM0_PIPEB_ILK, _WM0_PIPEC_IVB) -#define WM0_PIPE_PLANE_MASK (0xffff << 16) -#define WM0_PIPE_PLANE_SHIFT 16 -#define WM0_PIPE_SPRITE_MASK (0xff << 8) -#define WM0_PIPE_SPRITE_SHIFT 8 -#define WM0_PIPE_CURSOR_MASK (0xff) +#define WM0_PIPE_PRIMARY_MASK REG_GENMASK(31, 16) +#define WM0_PIPE_SPRITE_MASK REG_GENMASK(15, 8) +#define WM0_PIPE_CURSOR_MASK REG_GENMASK(7, 0) +#define WM0_PIPE_PRIMARY(x) REG_FIELD_PREP(WM0_PIPE_PRIMARY_MASK, (x)) +#define WM0_PIPE_SPRITE(x) REG_FIELD_PREP(WM0_PIPE_SPRITE_MASK, (x)) +#define WM0_PIPE_CURSOR(x) REG_FIELD_PREP(WM0_PIPE_CURSOR_MASK, (x)) #define WM1_LP_ILK _MMIO(0x45108) -#define WM1_LP_SR_EN (1 << 31) -#define WM1_LP_LATENCY_SHIFT 24 -#define WM1_LP_LATENCY_MASK (0x7f << 24) -#define WM1_LP_FBC_MASK (0xf << 20) -#define WM1_LP_FBC_SHIFT 20 -#define WM1_LP_FBC_SHIFT_BDW 19 -#define WM1_LP_SR_MASK (0x7ff << 8) -#define WM1_LP_SR_SHIFT 8 -#define WM1_LP_CURSOR_MASK (0xff) #define WM2_LP_ILK _MMIO(0x4510c) -#define WM2_LP_EN (1 << 31) #define WM3_LP_ILK _MMIO(0x45110) -#define WM3_LP_EN (1 << 31) +#define WM_LP_ENABLE REG_BIT(31) +#define WM_LP_LATENCY_MASK REG_GENMASK(30, 24) +#define WM_LP_FBC_MASK_BDW REG_GENMASK(23, 19) +#define WM_LP_FBC_MASK_ILK REG_GENMASK(23, 20) +#define WM_LP_PRIMARY_MASK REG_GENMASK(18, 8) +#define WM_LP_CURSOR_MASK REG_GENMASK(7, 0) +#define WM_LP_LATENCY(x) REG_FIELD_PREP(WM_LP_LATENCY_MASK, (x)) +#define WM_LP_FBC_BDW(x) REG_FIELD_PREP(WM_LP_FBC_MASK_BDW, (x)) +#define WM_LP_FBC_ILK(x) REG_FIELD_PREP(WM_LP_FBC_MASK_ILK, (x)) +#define WM_LP_PRIMARY(x) REG_FIELD_PREP(WM_LP_PRIMARY_MASK, (x)) +#define WM_LP_CURSOR(x) REG_FIELD_PREP(WM_LP_CURSOR_MASK, (x)) #define WM1S_LP_ILK _MMIO(0x45120) #define WM2S_LP_IVB _MMIO(0x45124) #define WM3S_LP_IVB _MMIO(0x45128) -#define WM1S_LP_EN (1 << 31) - -#define HSW_WM_LP_VAL(lat, fbc, pri, cur) \ - (WM3_LP_EN | ((lat) << WM1_LP_LATENCY_SHIFT) | \ - ((fbc) << WM1_LP_FBC_SHIFT) | ((pri) << WM1_LP_SR_SHIFT) | (cur)) - -/* Memory latency timer register */ -#define MLTR_ILK _MMIO(0x11222) -#define MLTR_WM1_SHIFT 0 -#define MLTR_WM2_SHIFT 8 -/* the unit of memory self-refresh latency time is 0.5us */ -#define ILK_SRLT_MASK 0x3f - - -/* the address where we get all kinds of latency value */ -#define SSKPD _MMIO(0x5d10) -#define SSKPD_WM_MASK 0x3f -#define SSKPD_WM0_SHIFT 0 -#define SSKPD_WM1_SHIFT 8 -#define SSKPD_WM2_SHIFT 16 -#define SSKPD_WM3_SHIFT 24 +#define WM_LP_SPRITE_ENABLE REG_BIT(31) /* ilk/snb WM1S only */ +#define WM_LP_SPRITE_MASK REG_GENMASK(10, 0) +#define WM_LP_SPRITE(x) REG_FIELD_PREP(WM_LP_SPRITE_MASK, (x)) /* * The two pipe frame counter registers are not synchronized, so @@ -6775,44 +4243,50 @@ enum { /* Cursor A & B regs */ #define _CURACNTR 0x70080 /* Old style CUR*CNTR flags (desktop 8xx) */ -#define CURSOR_ENABLE 0x80000000 -#define CURSOR_GAMMA_ENABLE 0x40000000 -#define CURSOR_STRIDE_SHIFT 28 -#define CURSOR_STRIDE(x) ((ffs(x) - 9) << CURSOR_STRIDE_SHIFT) /* 256,512,1k,2k */ -#define CURSOR_FORMAT_SHIFT 24 -#define CURSOR_FORMAT_MASK (0x07 << CURSOR_FORMAT_SHIFT) -#define CURSOR_FORMAT_2C (0x00 << CURSOR_FORMAT_SHIFT) -#define CURSOR_FORMAT_3C (0x01 << CURSOR_FORMAT_SHIFT) -#define CURSOR_FORMAT_4C (0x02 << CURSOR_FORMAT_SHIFT) -#define CURSOR_FORMAT_ARGB (0x04 << CURSOR_FORMAT_SHIFT) -#define CURSOR_FORMAT_XRGB (0x05 << CURSOR_FORMAT_SHIFT) +#define CURSOR_ENABLE REG_BIT(31) +#define CURSOR_PIPE_GAMMA_ENABLE REG_BIT(30) +#define CURSOR_STRIDE_MASK REG_GENMASK(29, 28) +#define CURSOR_STRIDE(stride) REG_FIELD_PREP(CURSOR_STRIDE_MASK, ffs(stride) - 9) /* 256,512,1k,2k */ +#define CURSOR_FORMAT_MASK REG_GENMASK(26, 24) +#define CURSOR_FORMAT_2C REG_FIELD_PREP(CURSOR_FORMAT_MASK, 0) +#define CURSOR_FORMAT_3C REG_FIELD_PREP(CURSOR_FORMAT_MASK, 1) +#define CURSOR_FORMAT_4C REG_FIELD_PREP(CURSOR_FORMAT_MASK, 2) +#define CURSOR_FORMAT_ARGB REG_FIELD_PREP(CURSOR_FORMAT_MASK, 4) +#define CURSOR_FORMAT_XRGB REG_FIELD_PREP(CURSOR_FORMAT_MASK, 5) /* New style CUR*CNTR flags */ -#define MCURSOR_MODE 0x27 -#define MCURSOR_MODE_DISABLE 0x00 -#define MCURSOR_MODE_128_32B_AX 0x02 -#define MCURSOR_MODE_256_32B_AX 0x03 -#define MCURSOR_MODE_64_32B_AX 0x07 -#define MCURSOR_MODE_128_ARGB_AX ((1 << 5) | MCURSOR_MODE_128_32B_AX) -#define MCURSOR_MODE_256_ARGB_AX ((1 << 5) | MCURSOR_MODE_256_32B_AX) -#define MCURSOR_MODE_64_ARGB_AX ((1 << 5) | MCURSOR_MODE_64_32B_AX) #define MCURSOR_ARB_SLOTS_MASK REG_GENMASK(30, 28) /* icl+ */ #define MCURSOR_ARB_SLOTS(x) REG_FIELD_PREP(MCURSOR_ARB_SLOTS_MASK, (x)) /* icl+ */ -#define MCURSOR_PIPE_SELECT_MASK (0x3 << 28) -#define MCURSOR_PIPE_SELECT_SHIFT 28 -#define MCURSOR_PIPE_SELECT(pipe) ((pipe) << 28) -#define MCURSOR_GAMMA_ENABLE (1 << 26) -#define MCURSOR_PIPE_CSC_ENABLE (1 << 24) /* ilk+ */ -#define MCURSOR_ROTATE_180 (1 << 15) -#define MCURSOR_TRICKLE_FEED_DISABLE (1 << 14) +#define MCURSOR_PIPE_SEL_MASK REG_GENMASK(29, 28) +#define MCURSOR_PIPE_SEL(pipe) REG_FIELD_PREP(MCURSOR_PIPE_SEL_MASK, (pipe)) +#define MCURSOR_PIPE_GAMMA_ENABLE REG_BIT(26) +#define MCURSOR_PIPE_CSC_ENABLE REG_BIT(24) /* ilk+ */ +#define MCURSOR_ROTATE_180 REG_BIT(15) +#define MCURSOR_TRICKLE_FEED_DISABLE REG_BIT(14) +#define MCURSOR_MODE_MASK 0x27 +#define MCURSOR_MODE_DISABLE 0x00 +#define MCURSOR_MODE_128_32B_AX 0x02 +#define MCURSOR_MODE_256_32B_AX 0x03 +#define MCURSOR_MODE_64_32B_AX 0x07 +#define MCURSOR_MODE_128_ARGB_AX (0x20 | MCURSOR_MODE_128_32B_AX) +#define MCURSOR_MODE_256_ARGB_AX (0x20 | MCURSOR_MODE_256_32B_AX) +#define MCURSOR_MODE_64_ARGB_AX (0x20 | MCURSOR_MODE_64_32B_AX) #define _CURABASE 0x70084 #define _CURAPOS 0x70088 -#define CURSOR_POS_MASK 0x007FF -#define CURSOR_POS_SIGN 0x8000 -#define CURSOR_X_SHIFT 0 -#define CURSOR_Y_SHIFT 16 -#define CURSIZE _MMIO(0x700a0) /* 845/865 */ +#define CURSOR_POS_Y_SIGN REG_BIT(31) +#define CURSOR_POS_Y_MASK REG_GENMASK(30, 16) +#define CURSOR_POS_Y(y) REG_FIELD_PREP(CURSOR_POS_Y_MASK, (y)) +#define CURSOR_POS_X_SIGN REG_BIT(15) +#define CURSOR_POS_X_MASK REG_GENMASK(14, 0) +#define CURSOR_POS_X(x) REG_FIELD_PREP(CURSOR_POS_X_MASK, (x)) +#define _CURASIZE 0x700a0 /* 845/865 */ +#define CURSOR_HEIGHT_MASK REG_GENMASK(21, 12) +#define CURSOR_HEIGHT(h) REG_FIELD_PREP(CURSOR_HEIGHT_MASK, (h)) +#define CURSOR_WIDTH_MASK REG_GENMASK(9, 0) +#define CURSOR_WIDTH(w) REG_FIELD_PREP(CURSOR_WIDTH_MASK, (w)) #define _CUR_FBC_CTL_A 0x700a0 /* ivb+ */ -#define CUR_FBC_CTL_EN (1 << 31) +#define CUR_FBC_EN REG_BIT(31) +#define CUR_FBC_HEIGHT_MASK REG_GENMASK(7, 0) +#define CUR_FBC_HEIGHT(h) REG_FIELD_PREP(CUR_FBC_HEIGHT_MASK, (h)) #define _CURASURFLIVE 0x700ac /* g4x+ */ #define _CURBCNTR 0x700c0 #define _CURBBASE 0x700c4 @@ -6825,6 +4299,7 @@ enum { #define CURCNTR(pipe) _CURSOR2(pipe, _CURACNTR) #define CURBASE(pipe) _CURSOR2(pipe, _CURABASE) #define CURPOS(pipe) _CURSOR2(pipe, _CURAPOS) +#define CURSIZE(pipe) _CURSOR2(pipe, _CURASIZE) #define CUR_FBC_CTL(pipe) _CURSOR2(pipe, _CUR_FBC_CTL_A) #define CURSURFLIVE(pipe) _CURSOR2(pipe, _CURASURFLIVE) @@ -6838,49 +4313,54 @@ enum { /* Display A control */ #define _DSPAADDR_VLV 0x7017C /* vlv/chv */ #define _DSPACNTR 0x70180 -#define DISPLAY_PLANE_ENABLE (1 << 31) -#define DISPLAY_PLANE_DISABLE 0 -#define DISPPLANE_GAMMA_ENABLE (1 << 30) -#define DISPPLANE_GAMMA_DISABLE 0 -#define DISPPLANE_PIXFORMAT_MASK (0xf << 26) -#define DISPPLANE_YUV422 (0x0 << 26) -#define DISPPLANE_8BPP (0x2 << 26) -#define DISPPLANE_BGRA555 (0x3 << 26) -#define DISPPLANE_BGRX555 (0x4 << 26) -#define DISPPLANE_BGRX565 (0x5 << 26) -#define DISPPLANE_BGRX888 (0x6 << 26) -#define DISPPLANE_BGRA888 (0x7 << 26) -#define DISPPLANE_RGBX101010 (0x8 << 26) -#define DISPPLANE_RGBA101010 (0x9 << 26) -#define DISPPLANE_BGRX101010 (0xa << 26) -#define DISPPLANE_BGRA101010 (0xb << 26) -#define DISPPLANE_RGBX161616 (0xc << 26) -#define DISPPLANE_RGBX888 (0xe << 26) -#define DISPPLANE_RGBA888 (0xf << 26) -#define DISPPLANE_STEREO_ENABLE (1 << 25) -#define DISPPLANE_STEREO_DISABLE 0 -#define DISPPLANE_PIPE_CSC_ENABLE (1 << 24) /* ilk+ */ -#define DISPPLANE_SEL_PIPE_SHIFT 24 -#define DISPPLANE_SEL_PIPE_MASK (3 << DISPPLANE_SEL_PIPE_SHIFT) -#define DISPPLANE_SEL_PIPE(pipe) ((pipe) << DISPPLANE_SEL_PIPE_SHIFT) -#define DISPPLANE_SRC_KEY_ENABLE (1 << 22) -#define DISPPLANE_SRC_KEY_DISABLE 0 -#define DISPPLANE_LINE_DOUBLE (1 << 20) -#define DISPPLANE_NO_LINE_DOUBLE 0 -#define DISPPLANE_STEREO_POLARITY_FIRST 0 -#define DISPPLANE_STEREO_POLARITY_SECOND (1 << 18) -#define DISPPLANE_ALPHA_PREMULTIPLY (1 << 16) /* CHV pipe B */ -#define DISPPLANE_ROTATE_180 (1 << 15) -#define DISPPLANE_TRICKLE_FEED_DISABLE (1 << 14) /* Ironlake */ -#define DISPPLANE_TILED (1 << 10) -#define DISPPLANE_ASYNC_FLIP (1 << 9) /* g4x+ */ -#define DISPPLANE_MIRROR (1 << 8) /* CHV pipe B */ +#define DISP_ENABLE REG_BIT(31) +#define DISP_PIPE_GAMMA_ENABLE REG_BIT(30) +#define DISP_FORMAT_MASK REG_GENMASK(29, 26) +#define DISP_FORMAT_8BPP REG_FIELD_PREP(DISP_FORMAT_MASK, 2) +#define DISP_FORMAT_BGRA555 REG_FIELD_PREP(DISP_FORMAT_MASK, 3) +#define DISP_FORMAT_BGRX555 REG_FIELD_PREP(DISP_FORMAT_MASK, 4) +#define DISP_FORMAT_BGRX565 REG_FIELD_PREP(DISP_FORMAT_MASK, 5) +#define DISP_FORMAT_BGRX888 REG_FIELD_PREP(DISP_FORMAT_MASK, 6) +#define DISP_FORMAT_BGRA888 REG_FIELD_PREP(DISP_FORMAT_MASK, 7) +#define DISP_FORMAT_RGBX101010 REG_FIELD_PREP(DISP_FORMAT_MASK, 8) +#define DISP_FORMAT_RGBA101010 REG_FIELD_PREP(DISP_FORMAT_MASK, 9) +#define DISP_FORMAT_BGRX101010 REG_FIELD_PREP(DISP_FORMAT_MASK, 10) +#define DISP_FORMAT_BGRA101010 REG_FIELD_PREP(DISP_FORMAT_MASK, 11) +#define DISP_FORMAT_RGBX161616 REG_FIELD_PREP(DISP_FORMAT_MASK, 12) +#define DISP_FORMAT_RGBX888 REG_FIELD_PREP(DISP_FORMAT_MASK, 14) +#define DISP_FORMAT_RGBA888 REG_FIELD_PREP(DISP_FORMAT_MASK, 15) +#define DISP_STEREO_ENABLE REG_BIT(25) +#define DISP_PIPE_CSC_ENABLE REG_BIT(24) /* ilk+ */ +#define DISP_PIPE_SEL_MASK REG_GENMASK(25, 24) +#define DISP_PIPE_SEL(pipe) REG_FIELD_PREP(DISP_PIPE_SEL_MASK, (pipe)) +#define DISP_SRC_KEY_ENABLE REG_BIT(22) +#define DISP_LINE_DOUBLE REG_BIT(20) +#define DISP_STEREO_POLARITY_SECOND REG_BIT(18) +#define DISP_ALPHA_PREMULTIPLY REG_BIT(16) /* CHV pipe B */ +#define DISP_ROTATE_180 REG_BIT(15) +#define DISP_TRICKLE_FEED_DISABLE REG_BIT(14) /* g4x+ */ +#define DISP_TILED REG_BIT(10) +#define DISP_ASYNC_FLIP REG_BIT(9) /* g4x+ */ +#define DISP_MIRROR REG_BIT(8) /* CHV pipe B */ #define _DSPAADDR 0x70184 #define _DSPASTRIDE 0x70188 #define _DSPAPOS 0x7018C /* reserved */ +#define DISP_POS_Y_MASK REG_GENMASK(31, 0) +#define DISP_POS_Y(y) REG_FIELD_PREP(DISP_POS_Y_MASK, (y)) +#define DISP_POS_X_MASK REG_GENMASK(15, 0) +#define DISP_POS_X(x) REG_FIELD_PREP(DISP_POS_X_MASK, (x)) #define _DSPASIZE 0x70190 +#define DISP_HEIGHT_MASK REG_GENMASK(31, 0) +#define DISP_HEIGHT(h) REG_FIELD_PREP(DISP_HEIGHT_MASK, (h)) +#define DISP_WIDTH_MASK REG_GENMASK(15, 0) +#define DISP_WIDTH(w) REG_FIELD_PREP(DISP_WIDTH_MASK, (w)) #define _DSPASURF 0x7019C /* 965+ only */ +#define DISP_ADDR_MASK REG_GENMASK(31, 12) #define _DSPATILEOFF 0x701A4 /* 965+ only */ +#define DISP_OFFSET_Y_MASK REG_GENMASK(31, 16) +#define DISP_OFFSET_Y(y) REG_FIELD_PREP(DISP_OFFSET_Y_MASK, (y)) +#define DISP_OFFSET_X_MASK REG_GENMASK(15, 0) +#define DISP_OFFSET_X(x) REG_FIELD_PREP(DISP_OFFSET_X_MASK, (x)) #define _DSPAOFFSET 0x701A4 /* HSW */ #define _DSPASURFLIVE 0x701AC #define _DSPAGAMC 0x701E0 @@ -6900,15 +4380,28 @@ enum { /* CHV pipe B blender and primary plane */ #define _CHV_BLEND_A 0x60a00 -#define CHV_BLEND_LEGACY (0 << 30) -#define CHV_BLEND_ANDROID (1 << 30) -#define CHV_BLEND_MPO (2 << 30) -#define CHV_BLEND_MASK (3 << 30) +#define CHV_BLEND_MASK REG_GENMASK(31, 30) +#define CHV_BLEND_LEGACY REG_FIELD_PREP(CHV_BLEND_MASK, 0) +#define CHV_BLEND_ANDROID REG_FIELD_PREP(CHV_BLEND_MASK, 1) +#define CHV_BLEND_MPO REG_FIELD_PREP(CHV_BLEND_MASK, 2) #define _CHV_CANVAS_A 0x60a04 +#define CHV_CANVAS_RED_MASK REG_GENMASK(29, 20) +#define CHV_CANVAS_GREEN_MASK REG_GENMASK(19, 10) +#define CHV_CANVAS_BLUE_MASK REG_GENMASK(9, 0) #define _PRIMPOS_A 0x60a08 +#define PRIM_POS_Y_MASK REG_GENMASK(31, 16) +#define PRIM_POS_Y(y) REG_FIELD_PREP(PRIM_POS_Y_MASK, (y)) +#define PRIM_POS_X_MASK REG_GENMASK(15, 0) +#define PRIM_POS_X(x) REG_FIELD_PREP(PRIM_POS_X_MASK, (x)) #define _PRIMSIZE_A 0x60a0c +#define PRIM_HEIGHT_MASK REG_GENMASK(31, 16) +#define PRIM_HEIGHT(h) REG_FIELD_PREP(PRIM_HEIGHT_MASK, (h)) +#define PRIM_WIDTH_MASK REG_GENMASK(15, 0) +#define PRIM_WIDTH(w) REG_FIELD_PREP(PRIM_WIDTH_MASK, (w)) #define _PRIMCNSTALPHA_A 0x60a10 -#define PRIM_CONST_ALPHA_ENABLE (1 << 31) +#define PRIM_CONST_ALPHA_ENABLE REG_BIT(31) +#define PRIM_CONST_ALPHA_MASK REG_GENMASK(7, 0) +#define PRIM_CONST_ALPHA(alpha) REG_FIELD_PREP(PRIM_CONST_ALPHA_MASK, (alpha)) #define CHV_BLEND(pipe) _MMIO_TRANS2(pipe, _CHV_BLEND_A) #define CHV_CANVAS(pipe) _MMIO_TRANS2(pipe, _CHV_CANVAS_A) @@ -6949,10 +4442,8 @@ enum { /* Display B control */ #define _DSPBCNTR (DISPLAY_MMIO_BASE(dev_priv) + 0x71180) -#define DISPPLANE_ALPHA_TRANS_ENABLE (1 << 15) -#define DISPPLANE_ALPHA_TRANS_DISABLE 0 -#define DISPPLANE_SPRITE_ABOVE_DISPLAY 0 -#define DISPPLANE_SPRITE_ABOVE_OVERLAY (1) +#define DISP_ALPHA_TRANS_ENABLE REG_BIT(15) +#define DISP_SPRITE_ABOVE_OVERLAY REG_BIT(0) #define _DSPBADDR (DISPLAY_MMIO_BASE(dev_priv) + 0x71184) #define _DSPBSTRIDE (DISPLAY_MMIO_BASE(dev_priv) + 0x71188) #define _DSPBPOS (DISPLAY_MMIO_BASE(dev_priv) + 0x7118C) @@ -6968,46 +4459,63 @@ enum { /* Sprite A control */ #define _DVSACNTR 0x72180 -#define DVS_ENABLE (1 << 31) -#define DVS_GAMMA_ENABLE (1 << 30) -#define DVS_YUV_RANGE_CORRECTION_DISABLE (1 << 27) -#define DVS_PIXFORMAT_MASK (3 << 25) -#define DVS_FORMAT_YUV422 (0 << 25) -#define DVS_FORMAT_RGBX101010 (1 << 25) -#define DVS_FORMAT_RGBX888 (2 << 25) -#define DVS_FORMAT_RGBX161616 (3 << 25) -#define DVS_PIPE_CSC_ENABLE (1 << 24) -#define DVS_SOURCE_KEY (1 << 22) -#define DVS_RGB_ORDER_XBGR (1 << 20) -#define DVS_YUV_FORMAT_BT709 (1 << 18) -#define DVS_YUV_ORDER_MASK (3 << 16) -#define DVS_YUV_ORDER_YUYV (0 << 16) -#define DVS_YUV_ORDER_UYVY (1 << 16) -#define DVS_YUV_ORDER_YVYU (2 << 16) -#define DVS_YUV_ORDER_VYUY (3 << 16) -#define DVS_ROTATE_180 (1 << 15) -#define DVS_DEST_KEY (1 << 2) -#define DVS_TRICKLE_FEED_DISABLE (1 << 14) -#define DVS_TILED (1 << 10) +#define DVS_ENABLE REG_BIT(31) +#define DVS_PIPE_GAMMA_ENABLE REG_BIT(30) +#define DVS_YUV_RANGE_CORRECTION_DISABLE REG_BIT(27) +#define DVS_FORMAT_MASK REG_GENMASK(26, 25) +#define DVS_FORMAT_YUV422 REG_FIELD_PREP(DVS_FORMAT_MASK, 0) +#define DVS_FORMAT_RGBX101010 REG_FIELD_PREP(DVS_FORMAT_MASK, 1) +#define DVS_FORMAT_RGBX888 REG_FIELD_PREP(DVS_FORMAT_MASK, 2) +#define DVS_FORMAT_RGBX161616 REG_FIELD_PREP(DVS_FORMAT_MASK, 3) +#define DVS_PIPE_CSC_ENABLE REG_BIT(24) +#define DVS_SOURCE_KEY REG_BIT(22) +#define DVS_RGB_ORDER_XBGR REG_BIT(20) +#define DVS_YUV_FORMAT_BT709 REG_BIT(18) +#define DVS_YUV_ORDER_MASK REG_GENMASK(17, 16) +#define DVS_YUV_ORDER_YUYV REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 0) +#define DVS_YUV_ORDER_UYVY REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 1) +#define DVS_YUV_ORDER_YVYU REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 2) +#define DVS_YUV_ORDER_VYUY REG_FIELD_PREP(DVS_YUV_ORDER_MASK, 3) +#define DVS_ROTATE_180 REG_BIT(15) +#define DVS_TRICKLE_FEED_DISABLE REG_BIT(14) +#define DVS_TILED REG_BIT(10) +#define DVS_DEST_KEY REG_BIT(2) #define _DVSALINOFF 0x72184 #define _DVSASTRIDE 0x72188 #define _DVSAPOS 0x7218c +#define DVS_POS_Y_MASK REG_GENMASK(31, 16) +#define DVS_POS_Y(y) REG_FIELD_PREP(DVS_POS_Y_MASK, (y)) +#define DVS_POS_X_MASK REG_GENMASK(15, 0) +#define DVS_POS_X(x) REG_FIELD_PREP(DVS_POS_X_MASK, (x)) #define _DVSASIZE 0x72190 +#define DVS_HEIGHT_MASK REG_GENMASK(31, 16) +#define DVS_HEIGHT(h) REG_FIELD_PREP(DVS_HEIGHT_MASK, (h)) +#define DVS_WIDTH_MASK REG_GENMASK(15, 0) +#define DVS_WIDTH(w) REG_FIELD_PREP(DVS_WIDTH_MASK, (w)) #define _DVSAKEYVAL 0x72194 #define _DVSAKEYMSK 0x72198 #define _DVSASURF 0x7219c +#define DVS_ADDR_MASK REG_GENMASK(31, 12) #define _DVSAKEYMAXVAL 0x721a0 #define _DVSATILEOFF 0x721a4 +#define DVS_OFFSET_Y_MASK REG_GENMASK(31, 16) +#define DVS_OFFSET_Y(y) REG_FIELD_PREP(DVS_OFFSET_Y_MASK, (y)) +#define DVS_OFFSET_X_MASK REG_GENMASK(15, 0) +#define DVS_OFFSET_X(x) REG_FIELD_PREP(DVS_OFFSET_X_MASK, (x)) #define _DVSASURFLIVE 0x721ac #define _DVSAGAMC_G4X 0x721e0 /* g4x */ #define _DVSASCALE 0x72204 -#define DVS_SCALE_ENABLE (1 << 31) -#define DVS_FILTER_MASK (3 << 29) -#define DVS_FILTER_MEDIUM (0 << 29) -#define DVS_FILTER_ENHANCING (1 << 29) -#define DVS_FILTER_SOFTENING (2 << 29) -#define DVS_VERTICAL_OFFSET_HALF (1 << 28) /* must be enabled below */ -#define DVS_VERTICAL_OFFSET_ENABLE (1 << 27) +#define DVS_SCALE_ENABLE REG_BIT(31) +#define DVS_FILTER_MASK REG_GENMASK(30, 29) +#define DVS_FILTER_MEDIUM REG_FIELD_PREP(DVS_FILTER_MASK, 0) +#define DVS_FILTER_ENHANCING REG_FIELD_PREP(DVS_FILTER_MASK, 1) +#define DVS_FILTER_SOFTENING REG_FIELD_PREP(DVS_FILTER_MASK, 2) +#define DVS_VERTICAL_OFFSET_HALF REG_BIT(28) /* must be enabled below */ +#define DVS_VERTICAL_OFFSET_ENABLE REG_BIT(27) +#define DVS_SRC_WIDTH_MASK REG_GENMASK(26, 16) +#define DVS_SRC_WIDTH(w) REG_FIELD_PREP(DVS_SRC_WIDTH_MASK, (w)) +#define DVS_SRC_HEIGHT_MASK REG_GENMASK(10, 0) +#define DVS_SRC_HEIGHT(h) REG_FIELD_PREP(DVS_SRC_HEIGHT_MASK, (h)) #define _DVSAGAMC_ILK 0x72300 /* ilk/snb */ #define _DVSAGAMCMAX_ILK 0x72340 /* ilk/snb */ @@ -7044,50 +4552,67 @@ enum { #define DVSGAMCMAX_ILK(pipe, i) _MMIO(_PIPE(pipe, _DVSAGAMCMAX_ILK, _DVSBGAMCMAX_ILK) + (i) * 4) /* 3 x u1.10 */ #define _SPRA_CTL 0x70280 -#define SPRITE_ENABLE (1 << 31) -#define SPRITE_GAMMA_ENABLE (1 << 30) -#define SPRITE_YUV_RANGE_CORRECTION_DISABLE (1 << 28) -#define SPRITE_PIXFORMAT_MASK (7 << 25) -#define SPRITE_FORMAT_YUV422 (0 << 25) -#define SPRITE_FORMAT_RGBX101010 (1 << 25) -#define SPRITE_FORMAT_RGBX888 (2 << 25) -#define SPRITE_FORMAT_RGBX161616 (3 << 25) -#define SPRITE_FORMAT_YUV444 (4 << 25) -#define SPRITE_FORMAT_XR_BGR101010 (5 << 25) /* Extended range */ -#define SPRITE_PIPE_CSC_ENABLE (1 << 24) -#define SPRITE_SOURCE_KEY (1 << 22) -#define SPRITE_RGB_ORDER_RGBX (1 << 20) /* only for 888 and 161616 */ -#define SPRITE_YUV_TO_RGB_CSC_DISABLE (1 << 19) -#define SPRITE_YUV_TO_RGB_CSC_FORMAT_BT709 (1 << 18) /* 0 is BT601 */ -#define SPRITE_YUV_ORDER_MASK (3 << 16) -#define SPRITE_YUV_ORDER_YUYV (0 << 16) -#define SPRITE_YUV_ORDER_UYVY (1 << 16) -#define SPRITE_YUV_ORDER_YVYU (2 << 16) -#define SPRITE_YUV_ORDER_VYUY (3 << 16) -#define SPRITE_ROTATE_180 (1 << 15) -#define SPRITE_TRICKLE_FEED_DISABLE (1 << 14) -#define SPRITE_INT_GAMMA_DISABLE (1 << 13) -#define SPRITE_TILED (1 << 10) -#define SPRITE_DEST_KEY (1 << 2) +#define SPRITE_ENABLE REG_BIT(31) +#define SPRITE_PIPE_GAMMA_ENABLE REG_BIT(30) +#define SPRITE_YUV_RANGE_CORRECTION_DISABLE REG_BIT(28) +#define SPRITE_FORMAT_MASK REG_GENMASK(27, 25) +#define SPRITE_FORMAT_YUV422 REG_FIELD_PREP(SPRITE_FORMAT_MASK, 0) +#define SPRITE_FORMAT_RGBX101010 REG_FIELD_PREP(SPRITE_FORMAT_MASK, 1) +#define SPRITE_FORMAT_RGBX888 REG_FIELD_PREP(SPRITE_FORMAT_MASK, 2) +#define SPRITE_FORMAT_RGBX161616 REG_FIELD_PREP(SPRITE_FORMAT_MASK, 3) +#define SPRITE_FORMAT_YUV444 REG_FIELD_PREP(SPRITE_FORMAT_MASK, 4) +#define SPRITE_FORMAT_XR_BGR101010 REG_FIELD_PREP(SPRITE_FORMAT_MASK, 5) /* Extended range */ +#define SPRITE_PIPE_CSC_ENABLE REG_BIT(24) +#define SPRITE_SOURCE_KEY REG_BIT(22) +#define SPRITE_RGB_ORDER_RGBX REG_BIT(20) /* only for 888 and 161616 */ +#define SPRITE_YUV_TO_RGB_CSC_DISABLE REG_BIT(19) +#define SPRITE_YUV_TO_RGB_CSC_FORMAT_BT709 REG_BIT(18) /* 0 is BT601 */ +#define SPRITE_YUV_ORDER_MASK REG_GENMASK(17, 16) +#define SPRITE_YUV_ORDER_YUYV REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 0) +#define SPRITE_YUV_ORDER_UYVY REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 1) +#define SPRITE_YUV_ORDER_YVYU REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 2) +#define SPRITE_YUV_ORDER_VYUY REG_FIELD_PREP(SPRITE_YUV_ORDER_MASK, 3) +#define SPRITE_ROTATE_180 REG_BIT(15) +#define SPRITE_TRICKLE_FEED_DISABLE REG_BIT(14) +#define SPRITE_PLANE_GAMMA_DISABLE REG_BIT(13) +#define SPRITE_TILED REG_BIT(10) +#define SPRITE_DEST_KEY REG_BIT(2) #define _SPRA_LINOFF 0x70284 #define _SPRA_STRIDE 0x70288 #define _SPRA_POS 0x7028c +#define SPRITE_POS_Y_MASK REG_GENMASK(31, 16) +#define SPRITE_POS_Y(y) REG_FIELD_PREP(SPRITE_POS_Y_MASK, (y)) +#define SPRITE_POS_X_MASK REG_GENMASK(15, 0) +#define SPRITE_POS_X(x) REG_FIELD_PREP(SPRITE_POS_X_MASK, (x)) #define _SPRA_SIZE 0x70290 +#define SPRITE_HEIGHT_MASK REG_GENMASK(31, 16) +#define SPRITE_HEIGHT(h) REG_FIELD_PREP(SPRITE_HEIGHT_MASK, (h)) +#define SPRITE_WIDTH_MASK REG_GENMASK(15, 0) +#define SPRITE_WIDTH(w) REG_FIELD_PREP(SPRITE_WIDTH_MASK, (w)) #define _SPRA_KEYVAL 0x70294 #define _SPRA_KEYMSK 0x70298 #define _SPRA_SURF 0x7029c +#define SPRITE_ADDR_MASK REG_GENMASK(31, 12) #define _SPRA_KEYMAX 0x702a0 #define _SPRA_TILEOFF 0x702a4 +#define SPRITE_OFFSET_Y_MASK REG_GENMASK(31, 16) +#define SPRITE_OFFSET_Y(y) REG_FIELD_PREP(SPRITE_OFFSET_Y_MASK, (y)) +#define SPRITE_OFFSET_X_MASK REG_GENMASK(15, 0) +#define SPRITE_OFFSET_X(x) REG_FIELD_PREP(SPRITE_OFFSET_X_MASK, (x)) #define _SPRA_OFFSET 0x702a4 #define _SPRA_SURFLIVE 0x702ac #define _SPRA_SCALE 0x70304 -#define SPRITE_SCALE_ENABLE (1 << 31) -#define SPRITE_FILTER_MASK (3 << 29) -#define SPRITE_FILTER_MEDIUM (0 << 29) -#define SPRITE_FILTER_ENHANCING (1 << 29) -#define SPRITE_FILTER_SOFTENING (2 << 29) -#define SPRITE_VERTICAL_OFFSET_HALF (1 << 28) /* must be enabled below */ -#define SPRITE_VERTICAL_OFFSET_ENABLE (1 << 27) +#define SPRITE_SCALE_ENABLE REG_BIT(31) +#define SPRITE_FILTER_MASK REG_GENMASK(30, 29) +#define SPRITE_FILTER_MEDIUM REG_FIELD_PREP(SPRITE_FILTER_MASK, 0) +#define SPRITE_FILTER_ENHANCING REG_FIELD_PREP(SPRITE_FILTER_MASK, 1) +#define SPRITE_FILTER_SOFTENING REG_FIELD_PREP(SPRITE_FILTER_MASK, 2) +#define SPRITE_VERTICAL_OFFSET_HALF REG_BIT(28) /* must be enabled below */ +#define SPRITE_VERTICAL_OFFSET_ENABLE REG_BIT(27) +#define SPRITE_SRC_WIDTH_MASK REG_GENMASK(26, 16) +#define SPRITE_SRC_WIDTH(w) REG_FIELD_PREP(SPRITE_SRC_WIDTH_MASK, (w)) +#define SPRITE_SRC_HEIGHT_MASK REG_GENMASK(10, 0) +#define SPRITE_SRC_HEIGHT(h) REG_FIELD_PREP(SPRITE_SRC_HEIGHT_MASK, (h)) #define _SPRA_GAMC 0x70400 #define _SPRA_GAMC16 0x70440 #define _SPRA_GAMC17 0x7044c @@ -7127,48 +4652,67 @@ enum { #define SPRSURFLIVE(pipe) _MMIO_PIPE(pipe, _SPRA_SURFLIVE, _SPRB_SURFLIVE) #define _SPACNTR (VLV_DISPLAY_BASE + 0x72180) -#define SP_ENABLE (1 << 31) -#define SP_GAMMA_ENABLE (1 << 30) -#define SP_PIXFORMAT_MASK (0xf << 26) -#define SP_FORMAT_YUV422 (0x0 << 26) -#define SP_FORMAT_8BPP (0x2 << 26) -#define SP_FORMAT_BGR565 (0x5 << 26) -#define SP_FORMAT_BGRX8888 (0x6 << 26) -#define SP_FORMAT_BGRA8888 (0x7 << 26) -#define SP_FORMAT_RGBX1010102 (0x8 << 26) -#define SP_FORMAT_RGBA1010102 (0x9 << 26) -#define SP_FORMAT_BGRX1010102 (0xa << 26) /* CHV pipe B */ -#define SP_FORMAT_BGRA1010102 (0xb << 26) /* CHV pipe B */ -#define SP_FORMAT_RGBX8888 (0xe << 26) -#define SP_FORMAT_RGBA8888 (0xf << 26) -#define SP_ALPHA_PREMULTIPLY (1 << 23) /* CHV pipe B */ -#define SP_SOURCE_KEY (1 << 22) -#define SP_YUV_FORMAT_BT709 (1 << 18) -#define SP_YUV_ORDER_MASK (3 << 16) -#define SP_YUV_ORDER_YUYV (0 << 16) -#define SP_YUV_ORDER_UYVY (1 << 16) -#define SP_YUV_ORDER_YVYU (2 << 16) -#define SP_YUV_ORDER_VYUY (3 << 16) -#define SP_ROTATE_180 (1 << 15) -#define SP_TILED (1 << 10) -#define SP_MIRROR (1 << 8) /* CHV pipe B */ +#define SP_ENABLE REG_BIT(31) +#define SP_PIPE_GAMMA_ENABLE REG_BIT(30) +#define SP_FORMAT_MASK REG_GENMASK(29, 26) +#define SP_FORMAT_YUV422 REG_FIELD_PREP(SP_FORMAT_MASK, 0) +#define SP_FORMAT_8BPP REG_FIELD_PREP(SP_FORMAT_MASK, 2) +#define SP_FORMAT_BGR565 REG_FIELD_PREP(SP_FORMAT_MASK, 5) +#define SP_FORMAT_BGRX8888 REG_FIELD_PREP(SP_FORMAT_MASK, 6) +#define SP_FORMAT_BGRA8888 REG_FIELD_PREP(SP_FORMAT_MASK, 7) +#define SP_FORMAT_RGBX1010102 REG_FIELD_PREP(SP_FORMAT_MASK, 8) +#define SP_FORMAT_RGBA1010102 REG_FIELD_PREP(SP_FORMAT_MASK, 9) +#define SP_FORMAT_BGRX1010102 REG_FIELD_PREP(SP_FORMAT_MASK, 10) /* CHV pipe B */ +#define SP_FORMAT_BGRA1010102 REG_FIELD_PREP(SP_FORMAT_MASK, 11) /* CHV pipe B */ +#define SP_FORMAT_RGBX8888 REG_FIELD_PREP(SP_FORMAT_MASK, 14) +#define SP_FORMAT_RGBA8888 REG_FIELD_PREP(SP_FORMAT_MASK, 15) +#define SP_ALPHA_PREMULTIPLY REG_BIT(23) /* CHV pipe B */ +#define SP_SOURCE_KEY REG_BIT(22) +#define SP_YUV_FORMAT_BT709 REG_BIT(18) +#define SP_YUV_ORDER_MASK REG_GENMASK(17, 16) +#define SP_YUV_ORDER_YUYV REG_FIELD_PREP(SP_YUV_ORDER_MASK, 0) +#define SP_YUV_ORDER_UYVY REG_FIELD_PREP(SP_YUV_ORDER_MASK, 1) +#define SP_YUV_ORDER_YVYU REG_FIELD_PREP(SP_YUV_ORDER_MASK, 2) +#define SP_YUV_ORDER_VYUY REG_FIELD_PREP(SP_YUV_ORDER_MASK, 3) +#define SP_ROTATE_180 REG_BIT(15) +#define SP_TILED REG_BIT(10) +#define SP_MIRROR REG_BIT(8) /* CHV pipe B */ #define _SPALINOFF (VLV_DISPLAY_BASE + 0x72184) #define _SPASTRIDE (VLV_DISPLAY_BASE + 0x72188) #define _SPAPOS (VLV_DISPLAY_BASE + 0x7218c) +#define SP_POS_Y_MASK REG_GENMASK(31, 16) +#define SP_POS_Y(y) REG_FIELD_PREP(SP_POS_Y_MASK, (y)) +#define SP_POS_X_MASK REG_GENMASK(15, 0) +#define SP_POS_X(x) REG_FIELD_PREP(SP_POS_X_MASK, (x)) #define _SPASIZE (VLV_DISPLAY_BASE + 0x72190) +#define SP_HEIGHT_MASK REG_GENMASK(31, 16) +#define SP_HEIGHT(h) REG_FIELD_PREP(SP_HEIGHT_MASK, (h)) +#define SP_WIDTH_MASK REG_GENMASK(15, 0) +#define SP_WIDTH(w) REG_FIELD_PREP(SP_WIDTH_MASK, (w)) #define _SPAKEYMINVAL (VLV_DISPLAY_BASE + 0x72194) #define _SPAKEYMSK (VLV_DISPLAY_BASE + 0x72198) #define _SPASURF (VLV_DISPLAY_BASE + 0x7219c) +#define SP_ADDR_MASK REG_GENMASK(31, 12) #define _SPAKEYMAXVAL (VLV_DISPLAY_BASE + 0x721a0) #define _SPATILEOFF (VLV_DISPLAY_BASE + 0x721a4) +#define SP_OFFSET_Y_MASK REG_GENMASK(31, 16) +#define SP_OFFSET_Y(y) REG_FIELD_PREP(SP_OFFSET_Y_MASK, (y)) +#define SP_OFFSET_X_MASK REG_GENMASK(15, 0) +#define SP_OFFSET_X(x) REG_FIELD_PREP(SP_OFFSET_X_MASK, (x)) #define _SPACONSTALPHA (VLV_DISPLAY_BASE + 0x721a8) -#define SP_CONST_ALPHA_ENABLE (1 << 31) +#define SP_CONST_ALPHA_ENABLE REG_BIT(31) +#define SP_CONST_ALPHA_MASK REG_GENMASK(7, 0) +#define SP_CONST_ALPHA(alpha) REG_FIELD_PREP(SP_CONST_ALPHA_MASK, (alpha)) #define _SPACLRC0 (VLV_DISPLAY_BASE + 0x721d0) -#define SP_CONTRAST(x) ((x) << 18) /* u3.6 */ -#define SP_BRIGHTNESS(x) ((x) & 0xff) /* s8 */ +#define SP_CONTRAST_MASK REG_GENMASK(26, 18) +#define SP_CONTRAST(x) REG_FIELD_PREP(SP_CONTRAST_MASK, (x)) /* u3.6 */ +#define SP_BRIGHTNESS_MASK REG_GENMASK(7, 0) +#define SP_BRIGHTNESS(x) REG_FIELD_PREP(SP_BRIGHTNESS_MASK, (x)) /* s8 */ #define _SPACLRC1 (VLV_DISPLAY_BASE + 0x721d4) -#define SP_SH_SIN(x) (((x) & 0x7ff) << 16) /* s4.7 */ -#define SP_SH_COS(x) (x) /* u3.7 */ +#define SP_SH_SIN_MASK REG_GENMASK(26, 16) +#define SP_SH_SIN(x) REG_FIELD_PREP(SP_SH_SIN_MASK, (x)) /* s4.7 */ +#define SP_SH_COS_MASK REG_GENMASK(9, 0) +#define SP_SH_COS(x) REG_FIELD_PREP(SP_SH_COS_MASK, (x)) /* u3.7 */ #define _SPAGAMC (VLV_DISPLAY_BASE + 0x721e0) #define _SPBCNTR (VLV_DISPLAY_BASE + 0x72280) @@ -7219,112 +4763,135 @@ enum { #define SPCSCYGOFF(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d900) #define SPCSCCBOFF(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d904) #define SPCSCCROFF(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d908) -#define SPCSC_OOFF(x) (((x) & 0x7ff) << 16) /* s11 */ -#define SPCSC_IOFF(x) (((x) & 0x7ff) << 0) /* s11 */ +#define SPCSC_OOFF_MASK REG_GENMASK(26, 16) +#define SPCSC_OOFF(x) REG_FIELD_PREP(SPCSC_OOFF_MASK, (x) & 0x7ff) /* s11 */ +#define SPCSC_IOFF_MASK REG_GENMASK(10, 0) +#define SPCSC_IOFF(x) REG_FIELD_PREP(SPCSC_IOFF_MASK, (x) & 0x7ff) /* s11 */ #define SPCSCC01(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d90c) #define SPCSCC23(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d910) #define SPCSCC45(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d914) #define SPCSCC67(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d918) #define SPCSCC8(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d91c) -#define SPCSC_C1(x) (((x) & 0x7fff) << 16) /* s3.12 */ -#define SPCSC_C0(x) (((x) & 0x7fff) << 0) /* s3.12 */ +#define SPCSC_C1_MASK REG_GENMASK(30, 16) +#define SPCSC_C1(x) REG_FIELD_PREP(SPCSC_C1_MASK, (x) & 0x7fff) /* s3.12 */ +#define SPCSC_C0_MASK REG_GENMASK(14, 0) +#define SPCSC_C0(x) REG_FIELD_PREP(SPCSC_C0_MASK, (x) & 0x7fff) /* s3.12 */ #define SPCSCYGICLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d920) #define SPCSCCBICLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d924) #define SPCSCCRICLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d928) -#define SPCSC_IMAX(x) (((x) & 0x7ff) << 16) /* s11 */ -#define SPCSC_IMIN(x) (((x) & 0x7ff) << 0) /* s11 */ +#define SPCSC_IMAX_MASK REG_GENMASK(26, 16) +#define SPCSC_IMAX(x) REG_FIELD_PREP(SPCSC_IMAX_MASK, (x) & 0x7ff) /* s11 */ +#define SPCSC_IMIN_MASK REG_GENMASK(10, 0) +#define SPCSC_IMIN(x) REG_FIELD_PREP(SPCSC_IMIN_MASK, (x) & 0x7ff) /* s11 */ #define SPCSCYGOCLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d92c) #define SPCSCCBOCLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d930) #define SPCSCCROCLAMP(plane_id) _MMIO_CHV_SPCSC(plane_id, 0x6d934) -#define SPCSC_OMAX(x) ((x) << 16) /* u10 */ -#define SPCSC_OMIN(x) ((x) << 0) /* u10 */ +#define SPCSC_OMAX_MASK REG_GENMASK(25, 16) +#define SPCSC_OMAX(x) REG_FIELD_PREP(SPCSC_OMAX_MASK, (x)) /* u10 */ +#define SPCSC_OMIN_MASK REG_GENMASK(9, 0) +#define SPCSC_OMIN(x) REG_FIELD_PREP(SPCSC_OMIN_MASK, (x)) /* u10 */ /* Skylake plane registers */ #define _PLANE_CTL_1_A 0x70180 #define _PLANE_CTL_2_A 0x70280 #define _PLANE_CTL_3_A 0x70380 -#define PLANE_CTL_ENABLE (1 << 31) +#define PLANE_CTL_ENABLE REG_BIT(31) #define PLANE_CTL_ARB_SLOTS_MASK REG_GENMASK(30, 28) /* icl+ */ #define PLANE_CTL_ARB_SLOTS(x) REG_FIELD_PREP(PLANE_CTL_ARB_SLOTS_MASK, (x)) /* icl+ */ -#define PLANE_CTL_PIPE_GAMMA_ENABLE (1 << 30) /* Pre-GLK */ -#define PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE (1 << 28) +#define PLANE_CTL_PIPE_GAMMA_ENABLE REG_BIT(30) /* Pre-GLK */ +#define PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE REG_BIT(28) /* * ICL+ uses the same PLANE_CTL_FORMAT bits, but the field definition * expanded to include bit 23 as well. However, the shift-24 based values * correctly map to the same formats in ICL, as long as bit 23 is set to 0 */ -#define PLANE_CTL_FORMAT_MASK (0xf << 24) -#define PLANE_CTL_FORMAT_YUV422 (0 << 24) -#define PLANE_CTL_FORMAT_NV12 (1 << 24) -#define PLANE_CTL_FORMAT_XRGB_2101010 (2 << 24) -#define PLANE_CTL_FORMAT_P010 (3 << 24) -#define PLANE_CTL_FORMAT_XRGB_8888 (4 << 24) -#define PLANE_CTL_FORMAT_P012 (5 << 24) -#define PLANE_CTL_FORMAT_XRGB_16161616F (6 << 24) -#define PLANE_CTL_FORMAT_P016 (7 << 24) -#define PLANE_CTL_FORMAT_XYUV (8 << 24) -#define PLANE_CTL_FORMAT_INDEXED (12 << 24) -#define PLANE_CTL_FORMAT_RGB_565 (14 << 24) -#define ICL_PLANE_CTL_FORMAT_MASK (0x1f << 23) -#define PLANE_CTL_PIPE_CSC_ENABLE (1 << 23) /* Pre-GLK */ -#define PLANE_CTL_FORMAT_Y210 (1 << 23) -#define PLANE_CTL_FORMAT_Y212 (3 << 23) -#define PLANE_CTL_FORMAT_Y216 (5 << 23) -#define PLANE_CTL_FORMAT_Y410 (7 << 23) -#define PLANE_CTL_FORMAT_Y412 (9 << 23) -#define PLANE_CTL_FORMAT_Y416 (0xb << 23) -#define PLANE_CTL_KEY_ENABLE_MASK (0x3 << 21) -#define PLANE_CTL_KEY_ENABLE_SOURCE (1 << 21) -#define PLANE_CTL_KEY_ENABLE_DESTINATION (2 << 21) -#define PLANE_CTL_ORDER_BGRX (0 << 20) -#define PLANE_CTL_ORDER_RGBX (1 << 20) -#define PLANE_CTL_YUV420_Y_PLANE (1 << 19) -#define PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709 (1 << 18) -#define PLANE_CTL_YUV422_ORDER_MASK (0x3 << 16) -#define PLANE_CTL_YUV422_ORDER_YUYV (0 << 16) -#define PLANE_CTL_YUV422_ORDER_UYVY (1 << 16) -#define PLANE_CTL_YUV422_ORDER_YVYU (2 << 16) -#define PLANE_CTL_YUV422_ORDER_VYUY (3 << 16) -#define PLANE_CTL_RENDER_DECOMPRESSION_ENABLE (1 << 15) -#define PLANE_CTL_TRICKLE_FEED_DISABLE (1 << 14) -#define PLANE_CTL_CLEAR_COLOR_DISABLE (1 << 13) /* TGL+ */ -#define PLANE_CTL_PLANE_GAMMA_DISABLE (1 << 13) /* Pre-GLK */ -#define PLANE_CTL_TILED_MASK (0x7 << 10) -#define PLANE_CTL_TILED_LINEAR (0 << 10) -#define PLANE_CTL_TILED_X (1 << 10) -#define PLANE_CTL_TILED_Y (4 << 10) -#define PLANE_CTL_TILED_YF (5 << 10) -#define PLANE_CTL_ASYNC_FLIP (1 << 9) -#define PLANE_CTL_FLIP_HORIZONTAL (1 << 8) -#define PLANE_CTL_MEDIA_DECOMPRESSION_ENABLE (1 << 4) /* TGL+ */ -#define PLANE_CTL_ALPHA_MASK (0x3 << 4) /* Pre-GLK */ -#define PLANE_CTL_ALPHA_DISABLE (0 << 4) -#define PLANE_CTL_ALPHA_SW_PREMULTIPLY (2 << 4) -#define PLANE_CTL_ALPHA_HW_PREMULTIPLY (3 << 4) -#define PLANE_CTL_ROTATE_MASK 0x3 -#define PLANE_CTL_ROTATE_0 0x0 -#define PLANE_CTL_ROTATE_90 0x1 -#define PLANE_CTL_ROTATE_180 0x2 -#define PLANE_CTL_ROTATE_270 0x3 +#define PLANE_CTL_FORMAT_MASK_SKL REG_GENMASK(27, 24) /* pre-icl */ +#define PLANE_CTL_FORMAT_MASK_ICL REG_GENMASK(27, 23) /* icl+ */ +#define PLANE_CTL_FORMAT_YUV422 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 0) +#define PLANE_CTL_FORMAT_NV12 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 1) +#define PLANE_CTL_FORMAT_XRGB_2101010 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 2) +#define PLANE_CTL_FORMAT_P010 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 3) +#define PLANE_CTL_FORMAT_XRGB_8888 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 4) +#define PLANE_CTL_FORMAT_P012 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 5) +#define PLANE_CTL_FORMAT_XRGB_16161616F REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 6) +#define PLANE_CTL_FORMAT_P016 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 7) +#define PLANE_CTL_FORMAT_XYUV REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 8) +#define PLANE_CTL_FORMAT_INDEXED REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 12) +#define PLANE_CTL_FORMAT_RGB_565 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_SKL, 14) +#define PLANE_CTL_FORMAT_Y210 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_ICL, 1) +#define PLANE_CTL_FORMAT_Y212 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_ICL, 3) +#define PLANE_CTL_FORMAT_Y216 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_ICL, 5) +#define PLANE_CTL_FORMAT_Y410 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_ICL, 7) +#define PLANE_CTL_FORMAT_Y412 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_ICL, 9) +#define PLANE_CTL_FORMAT_Y416 REG_FIELD_PREP(PLANE_CTL_FORMAT_MASK_ICL, 11) +#define PLANE_CTL_PIPE_CSC_ENABLE REG_BIT(23) /* Pre-GLK */ +#define PLANE_CTL_KEY_ENABLE_MASK REG_GENMASK(22, 21) +#define PLANE_CTL_KEY_ENABLE_SOURCE REG_FIELD_PREP(PLANE_CTL_KEY_ENABLE_MASK, 1) +#define PLANE_CTL_KEY_ENABLE_DESTINATION REG_FIELD_PREP(PLANE_CTL_KEY_ENABLE_MASK, 2) +#define PLANE_CTL_ORDER_RGBX REG_BIT(20) +#define PLANE_CTL_YUV420_Y_PLANE REG_BIT(19) +#define PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709 REG_BIT(18) +#define PLANE_CTL_YUV422_ORDER_MASK REG_GENMASK(17, 16) +#define PLANE_CTL_YUV422_ORDER_YUYV REG_FIELD_PREP(PLANE_CTL_YUV422_ORDER_MASK, 0) +#define PLANE_CTL_YUV422_ORDER_UYVY REG_FIELD_PREP(PLANE_CTL_YUV422_ORDER_MASK, 1) +#define PLANE_CTL_YUV422_ORDER_YVYU REG_FIELD_PREP(PLANE_CTL_YUV422_ORDER_MASK, 2) +#define PLANE_CTL_YUV422_ORDER_VYUY REG_FIELD_PREP(PLANE_CTL_YUV422_ORDER_MASK, 3) +#define PLANE_CTL_RENDER_DECOMPRESSION_ENABLE REG_BIT(15) +#define PLANE_CTL_TRICKLE_FEED_DISABLE REG_BIT(14) +#define PLANE_CTL_CLEAR_COLOR_DISABLE REG_BIT(13) /* TGL+ */ +#define PLANE_CTL_PLANE_GAMMA_DISABLE REG_BIT(13) /* Pre-GLK */ +#define PLANE_CTL_TILED_MASK REG_GENMASK(12, 10) +#define PLANE_CTL_TILED_LINEAR REG_FIELD_PREP(PLANE_CTL_TILED_MASK, 0) +#define PLANE_CTL_TILED_X REG_FIELD_PREP(PLANE_CTL_TILED_MASK, 1) +#define PLANE_CTL_TILED_Y REG_FIELD_PREP(PLANE_CTL_TILED_MASK, 4) +#define PLANE_CTL_TILED_YF REG_FIELD_PREP(PLANE_CTL_TILED_MASK, 5) +#define PLANE_CTL_ASYNC_FLIP REG_BIT(9) +#define PLANE_CTL_FLIP_HORIZONTAL REG_BIT(8) +#define PLANE_CTL_MEDIA_DECOMPRESSION_ENABLE REG_BIT(4) /* TGL+ */ +#define PLANE_CTL_ALPHA_MASK REG_GENMASK(5, 4) /* Pre-GLK */ +#define PLANE_CTL_ALPHA_DISABLE REG_FIELD_PREP(PLANE_CTL_ALPHA_MASK, 0) +#define PLANE_CTL_ALPHA_SW_PREMULTIPLY REG_FIELD_PREP(PLANE_CTL_ALPHA_MASK, 2) +#define PLANE_CTL_ALPHA_HW_PREMULTIPLY REG_FIELD_PREP(PLANE_CTL_ALPHA_MASK, 3) +#define PLANE_CTL_ROTATE_MASK REG_GENMASK(1, 0) +#define PLANE_CTL_ROTATE_0 REG_FIELD_PREP(PLANE_CTL_ROTATE_MASK, 0) +#define PLANE_CTL_ROTATE_90 REG_FIELD_PREP(PLANE_CTL_ROTATE_MASK, 1) +#define PLANE_CTL_ROTATE_180 REG_FIELD_PREP(PLANE_CTL_ROTATE_MASK, 2) +#define PLANE_CTL_ROTATE_270 REG_FIELD_PREP(PLANE_CTL_ROTATE_MASK, 3) #define _PLANE_STRIDE_1_A 0x70188 #define _PLANE_STRIDE_2_A 0x70288 #define _PLANE_STRIDE_3_A 0x70388 +#define PLANE_STRIDE__MASK REG_GENMASK(11, 0) +#define PLANE_STRIDE_(stride) REG_FIELD_PREP(PLANE_STRIDE__MASK, (stride)) #define _PLANE_POS_1_A 0x7018c #define _PLANE_POS_2_A 0x7028c #define _PLANE_POS_3_A 0x7038c +#define PLANE_POS_Y_MASK REG_GENMASK(31, 16) +#define PLANE_POS_Y(y) REG_FIELD_PREP(PLANE_POS_Y_MASK, (y)) +#define PLANE_POS_X_MASK REG_GENMASK(15, 0) +#define PLANE_POS_X(x) REG_FIELD_PREP(PLANE_POS_X_MASK, (x)) #define _PLANE_SIZE_1_A 0x70190 #define _PLANE_SIZE_2_A 0x70290 #define _PLANE_SIZE_3_A 0x70390 +#define PLANE_HEIGHT_MASK REG_GENMASK(31, 16) +#define PLANE_HEIGHT(h) REG_FIELD_PREP(PLANE_HEIGHT_MASK, (h)) +#define PLANE_WIDTH_MASK REG_GENMASK(15, 0) +#define PLANE_WIDTH(w) REG_FIELD_PREP(PLANE_WIDTH_MASK, (w)) #define _PLANE_SURF_1_A 0x7019c #define _PLANE_SURF_2_A 0x7029c #define _PLANE_SURF_3_A 0x7039c +#define PLANE_SURF_ADDR_MASK REG_GENMASK(31, 12) +#define PLANE_SURF_DECRYPT REG_BIT(2) #define _PLANE_OFFSET_1_A 0x701a4 #define _PLANE_OFFSET_2_A 0x702a4 #define _PLANE_OFFSET_3_A 0x703a4 +#define PLANE_OFFSET_Y_MASK REG_GENMASK(31, 16) +#define PLANE_OFFSET_Y(y) REG_FIELD_PREP(PLANE_OFFSET_Y_MASK, (y)) +#define PLANE_OFFSET_X_MASK REG_GENMASK(15, 0) +#define PLANE_OFFSET_X(x) REG_FIELD_PREP(PLANE_OFFSET_X_MASK, (x)) #define _PLANE_KEYVAL_1_A 0x70194 #define _PLANE_KEYVAL_2_A 0x70294 #define _PLANE_KEYMSK_1_A 0x70198 @@ -7336,42 +4903,49 @@ enum { #define _PLANE_CC_VAL_1_A 0x701b4 #define _PLANE_CC_VAL_2_A 0x702b4 #define _PLANE_AUX_DIST_1_A 0x701c0 +#define PLANE_AUX_DISTANCE_MASK REG_GENMASK(31, 12) +#define PLANE_AUX_STRIDE_MASK REG_GENMASK(11, 0) +#define PLANE_AUX_STRIDE(stride) REG_FIELD_PREP(PLANE_AUX_STRIDE_MASK, (stride)) #define _PLANE_AUX_DIST_2_A 0x702c0 #define _PLANE_AUX_OFFSET_1_A 0x701c4 #define _PLANE_AUX_OFFSET_2_A 0x702c4 #define _PLANE_CUS_CTL_1_A 0x701c8 #define _PLANE_CUS_CTL_2_A 0x702c8 -#define PLANE_CUS_ENABLE (1 << 31) -#define PLANE_CUS_Y_PLANE_4_RKL (0 << 30) -#define PLANE_CUS_Y_PLANE_5_RKL (1 << 30) -#define PLANE_CUS_Y_PLANE_6_ICL (0 << 30) -#define PLANE_CUS_Y_PLANE_7_ICL (1 << 30) -#define PLANE_CUS_HPHASE_SIGN_NEGATIVE (1 << 19) -#define PLANE_CUS_HPHASE_0 (0 << 16) -#define PLANE_CUS_HPHASE_0_25 (1 << 16) -#define PLANE_CUS_HPHASE_0_5 (2 << 16) -#define PLANE_CUS_VPHASE_SIGN_NEGATIVE (1 << 15) -#define PLANE_CUS_VPHASE_0 (0 << 12) -#define PLANE_CUS_VPHASE_0_25 (1 << 12) -#define PLANE_CUS_VPHASE_0_5 (2 << 12) +#define PLANE_CUS_ENABLE REG_BIT(31) +#define PLANE_CUS_Y_PLANE_MASK REG_BIT(30) +#define PLANE_CUS_Y_PLANE_4_RKL REG_FIELD_PREP(PLANE_CUS_Y_PLANE_MASK, 0) +#define PLANE_CUS_Y_PLANE_5_RKL REG_FIELD_PREP(PLANE_CUS_Y_PLANE_MASK, 1) +#define PLANE_CUS_Y_PLANE_6_ICL REG_FIELD_PREP(PLANE_CUS_Y_PLANE_MASK, 0) +#define PLANE_CUS_Y_PLANE_7_ICL REG_FIELD_PREP(PLANE_CUS_Y_PLANE_MASK, 1) +#define PLANE_CUS_HPHASE_SIGN_NEGATIVE REG_BIT(19) +#define PLANE_CUS_HPHASE_MASK REG_GENMASK(17, 16) +#define PLANE_CUS_HPHASE_0 REG_FIELD_PREP(PLANE_CUS_HPHASE_MASK, 0) +#define PLANE_CUS_HPHASE_0_25 REG_FIELD_PREP(PLANE_CUS_HPHASE_MASK, 1) +#define PLANE_CUS_HPHASE_0_5 REG_FIELD_PREP(PLANE_CUS_HPHASE_MASK, 2) +#define PLANE_CUS_VPHASE_SIGN_NEGATIVE REG_BIT(15) +#define PLANE_CUS_VPHASE_MASK REG_GENMASK(13, 12) +#define PLANE_CUS_VPHASE_0 REG_FIELD_PREP(PLANE_CUS_VPHASE_MASK, 0) +#define PLANE_CUS_VPHASE_0_25 REG_FIELD_PREP(PLANE_CUS_VPHASE_MASK, 1) +#define PLANE_CUS_VPHASE_0_5 REG_FIELD_PREP(PLANE_CUS_VPHASE_MASK, 2) #define _PLANE_COLOR_CTL_1_A 0x701CC /* GLK+ */ #define _PLANE_COLOR_CTL_2_A 0x702CC /* GLK+ */ #define _PLANE_COLOR_CTL_3_A 0x703CC /* GLK+ */ -#define PLANE_COLOR_PIPE_GAMMA_ENABLE (1 << 30) /* Pre-ICL */ -#define PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE (1 << 28) +#define PLANE_COLOR_PIPE_GAMMA_ENABLE REG_BIT(30) /* Pre-ICL */ +#define PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE REG_BIT(28) +#define PLANE_COLOR_PIPE_CSC_ENABLE REG_BIT(23) /* Pre-ICL */ #define PLANE_COLOR_PLANE_CSC_ENABLE REG_BIT(21) /* ICL+ */ -#define PLANE_COLOR_INPUT_CSC_ENABLE (1 << 20) /* ICL+ */ -#define PLANE_COLOR_PIPE_CSC_ENABLE (1 << 23) /* Pre-ICL */ -#define PLANE_COLOR_CSC_MODE_BYPASS (0 << 17) -#define PLANE_COLOR_CSC_MODE_YUV601_TO_RGB601 (1 << 17) -#define PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709 (2 << 17) -#define PLANE_COLOR_CSC_MODE_YUV2020_TO_RGB2020 (3 << 17) -#define PLANE_COLOR_CSC_MODE_RGB709_TO_RGB2020 (4 << 17) -#define PLANE_COLOR_PLANE_GAMMA_DISABLE (1 << 13) -#define PLANE_COLOR_ALPHA_MASK (0x3 << 4) -#define PLANE_COLOR_ALPHA_DISABLE (0 << 4) -#define PLANE_COLOR_ALPHA_SW_PREMULTIPLY (2 << 4) -#define PLANE_COLOR_ALPHA_HW_PREMULTIPLY (3 << 4) +#define PLANE_COLOR_INPUT_CSC_ENABLE REG_BIT(20) /* ICL+ */ +#define PLANE_COLOR_CSC_MODE_MASK REG_GENMASK(19, 17) +#define PLANE_COLOR_CSC_MODE_BYPASS REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 0) +#define PLANE_COLOR_CSC_MODE_YUV601_TO_RGB601 REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 1) +#define PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709 REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 2) +#define PLANE_COLOR_CSC_MODE_YUV2020_TO_RGB2020 REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 3) +#define PLANE_COLOR_CSC_MODE_RGB709_TO_RGB2020 REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 4) +#define PLANE_COLOR_PLANE_GAMMA_DISABLE REG_BIT(13) +#define PLANE_COLOR_ALPHA_MASK REG_GENMASK(5, 4) +#define PLANE_COLOR_ALPHA_DISABLE REG_FIELD_PREP(PLANE_COLOR_ALPHA_MASK, 0) +#define PLANE_COLOR_ALPHA_SW_PREMULTIPLY REG_FIELD_PREP(PLANE_COLOR_ALPHA_MASK, 2) +#define PLANE_COLOR_ALPHA_HW_PREMULTIPLY REG_FIELD_PREP(PLANE_COLOR_ALPHA_MASK, 3) #define _PLANE_BUF_CFG_1_A 0x7027c #define _PLANE_BUF_CFG_2_A 0x7037c #define _PLANE_NV12_BUF_CFG_1_A 0x70278 @@ -7454,8 +5028,6 @@ enum { _PIPE(pipe, _PLANE_STRIDE_3_A, _PLANE_STRIDE_3_B) #define PLANE_STRIDE(pipe, plane) \ _MMIO_PLANE(plane, _PLANE_STRIDE_1(pipe), _PLANE_STRIDE_2(pipe)) -#define PLANE_STRIDE_MASK REG_GENMASK(10, 0) -#define PLANE_STRIDE_MASK_XELPD REG_GENMASK(11, 0) #define _PLANE_POS_1_B 0x7118c #define _PLANE_POS_2_B 0x7128c @@ -7483,7 +5055,6 @@ enum { #define _PLANE_SURF_3(pipe) _PIPE(pipe, _PLANE_SURF_3_A, _PLANE_SURF_3_B) #define PLANE_SURF(pipe, plane) \ _MMIO_PLANE(plane, _PLANE_SURF_1(pipe), _PLANE_SURF_2(pipe)) -#define PLANE_SURF_DECRYPT REG_BIT(2) #define _PLANE_OFFSET_1_B 0x711a4 #define _PLANE_OFFSET_2_B 0x712a4 @@ -7515,8 +5086,11 @@ enum { #define _PLANE_BUF_CFG_1_B 0x7127c #define _PLANE_BUF_CFG_2_B 0x7137c -#define DDB_ENTRY_MASK 0xFFF /* skl+: 10 bits, icl+ 11 bits, adlp+ 12 bits */ -#define DDB_ENTRY_END_SHIFT 16 +/* skl+: 10 bits, icl+ 11 bits, adlp+ 12 bits */ +#define PLANE_BUF_END_MASK REG_GENMASK(27, 16) +#define PLANE_BUF_END(end) REG_FIELD_PREP(PLANE_BUF_END_MASK, (end)) +#define PLANE_BUF_START_MASK REG_GENMASK(11, 0) +#define PLANE_BUF_START(start) REG_FIELD_PREP(PLANE_BUF_START_MASK, (start)) #define _PLANE_BUF_CFG_1(pipe) \ _PIPE(pipe, _PLANE_BUF_CFG_1_A, _PLANE_BUF_CFG_1_B) #define _PLANE_BUF_CFG_2(pipe) \ @@ -7671,24 +5245,13 @@ enum { #define _PIPEA_DATA_M1 0x60030 -#define PIPE_DATA_M1_OFFSET 0 #define _PIPEA_DATA_N1 0x60034 -#define PIPE_DATA_N1_OFFSET 0 - #define _PIPEA_DATA_M2 0x60038 -#define PIPE_DATA_M2_OFFSET 0 #define _PIPEA_DATA_N2 0x6003c -#define PIPE_DATA_N2_OFFSET 0 - #define _PIPEA_LINK_M1 0x60040 -#define PIPE_LINK_M1_OFFSET 0 #define _PIPEA_LINK_N1 0x60044 -#define PIPE_LINK_N1_OFFSET 0 - #define _PIPEA_LINK_M2 0x60048 -#define PIPE_LINK_M2_OFFSET 0 #define _PIPEA_LINK_N2 0x6004c -#define PIPE_LINK_N2_OFFSET 0 /* PIPEB timing regs are same start from 0x61000 */ @@ -7945,7 +5508,8 @@ enum { #define TGL_DMC_DEBUG_DC6_COUNT _MMIO(0x101088) #define DG1_DMC_DEBUG_DC5_COUNT _MMIO(0x134154) -#define DMC_DEBUG3 _MMIO(0x101090) +#define TGL_DMC_DEBUG3 _MMIO(0x101090) +#define DG1_DMC_DEBUG3 _MMIO(0x13415c) /* Display Internal Timeout Register */ #define RM_TIMEOUT _MMIO(0x42060) @@ -8200,63 +5764,6 @@ enum { #define GEN11_HOTPLUG_CTL_SHORT_DETECT(hpd_pin) (1 << (_HPD_PIN_TC(hpd_pin) * 4)) #define GEN11_HOTPLUG_CTL_NO_DETECT(hpd_pin) (0 << (_HPD_PIN_TC(hpd_pin) * 4)) -#define GEN11_GT_INTR_DW0 _MMIO(0x190018) -#define GEN11_CSME (31) -#define GEN11_GUNIT (28) -#define GEN11_GUC (25) -#define GEN11_WDPERF (20) -#define GEN11_KCR (19) -#define GEN11_GTPM (16) -#define GEN11_BCS (15) -#define GEN11_RCS0 (0) - -#define GEN11_GT_INTR_DW1 _MMIO(0x19001c) -#define GEN11_VECS(x) (31 - (x)) -#define GEN11_VCS(x) (x) - -#define GEN11_GT_INTR_DW(x) _MMIO(0x190018 + ((x) * 4)) - -#define GEN11_INTR_IDENTITY_REG0 _MMIO(0x190060) -#define GEN11_INTR_IDENTITY_REG1 _MMIO(0x190064) -#define GEN11_INTR_DATA_VALID (1 << 31) -#define GEN11_INTR_ENGINE_CLASS(x) (((x) & GENMASK(18, 16)) >> 16) -#define GEN11_INTR_ENGINE_INSTANCE(x) (((x) & GENMASK(25, 20)) >> 20) -#define GEN11_INTR_ENGINE_INTR(x) ((x) & 0xffff) -/* irq instances for OTHER_CLASS */ -#define OTHER_GUC_INSTANCE 0 -#define OTHER_GTPM_INSTANCE 1 -#define OTHER_KCR_INSTANCE 4 - -#define GEN11_INTR_IDENTITY_REG(x) _MMIO(0x190060 + ((x) * 4)) - -#define GEN11_IIR_REG0_SELECTOR _MMIO(0x190070) -#define GEN11_IIR_REG1_SELECTOR _MMIO(0x190074) - -#define GEN11_IIR_REG_SELECTOR(x) _MMIO(0x190070 + ((x) * 4)) - -#define GEN11_RENDER_COPY_INTR_ENABLE _MMIO(0x190030) -#define GEN11_VCS_VECS_INTR_ENABLE _MMIO(0x190034) -#define GEN11_GUC_SG_INTR_ENABLE _MMIO(0x190038) -#define GEN11_GPM_WGBOXPERF_INTR_ENABLE _MMIO(0x19003c) -#define GEN11_CRYPTO_RSVD_INTR_ENABLE _MMIO(0x190040) -#define GEN11_GUNIT_CSME_INTR_ENABLE _MMIO(0x190044) - -#define GEN11_RCS0_RSVD_INTR_MASK _MMIO(0x190090) -#define GEN11_BCS_RSVD_INTR_MASK _MMIO(0x1900a0) -#define GEN11_VCS0_VCS1_INTR_MASK _MMIO(0x1900a8) -#define GEN11_VCS2_VCS3_INTR_MASK _MMIO(0x1900ac) -#define GEN12_VCS4_VCS5_INTR_MASK _MMIO(0x1900b0) -#define GEN12_VCS6_VCS7_INTR_MASK _MMIO(0x1900b4) -#define GEN11_VECS0_VECS1_INTR_MASK _MMIO(0x1900d0) -#define GEN12_VECS2_VECS3_INTR_MASK _MMIO(0x1900d4) -#define GEN11_GUC_SG_INTR_MASK _MMIO(0x1900e8) -#define GEN11_GPM_WGBOXPERF_INTR_MASK _MMIO(0x1900ec) -#define GEN11_CRYPTO_RSVD_INTR_MASK _MMIO(0x1900f0) -#define GEN11_GUNIT_CSME_INTR_MASK _MMIO(0x1900f4) - -#define ENGINE1_MASK REG_GENMASK(31, 16) -#define ENGINE0_MASK REG_GENMASK(15, 0) - #define ILK_DISPLAY_CHICKEN2 _MMIO(0x42004) /* Required on all Ironlake and Sandybridge according to the B-Spec. */ #define ILK_ELPIN_409_SELECT (1 << 25) @@ -8410,11 +5917,14 @@ enum { #define HSW_NDE_RSTWRN_OPT _MMIO(0x46408) #define RESET_PCH_HANDSHAKE_ENABLE (1 << 4) -#define GEN8_CHICKEN_DCPR_1 _MMIO(0x46430) -#define SKL_SELECT_ALTERNATE_DC_EXIT REG_BIT(30) -#define ICL_DELAY_PMRSP REG_BIT(22) -#define DISABLE_FLR_SRC REG_BIT(15) -#define MASK_WAKEMEM REG_BIT(13) +#define GEN8_CHICKEN_DCPR_1 _MMIO(0x46430) +#define SKL_SELECT_ALTERNATE_DC_EXIT REG_BIT(30) +#define LATENCY_REPORTING_REMOVED_PIPE_C REG_BIT(25) +#define LATENCY_REPORTING_REMOVED_PIPE_B REG_BIT(24) +#define LATENCY_REPORTING_REMOVED_PIPE_A REG_BIT(23) +#define ICL_DELAY_PMRSP REG_BIT(22) +#define DISABLE_FLR_SRC REG_BIT(15) +#define MASK_WAKEMEM REG_BIT(13) #define GEN11_CHICKEN_DCPR_2 _MMIO(0x46434) #define DCPR_MASK_MAXLATENCY_MEMUP_CLR REG_BIT(27) @@ -8443,142 +5953,6 @@ enum { #define ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz (1 << 29) #define ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz (2 << 29) -#define GEN7_FF_SLICE_CS_CHICKEN1 _MMIO(0x20e0) -#define GEN9_FFSC_PERCTX_PREEMPT_CTRL (1 << 14) - -#define FF_SLICE_CS_CHICKEN2 _MMIO(0x20e4) -#define GEN9_TSG_BARRIER_ACK_DISABLE (1 << 8) -#define GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE (1 << 10) - -#define GEN9_CS_DEBUG_MODE1 _MMIO(0x20ec) -#define FF_DOP_CLOCK_GATE_DISABLE REG_BIT(1) -#define GEN9_CTX_PREEMPT_REG _MMIO(0x2248) -#define GEN12_DISABLE_POSH_BUSY_FF_DOP_CG REG_BIT(11) - -#define GEN12_CS_DEBUG_MODE1_CCCSUNIT_BE_COMMON _MMIO(0x20EC) -#define GEN12_REPLAY_MODE_GRANULARITY REG_BIT(0) - -#define GEN8_CS_CHICKEN1 _MMIO(0x2580) -#define GEN9_PREEMPT_3D_OBJECT_LEVEL (1 << 0) -#define GEN9_PREEMPT_GPGPU_LEVEL(hi, lo) (((hi) << 2) | ((lo) << 1)) -#define GEN9_PREEMPT_GPGPU_MID_THREAD_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(0, 0) -#define GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(0, 1) -#define GEN9_PREEMPT_GPGPU_COMMAND_LEVEL GEN9_PREEMPT_GPGPU_LEVEL(1, 0) -#define GEN9_PREEMPT_GPGPU_LEVEL_MASK GEN9_PREEMPT_GPGPU_LEVEL(1, 1) - -/* GEN7 chicken */ -#define GEN7_COMMON_SLICE_CHICKEN1 _MMIO(0x7010) - #define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC (1 << 10) - #define GEN9_RHWO_OPTIMIZATION_DISABLE (1 << 14) - -#define COMMON_SLICE_CHICKEN2 _MMIO(0x7014) - #define GEN9_PBE_COMPRESSED_HASH_SELECTION (1 << 13) - #define GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE (1 << 12) - #define GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION (1 << 8) - #define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE (1 << 0) - -#define GEN8_L3CNTLREG _MMIO(0x7034) - #define GEN8_ERRDETBCTRL (1 << 9) - -#define GEN11_COMMON_SLICE_CHICKEN3 _MMIO(0x7304) -#define DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN REG_BIT(12) -#define XEHP_DUAL_SIMD8_SEQ_MERGE_DISABLE REG_BIT(12) -#define GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC REG_BIT(11) -#define GEN12_DISABLE_CPS_AWARE_COLOR_PIPE REG_BIT(9) - -#define HIZ_CHICKEN _MMIO(0x7018) -# define CHV_HZ_8X8_MODE_IN_1X REG_BIT(15) -# define DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE REG_BIT(14) -# define BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE REG_BIT(3) - -#define GEN9_SLICE_COMMON_ECO_CHICKEN0 _MMIO(0x7308) -#define DISABLE_PIXEL_MASK_CAMMING (1 << 14) - -#define GEN9_SLICE_COMMON_ECO_CHICKEN1 _MMIO(0x731c) -#define GEN11_STATE_CACHE_REDIRECT_TO_CS (1 << 11) - -#define GEN7_SARCHKMD _MMIO(0xB000) -#define GEN7_DISABLE_DEMAND_PREFETCH (1 << 31) -#define GEN7_DISABLE_SAMPLER_PREFETCH (1 << 30) - -#define GEN7_L3SQCREG1 _MMIO(0xB010) -#define VLV_B0_WA_L3SQCREG1_VALUE 0x00D30000 - -#define GEN8_L3SQCREG1 _MMIO(0xB100) -/* - * Note that on CHV the following has an off-by-one error wrt. to BSpec. - * Using the formula in BSpec leads to a hang, while the formula here works - * fine and matches the formulas for all other platforms. A BSpec change - * request has been filed to clarify this. - */ -#define L3_GENERAL_PRIO_CREDITS(x) (((x) >> 1) << 19) -#define L3_HIGH_PRIO_CREDITS(x) (((x) >> 1) << 14) -#define L3_PRIO_CREDITS_MASK ((0x1f << 19) | (0x1f << 14)) - -#define GEN7_L3CNTLREG1 _MMIO(0xB01C) -#define GEN7_WA_FOR_GEN7_L3_CONTROL 0x3C47FF8C -#define GEN7_L3AGDIS (1 << 19) -#define GEN7_L3CNTLREG2 _MMIO(0xB020) -#define GEN7_L3CNTLREG3 _MMIO(0xB024) - -#define GEN7_L3_CHICKEN_MODE_REGISTER _MMIO(0xB030) -#define GEN7_WA_L3_CHICKEN_MODE 0x20000000 -#define GEN10_L3_CHICKEN_MODE_REGISTER _MMIO(0xB114) -#define GEN11_I2M_WRITE_DISABLE (1 << 28) - -#define GEN7_L3SQCREG4 _MMIO(0xb034) -#define L3SQ_URB_READ_CAM_MATCH_DISABLE (1 << 27) - -#define GEN11_SCRATCH2 _MMIO(0xb140) -#define GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE (1 << 19) - -#define GEN8_L3SQCREG4 _MMIO(0xb118) -#define GEN11_LQSC_CLEAN_EVICT_DISABLE (1 << 6) -#define GEN8_LQSC_RO_PERF_DIS (1 << 27) -#define GEN8_LQSC_FLUSH_COHERENT_LINES (1 << 21) -#define GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE REG_BIT(22) - -#define GEN11_L3SQCREG5 _MMIO(0xb158) -#define L3_PWM_TIMER_INIT_VAL_MASK REG_GENMASK(9, 0) - -#define XEHP_L3SCQREG7 _MMIO(0xb188) -#define BLEND_FILL_CACHING_OPT_DIS REG_BIT(3) - -/* GEN8 chicken */ -#define HDC_CHICKEN0 _MMIO(0x7300) -#define ICL_HDC_MODE _MMIO(0xE5F4) -#define HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE (1 << 15) -#define HDC_FENCE_DEST_SLM_DISABLE (1 << 14) -#define HDC_DONOT_FETCH_MEM_WHEN_MASKED (1 << 11) -#define HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT (1 << 5) -#define HDC_FORCE_NON_COHERENT (1 << 4) -#define HDC_BARRIER_PERFORMANCE_DISABLE (1 << 10) - -#define GEN12_HDC_CHICKEN0 _MMIO(0xE5F0) -#define LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK REG_GENMASK(13, 11) - -#define SARB_CHICKEN1 _MMIO(0xe90c) -#define COMP_CKN_IN REG_GENMASK(30, 29) - -#define GEN8_HDC_CHICKEN1 _MMIO(0x7304) - -/* GEN9 chicken */ -#define SLICE_ECO_CHICKEN0 _MMIO(0x7308) -#define PIXEL_MASK_CAMMING_DISABLE (1 << 14) - -#define GEN9_WM_CHICKEN3 _MMIO(0x5588) -#define GEN9_FACTOR_IN_CLR_VAL_HIZ (1 << 9) - -/* WaCatErrorRejectionIssue */ -#define GEN7_SQ_CHICKEN_MBCUNIT_CONFIG _MMIO(0x9030) -#define GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB (1 << 11) - -#define HSW_SCRATCH1 _MMIO(0xb038) -#define HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE (1 << 27) - -#define BDW_SCRATCH1 _MMIO(0xb11c) -#define GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE (1 << 2) - /*GEN11 chicken */ #define _PIPEA_CHICKEN 0x70038 #define _PIPEB_CHICKEN 0x71038 @@ -8591,16 +5965,6 @@ enum { #define DG2_RENDER_CCSTAG_4_3_EN REG_BIT(12) #define PER_PIXEL_ALPHA_BYPASS_EN REG_BIT(7) -#define VFLSKPD _MMIO(0x62a8) -#define DIS_OVER_FETCH_CACHE REG_BIT(1) -#define DIS_MULT_MISS_RD_SQUASH REG_BIT(0) - -#define FF_MODE2 _MMIO(0x6604) -#define FF_MODE2_GS_TIMER_MASK REG_GENMASK(31, 24) -#define FF_MODE2_GS_TIMER_224 REG_FIELD_PREP(FF_MODE2_GS_TIMER_MASK, 224) -#define FF_MODE2_TDS_TIMER_MASK REG_GENMASK(23, 16) -#define FF_MODE2_TDS_TIMER_128 REG_FIELD_PREP(FF_MODE2_TDS_TIMER_MASK, 4) - /* PCH */ #define PCH_DISPLAY_BASE 0xc0000u @@ -8697,6 +6061,7 @@ enum { /* south display engine interrupt: ICP/TGP */ #define SDE_GMBUS_ICP (1 << 23) #define SDE_TC_HOTPLUG_ICP(hpd_pin) REG_BIT(24 + _HPD_PIN_TC(hpd_pin)) +#define SDE_TC_HOTPLUG_DG2(hpd_pin) REG_BIT(25 + _HPD_PIN_TC(hpd_pin)) /* sigh */ #define SDE_DDI_HOTPLUG_ICP(hpd_pin) REG_BIT(16 + _HPD_PIN_DDI(hpd_pin)) #define SDE_DDI_HOTPLUG_MASK_ICP (SDE_DDI_HOTPLUG_ICP(HPD_PORT_D) | \ SDE_DDI_HOTPLUG_ICP(HPD_PORT_C) | \ @@ -9015,22 +6380,19 @@ enum { #define _PCH_TRANSBCONF 0xf1008 #define PCH_TRANSCONF(pipe) _MMIO_PIPE(pipe, _PCH_TRANSACONF, _PCH_TRANSBCONF) #define LPT_TRANSCONF PCH_TRANSCONF(PIPE_A) /* lpt has only one transcoder */ -#define TRANS_DISABLE (0 << 31) -#define TRANS_ENABLE (1 << 31) -#define TRANS_STATE_MASK (1 << 30) -#define TRANS_STATE_DISABLE (0 << 30) -#define TRANS_STATE_ENABLE (1 << 30) -#define TRANS_FRAME_START_DELAY_MASK (3 << 27) /* ibx */ -#define TRANS_FRAME_START_DELAY(x) ((x) << 27) /* ibx: 0-3 */ -#define TRANS_INTERLACE_MASK (7 << 21) -#define TRANS_PROGRESSIVE (0 << 21) -#define TRANS_INTERLACED (3 << 21) -#define TRANS_LEGACY_INTERLACED_ILK (2 << 21) -#define TRANS_8BPC (0 << 5) -#define TRANS_10BPC (1 << 5) -#define TRANS_6BPC (2 << 5) -#define TRANS_12BPC (3 << 5) - +#define TRANS_ENABLE REG_BIT(31) +#define TRANS_STATE_ENABLE REG_BIT(30) +#define TRANS_FRAME_START_DELAY_MASK REG_GENMASK(28, 27) /* ibx */ +#define TRANS_FRAME_START_DELAY(x) REG_FIELD_PREP(TRANS_FRAME_START_DELAY_MASK, (x)) /* ibx: 0-3 */ +#define TRANS_INTERLACE_MASK REG_GENMASK(23, 21) +#define TRANS_INTERLACE_PROGRESSIVE REG_FIELD_PREP(TRANS_INTERLACE_MASK, 0) +#define TRANS_INTERLACE_LEGACY_VSYNC_IBX REG_FIELD_PREP(TRANS_INTERLACE_MASK, 2) /* ibx */ +#define TRANS_INTERLACE_INTERLACED REG_FIELD_PREP(TRANS_INTERLACE_MASK, 3) +#define TRANS_BPC_MASK REG_GENMASK(7, 5) /* ibx */ +#define TRANS_BPC_8 REG_FIELD_PREP(TRANS_BPC_MASK, 0) +#define TRANS_BPC_10 REG_FIELD_PREP(TRANS_BPC_MASK, 1) +#define TRANS_BPC_6 REG_FIELD_PREP(TRANS_BPC_MASK, 2) +#define TRANS_BPC_12 REG_FIELD_PREP(TRANS_BPC_MASK, 3) #define _TRANSA_CHICKEN1 0xf0060 #define _TRANSB_CHICKEN1 0xf1060 #define TRANS_CHICKEN1(pipe) _MMIO_PIPE(pipe, _TRANSA_CHICKEN1, _TRANSB_CHICKEN1) @@ -9240,22 +6602,19 @@ enum { #define _TRANS_DP_CTL_B 0xe1300 #define _TRANS_DP_CTL_C 0xe2300 #define TRANS_DP_CTL(pipe) _MMIO_PIPE(pipe, _TRANS_DP_CTL_A, _TRANS_DP_CTL_B) -#define TRANS_DP_OUTPUT_ENABLE (1 << 31) -#define TRANS_DP_PORT_SEL_MASK (3 << 29) -#define TRANS_DP_PORT_SEL_NONE (3 << 29) -#define TRANS_DP_PORT_SEL(port) (((port) - PORT_B) << 29) -#define TRANS_DP_AUDIO_ONLY (1 << 26) -#define TRANS_DP_ENH_FRAMING (1 << 18) -#define TRANS_DP_8BPC (0 << 9) -#define TRANS_DP_10BPC (1 << 9) -#define TRANS_DP_6BPC (2 << 9) -#define TRANS_DP_12BPC (3 << 9) -#define TRANS_DP_BPC_MASK (3 << 9) -#define TRANS_DP_VSYNC_ACTIVE_HIGH (1 << 4) -#define TRANS_DP_VSYNC_ACTIVE_LOW 0 -#define TRANS_DP_HSYNC_ACTIVE_HIGH (1 << 3) -#define TRANS_DP_HSYNC_ACTIVE_LOW 0 -#define TRANS_DP_SYNC_MASK (3 << 3) +#define TRANS_DP_OUTPUT_ENABLE REG_BIT(31) +#define TRANS_DP_PORT_SEL_MASK REG_GENMASK(30, 29) +#define TRANS_DP_PORT_SEL_NONE REG_FIELD_PREP(TRANS_DP_PORT_SEL_MASK, 3) +#define TRANS_DP_PORT_SEL(port) REG_FIELD_PREP(TRANS_DP_PORT_SEL_MASK, (port) - PORT_B) +#define TRANS_DP_AUDIO_ONLY REG_BIT(26) +#define TRANS_DP_ENH_FRAMING REG_BIT(18) +#define TRANS_DP_BPC_MASK REG_GENMASK(10, 9) +#define TRANS_DP_BPC_8 REG_FIELD_PREP(TRANS_DP_BPC_MASK, 0) +#define TRANS_DP_BPC_10 REG_FIELD_PREP(TRANS_DP_BPC_MASK, 1) +#define TRANS_DP_BPC_6 REG_FIELD_PREP(TRANS_DP_BPC_MASK, 2) +#define TRANS_DP_BPC_12 REG_FIELD_PREP(TRANS_DP_BPC_MASK, 3) +#define TRANS_DP_VSYNC_ACTIVE_HIGH REG_BIT(4) +#define TRANS_DP_HSYNC_ACTIVE_HIGH REG_BIT(3) #define _TRANS_DP2_CTL_A 0x600a0 #define _TRANS_DP2_CTL_B 0x610a0 @@ -9314,261 +6673,16 @@ enum { #define VLV_PMWGICZ _MMIO(0x1300a4) -#define RC6_LOCATION _MMIO(0xD40) -#define RC6_CTX_IN_DRAM (1 << 0) -#define RC6_CTX_BASE _MMIO(0xD48) -#define RC6_CTX_BASE_MASK 0xFFFFFFF0 -#define PWRCTX_MAXCNT_RCSUNIT _MMIO(0x2054) -#define PWRCTX_MAXCNT_VCSUNIT0 _MMIO(0x12054) -#define PWRCTX_MAXCNT_BCSUNIT _MMIO(0x22054) -#define PWRCTX_MAXCNT_VECSUNIT _MMIO(0x1A054) -#define PWRCTX_MAXCNT_VCSUNIT1 _MMIO(0x1C054) -#define IDLE_TIME_MASK 0xFFFFF -#define FORCEWAKE _MMIO(0xA18C) -#define FORCEWAKE_VLV _MMIO(0x1300b0) -#define FORCEWAKE_ACK_VLV _MMIO(0x1300b4) -#define FORCEWAKE_MEDIA_VLV _MMIO(0x1300b8) -#define FORCEWAKE_ACK_MEDIA_VLV _MMIO(0x1300bc) -#define FORCEWAKE_ACK_HSW _MMIO(0x130044) -#define FORCEWAKE_ACK _MMIO(0x130090) -#define VLV_GTLC_WAKE_CTRL _MMIO(0x130090) -#define VLV_GTLC_RENDER_CTX_EXISTS (1 << 25) -#define VLV_GTLC_MEDIA_CTX_EXISTS (1 << 24) -#define VLV_GTLC_ALLOWWAKEREQ (1 << 0) - -#define VLV_GTLC_PW_STATUS _MMIO(0x130094) -#define VLV_GTLC_ALLOWWAKEACK (1 << 0) -#define VLV_GTLC_ALLOWWAKEERR (1 << 1) -#define VLV_GTLC_PW_MEDIA_STATUS_MASK (1 << 5) -#define VLV_GTLC_PW_RENDER_STATUS_MASK (1 << 7) -#define FORCEWAKE_MT _MMIO(0xa188) /* multi-threaded */ -#define FORCEWAKE_MEDIA_GEN9 _MMIO(0xa270) -#define FORCEWAKE_MEDIA_VDBOX_GEN11(n) _MMIO(0xa540 + (n) * 4) -#define FORCEWAKE_MEDIA_VEBOX_GEN11(n) _MMIO(0xa560 + (n) * 4) -#define FORCEWAKE_RENDER_GEN9 _MMIO(0xa278) -#define FORCEWAKE_GT_GEN9 _MMIO(0xa188) -#define FORCEWAKE_ACK_MEDIA_GEN9 _MMIO(0x0D88) -#define FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(n) _MMIO(0x0D50 + (n) * 4) -#define FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(n) _MMIO(0x0D70 + (n) * 4) -#define FORCEWAKE_ACK_RENDER_GEN9 _MMIO(0x0D84) -#define FORCEWAKE_ACK_GT_GEN9 _MMIO(0x130044) -#define FORCEWAKE_KERNEL BIT(0) -#define FORCEWAKE_USER BIT(1) -#define FORCEWAKE_KERNEL_FALLBACK BIT(15) -#define FORCEWAKE_MT_ACK _MMIO(0x130040) -#define ECOBUS _MMIO(0xa180) -#define FORCEWAKE_MT_ENABLE (1 << 5) -#define VLV_SPAREG2H _MMIO(0xA194) -#define GEN9_PWRGT_DOMAIN_STATUS _MMIO(0xA2A0) -#define GEN9_PWRGT_MEDIA_STATUS_MASK (1 << 0) -#define GEN9_PWRGT_RENDER_STATUS_MASK (1 << 1) - -#define GTFIFODBG _MMIO(0x120000) -#define GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV (0x1f << 20) -#define GT_FIFO_FREE_ENTRIES_CHV (0x7f << 13) -#define GT_FIFO_SBDROPERR (1 << 6) -#define GT_FIFO_BLOBDROPERR (1 << 5) -#define GT_FIFO_SB_READ_ABORTERR (1 << 4) -#define GT_FIFO_DROPERR (1 << 3) -#define GT_FIFO_OVFERR (1 << 2) -#define GT_FIFO_IAWRERR (1 << 1) -#define GT_FIFO_IARDERR (1 << 0) - -#define GTFIFOCTL _MMIO(0x120008) -#define GT_FIFO_FREE_ENTRIES_MASK 0x7f -#define GT_FIFO_NUM_RESERVED_ENTRIES 20 -#define GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL (1 << 12) -#define GT_FIFO_CTL_RC6_POLICY_STALL (1 << 11) - -#define HSW_IDICR _MMIO(0x9008) -#define IDIHASHMSK(x) (((x) & 0x3f) << 16) #define HSW_EDRAM_CAP _MMIO(0x120010) #define EDRAM_ENABLED 0x1 #define EDRAM_NUM_BANKS(cap) (((cap) >> 1) & 0xf) #define EDRAM_WAYS_IDX(cap) (((cap) >> 5) & 0x7) #define EDRAM_SETS_IDX(cap) (((cap) >> 8) & 0x3) -#define GEN6_UCGCTL1 _MMIO(0x9400) -# define GEN6_GAMUNIT_CLOCK_GATE_DISABLE (1 << 22) -# define GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE (1 << 16) -# define GEN6_BLBUNIT_CLOCK_GATE_DISABLE (1 << 5) -# define GEN6_CSUNIT_CLOCK_GATE_DISABLE (1 << 7) - -#define GEN6_UCGCTL2 _MMIO(0x9404) -# define GEN6_VFUNIT_CLOCK_GATE_DISABLE (1 << 31) -# define GEN7_VDSUNIT_CLOCK_GATE_DISABLE (1 << 30) -# define GEN7_TDLUNIT_CLOCK_GATE_DISABLE (1 << 22) -# define GEN6_RCZUNIT_CLOCK_GATE_DISABLE (1 << 13) -# define GEN6_RCPBUNIT_CLOCK_GATE_DISABLE (1 << 12) -# define GEN6_RCCUNIT_CLOCK_GATE_DISABLE (1 << 11) - -#define GEN6_UCGCTL3 _MMIO(0x9408) -# define GEN6_OACSUNIT_CLOCK_GATE_DISABLE (1 << 20) - -#define GEN7_UCGCTL4 _MMIO(0x940c) -#define GEN7_L3BANK2X_CLOCK_GATE_DISABLE (1 << 25) -#define GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE (1 << 14) - -#define GEN6_RCGCTL1 _MMIO(0x9410) -#define GEN6_RCGCTL2 _MMIO(0x9414) -#define GEN6_RSTCTL _MMIO(0x9420) - -#define GEN8_UCGCTL6 _MMIO(0x9430) -#define GEN8_GAPSUNIT_CLOCK_GATE_DISABLE (1 << 24) -#define GEN8_SDEUNIT_CLOCK_GATE_DISABLE (1 << 14) -#define GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ (1 << 28) - -#define UNSLCGCTL9430 _MMIO(0x9430) -#define MSQDUNIT_CLKGATE_DIS REG_BIT(3) - -#define GEN6_GFXPAUSE _MMIO(0xA000) -#define GEN6_RPNSWREQ _MMIO(0xA008) -#define GEN6_TURBO_DISABLE (1 << 31) -#define GEN6_FREQUENCY(x) ((x) << 25) -#define HSW_FREQUENCY(x) ((x) << 24) -#define GEN9_FREQUENCY(x) ((x) << 23) -#define GEN6_OFFSET(x) ((x) << 19) -#define GEN6_AGGRESSIVE_TURBO (0 << 15) -#define GEN9_SW_REQ_UNSLICE_RATIO_SHIFT 23 -#define GEN9_IGNORE_SLICE_RATIO (0 << 0) - -#define GEN6_RC_VIDEO_FREQ _MMIO(0xA00C) -#define GEN6_RC_CONTROL _MMIO(0xA090) -#define GEN6_RC_CTL_RC6pp_ENABLE (1 << 16) -#define GEN6_RC_CTL_RC6p_ENABLE (1 << 17) -#define GEN6_RC_CTL_RC6_ENABLE (1 << 18) -#define GEN6_RC_CTL_RC1e_ENABLE (1 << 20) -#define GEN6_RC_CTL_RC7_ENABLE (1 << 22) -#define VLV_RC_CTL_CTX_RST_PARALLEL (1 << 24) -#define GEN7_RC_CTL_TO_MODE (1 << 28) -#define GEN6_RC_CTL_EI_MODE(x) ((x) << 27) -#define GEN6_RC_CTL_HW_ENABLE (1 << 31) -#define GEN6_RP_DOWN_TIMEOUT _MMIO(0xA010) -#define GEN6_RP_INTERRUPT_LIMITS _MMIO(0xA014) -#define GEN6_RPSTAT1 _MMIO(0xA01C) -#define GEN6_CAGF_SHIFT 8 -#define HSW_CAGF_SHIFT 7 -#define GEN9_CAGF_SHIFT 23 -#define GEN6_CAGF_MASK (0x7f << GEN6_CAGF_SHIFT) -#define HSW_CAGF_MASK (0x7f << HSW_CAGF_SHIFT) -#define GEN9_CAGF_MASK (0x1ff << GEN9_CAGF_SHIFT) -#define GEN6_RP_CONTROL _MMIO(0xA024) -#define GEN6_RP_MEDIA_TURBO (1 << 11) -#define GEN6_RP_MEDIA_MODE_MASK (3 << 9) -#define GEN6_RP_MEDIA_HW_TURBO_MODE (3 << 9) -#define GEN6_RP_MEDIA_HW_NORMAL_MODE (2 << 9) -#define GEN6_RP_MEDIA_HW_MODE (1 << 9) -#define GEN6_RP_MEDIA_SW_MODE (0 << 9) -#define GEN6_RP_MEDIA_IS_GFX (1 << 8) -#define GEN6_RP_ENABLE (1 << 7) -#define GEN6_RP_UP_IDLE_MIN (0x1 << 3) -#define GEN6_RP_UP_BUSY_AVG (0x2 << 3) -#define GEN6_RP_UP_BUSY_CONT (0x4 << 3) -#define GEN6_RP_DOWN_IDLE_AVG (0x2 << 0) -#define GEN6_RP_DOWN_IDLE_CONT (0x1 << 0) -#define GEN6_RPSWCTL_SHIFT 9 -#define GEN9_RPSWCTL_ENABLE (0x2 << GEN6_RPSWCTL_SHIFT) -#define GEN9_RPSWCTL_DISABLE (0x0 << GEN6_RPSWCTL_SHIFT) -#define GEN6_RP_UP_THRESHOLD _MMIO(0xA02C) -#define GEN6_RP_DOWN_THRESHOLD _MMIO(0xA030) -#define GEN6_RP_CUR_UP_EI _MMIO(0xA050) -#define GEN6_RP_EI_MASK 0xffffff -#define GEN6_CURICONT_MASK GEN6_RP_EI_MASK -#define GEN6_RP_CUR_UP _MMIO(0xA054) -#define GEN6_CURBSYTAVG_MASK GEN6_RP_EI_MASK -#define GEN6_RP_PREV_UP _MMIO(0xA058) -#define GEN6_RP_CUR_DOWN_EI _MMIO(0xA05C) -#define GEN6_CURIAVG_MASK GEN6_RP_EI_MASK -#define GEN6_RP_CUR_DOWN _MMIO(0xA060) -#define GEN6_RP_PREV_DOWN _MMIO(0xA064) -#define GEN6_RP_UP_EI _MMIO(0xA068) -#define GEN6_RP_DOWN_EI _MMIO(0xA06C) -#define GEN6_RP_IDLE_HYSTERSIS _MMIO(0xA070) -#define GEN6_RPDEUHWTC _MMIO(0xA080) -#define GEN6_RPDEUC _MMIO(0xA084) -#define GEN6_RPDEUCSW _MMIO(0xA088) -#define GEN6_RC_STATE _MMIO(0xA094) -#define RC_SW_TARGET_STATE_SHIFT 16 -#define RC_SW_TARGET_STATE_MASK (7 << RC_SW_TARGET_STATE_SHIFT) -#define GEN6_RC1_WAKE_RATE_LIMIT _MMIO(0xA098) -#define GEN6_RC6_WAKE_RATE_LIMIT _MMIO(0xA09C) -#define GEN6_RC6pp_WAKE_RATE_LIMIT _MMIO(0xA0A0) -#define GEN10_MEDIA_WAKE_RATE_LIMIT _MMIO(0xA0A0) -#define GEN6_RC_EVALUATION_INTERVAL _MMIO(0xA0A8) -#define GEN6_RC_IDLE_HYSTERSIS _MMIO(0xA0AC) -#define GEN6_RC_SLEEP _MMIO(0xA0B0) -#define GEN6_RCUBMABDTMR _MMIO(0xA0B0) -#define GEN6_RC1e_THRESHOLD _MMIO(0xA0B4) -#define GEN6_RC6_THRESHOLD _MMIO(0xA0B8) -#define GEN6_RC6p_THRESHOLD _MMIO(0xA0BC) -#define VLV_RCEDATA _MMIO(0xA0BC) -#define GEN6_RC6pp_THRESHOLD _MMIO(0xA0C0) -#define GEN6_PMINTRMSK _MMIO(0xA168) -#define GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC (1 << 31) -#define ARAT_EXPIRED_INTRMSK (1 << 9) -#define GEN8_MISC_CTRL0 _MMIO(0xA180) -#define VLV_PWRDWNUPCTL _MMIO(0xA294) -#define GEN9_MEDIA_PG_IDLE_HYSTERESIS _MMIO(0xA0C4) -#define GEN9_RENDER_PG_IDLE_HYSTERESIS _MMIO(0xA0C8) -#define GEN9_PG_ENABLE _MMIO(0xA210) -#define GEN9_RENDER_PG_ENABLE REG_BIT(0) -#define GEN9_MEDIA_PG_ENABLE REG_BIT(1) -#define GEN11_MEDIA_SAMPLER_PG_ENABLE REG_BIT(2) -#define VDN_HCP_POWERGATE_ENABLE(n) REG_BIT(3 + 2 * (n)) -#define VDN_MFX_POWERGATE_ENABLE(n) REG_BIT(4 + 2 * (n)) -#define GEN8_PUSHBUS_CONTROL _MMIO(0xA248) -#define GEN8_PUSHBUS_ENABLE _MMIO(0xA250) -#define GEN8_PUSHBUS_SHIFT _MMIO(0xA25C) - #define VLV_CHICKEN_3 _MMIO(VLV_DISPLAY_BASE + 0x7040C) #define PIXEL_OVERLAP_CNT_MASK (3 << 30) #define PIXEL_OVERLAP_CNT_SHIFT 30 -#define GEN6_PMISR _MMIO(0x44020) -#define GEN6_PMIMR _MMIO(0x44024) /* rps_lock */ -#define GEN6_PMIIR _MMIO(0x44028) -#define GEN6_PMIER _MMIO(0x4402C) -#define GEN6_PM_MBOX_EVENT (1 << 25) -#define GEN6_PM_THERMAL_EVENT (1 << 24) - -/* - * For Gen11 these are in the upper word of the GPM_WGBOXPERF - * registers. Shifting is handled on accessing the imr and ier. - */ -#define GEN6_PM_RP_DOWN_TIMEOUT (1 << 6) -#define GEN6_PM_RP_UP_THRESHOLD (1 << 5) -#define GEN6_PM_RP_DOWN_THRESHOLD (1 << 4) -#define GEN6_PM_RP_UP_EI_EXPIRED (1 << 2) -#define GEN6_PM_RP_DOWN_EI_EXPIRED (1 << 1) -#define GEN6_PM_RPS_EVENTS (GEN6_PM_RP_UP_EI_EXPIRED | \ - GEN6_PM_RP_UP_THRESHOLD | \ - GEN6_PM_RP_DOWN_EI_EXPIRED | \ - GEN6_PM_RP_DOWN_THRESHOLD | \ - GEN6_PM_RP_DOWN_TIMEOUT) - -#define GEN7_GT_SCRATCH(i) _MMIO(0x4F100 + (i) * 4) -#define GEN7_GT_SCRATCH_REG_NUM 8 - -#define VLV_GTLC_SURVIVABILITY_REG _MMIO(0x130098) -#define VLV_GFX_CLK_STATUS_BIT (1 << 3) -#define VLV_GFX_CLK_FORCE_ON_BIT (1 << 2) - -#define GEN6_GT_GFX_RC6_LOCKED _MMIO(0x138104) -#define VLV_COUNTER_CONTROL _MMIO(0x138104) -#define VLV_COUNT_RANGE_HIGH (1 << 15) -#define VLV_MEDIA_RC0_COUNT_EN (1 << 5) -#define VLV_RENDER_RC0_COUNT_EN (1 << 4) -#define VLV_MEDIA_RC6_COUNT_EN (1 << 1) -#define VLV_RENDER_RC6_COUNT_EN (1 << 0) -#define GEN6_GT_GFX_RC6 _MMIO(0x138108) -#define VLV_GT_RENDER_RC6 _MMIO(0x138108) -#define VLV_GT_MEDIA_RC6 _MMIO(0x13810C) - -#define GEN6_GT_GFX_RC6p _MMIO(0x13810C) -#define GEN6_GT_GFX_RC6pp _MMIO(0x138110) -#define VLV_RENDER_C0_COUNT _MMIO(0x138118) -#define VLV_MEDIA_C0_COUNT _MMIO(0x13811C) - #define GEN6_PCODE_MAILBOX _MMIO(0x138124) #define GEN6_PCODE_READY (1 << 31) #define GEN6_PCODE_ERROR_MASK 0xFF @@ -9635,82 +6749,6 @@ enum { #define GEN6_PCODE_FREQ_RING_RATIO_SHIFT 16 #define GEN6_PCODE_DATA1 _MMIO(0x13812C) -#define GEN6_GT_CORE_STATUS _MMIO(0x138060) -#define GEN6_CORE_CPD_STATE_MASK (7 << 4) -#define GEN6_RCn_MASK 7 -#define GEN6_RC0 0 -#define GEN6_RC3 2 -#define GEN6_RC6 3 -#define GEN6_RC7 4 - -#define GEN8_GT_SLICE_INFO _MMIO(0x138064) -#define GEN8_LSLICESTAT_MASK 0x7 - -#define CHV_POWER_SS0_SIG1 _MMIO(0xa720) -#define CHV_POWER_SS1_SIG1 _MMIO(0xa728) -#define CHV_SS_PG_ENABLE (1 << 1) -#define CHV_EU08_PG_ENABLE (1 << 9) -#define CHV_EU19_PG_ENABLE (1 << 17) -#define CHV_EU210_PG_ENABLE (1 << 25) - -#define CHV_POWER_SS0_SIG2 _MMIO(0xa724) -#define CHV_POWER_SS1_SIG2 _MMIO(0xa72c) -#define CHV_EU311_PG_ENABLE (1 << 1) - -#define GEN9_SLICE_PGCTL_ACK(slice) _MMIO(0x804c + (slice) * 0x4) -#define GEN10_SLICE_PGCTL_ACK(slice) _MMIO(0x804c + ((slice) / 3) * 0x34 + \ - ((slice) % 3) * 0x4) -#define GEN9_PGCTL_SLICE_ACK (1 << 0) -#define GEN9_PGCTL_SS_ACK(subslice) (1 << (2 + (subslice) * 2)) -#define GEN10_PGCTL_VALID_SS_MASK(slice) ((slice) == 0 ? 0x7F : 0x1F) - -#define GEN9_SS01_EU_PGCTL_ACK(slice) _MMIO(0x805c + (slice) * 0x8) -#define GEN10_SS01_EU_PGCTL_ACK(slice) _MMIO(0x805c + ((slice) / 3) * 0x30 + \ - ((slice) % 3) * 0x8) -#define GEN9_SS23_EU_PGCTL_ACK(slice) _MMIO(0x8060 + (slice) * 0x8) -#define GEN10_SS23_EU_PGCTL_ACK(slice) _MMIO(0x8060 + ((slice) / 3) * 0x30 + \ - ((slice) % 3) * 0x8) -#define GEN9_PGCTL_SSA_EU08_ACK (1 << 0) -#define GEN9_PGCTL_SSA_EU19_ACK (1 << 2) -#define GEN9_PGCTL_SSA_EU210_ACK (1 << 4) -#define GEN9_PGCTL_SSA_EU311_ACK (1 << 6) -#define GEN9_PGCTL_SSB_EU08_ACK (1 << 8) -#define GEN9_PGCTL_SSB_EU19_ACK (1 << 10) -#define GEN9_PGCTL_SSB_EU210_ACK (1 << 12) -#define GEN9_PGCTL_SSB_EU311_ACK (1 << 14) - -#define GEN7_MISCCPCTL _MMIO(0x9424) -#define GEN7_DOP_CLOCK_GATE_ENABLE (1 << 0) -#define GEN8_DOP_CLOCK_GATE_CFCLK_ENABLE (1 << 2) -#define GEN8_DOP_CLOCK_GATE_GUC_ENABLE (1 << 4) -#define GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE (1 << 6) - -#define GEN8_GARBCNTL _MMIO(0xB004) -#define GEN9_GAPS_TSV_CREDIT_DISABLE (1 << 7) -#define GEN11_ARBITRATION_PRIO_ORDER_MASK (0x3f << 22) -#define GEN11_HASH_CTRL_EXCL_MASK (0x7f << 0) -#define GEN11_HASH_CTRL_EXCL_BIT0 (1 << 0) - -#define GEN11_GLBLINVL _MMIO(0xB404) -#define GEN11_BANK_HASH_ADDR_EXCL_MASK (0x7f << 5) -#define GEN11_BANK_HASH_ADDR_EXCL_BIT0 (1 << 5) - -#define GEN10_DFR_RATIO_EN_AND_CHICKEN _MMIO(0x9550) -#define DFR_DISABLE (1 << 9) - -#define GEN11_GACB_PERF_CTRL _MMIO(0x4B80) -#define GEN11_HASH_CTRL_MASK (0x3 << 12 | 0xf << 0) -#define GEN11_HASH_CTRL_BIT0 (1 << 0) -#define GEN11_HASH_CTRL_BIT4 (1 << 12) - -#define GEN11_LSN_UNSLCVC _MMIO(0xB43C) -#define GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC (1 << 9) -#define GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC (1 << 7) - -#define GEN10_SAMPLER_MODE _MMIO(0xE18C) -#define ENABLE_SMALLPL REG_BIT(15) -#define GEN11_SAMPLER_ENABLE_HEADLESS_MSG REG_BIT(5) - /* IVYBRIDGE DPF */ #define GEN7_L3CDERRST1(slice) _MMIO(0xB008 + (slice) * 0x200) /* L3CD Error Status 1 */ #define GEN7_L3CDERRST1_ROW_MASK (0x7ff << 14) @@ -9725,73 +6763,6 @@ enum { (((reg) & GEN7_L3CDERRST1_SUBBANK_MASK) >> 8) #define GEN7_L3CDERRST1_ENABLE (1 << 7) -#define GEN7_L3LOG(slice, i) _MMIO(0xB070 + (slice) * 0x200 + (i) * 4) -#define GEN7_L3LOG_SIZE 0x80 - -#define GEN7_HALF_SLICE_CHICKEN1 _MMIO(0xe100) /* IVB GT1 + VLV */ -#define GEN7_HALF_SLICE_CHICKEN1_GT2 _MMIO(0xf100) -#define GEN7_MAX_PS_THREAD_DEP (8 << 12) -#define GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE (1 << 10) -#define GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE (1 << 4) -#define GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE (1 << 3) - -#define GEN9_HALF_SLICE_CHICKEN5 _MMIO(0xe188) -#define GEN9_DG_MIRROR_FIX_ENABLE (1 << 5) -#define GEN9_CCS_TLB_PREFETCH_ENABLE (1 << 3) - -#define GEN8_ROW_CHICKEN _MMIO(0xe4f0) -#define FLOW_CONTROL_ENABLE REG_BIT(15) -#define UGM_BACKUP_MODE REG_BIT(13) -#define MDQ_ARBITRATION_MODE REG_BIT(12) -#define PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE REG_BIT(8) -#define STALL_DOP_GATING_DISABLE REG_BIT(5) -#define THROTTLE_12_5 REG_GENMASK(4, 2) -#define DISABLE_EARLY_EOT REG_BIT(1) - -#define GEN7_ROW_CHICKEN2 _MMIO(0xe4f4) -#define GEN12_DISABLE_READ_SUPPRESSION REG_BIT(15) -#define GEN12_DISABLE_EARLY_READ REG_BIT(14) -#define GEN12_ENABLE_LARGE_GRF_MODE REG_BIT(12) -#define GEN12_PUSH_CONST_DEREF_HOLD_DIS REG_BIT(8) - -#define LSC_CHICKEN_BIT_0 _MMIO(0xe7c8) -#define FORCE_1_SUB_MESSAGE_PER_FRAGMENT REG_BIT(15) -#define LSC_CHICKEN_BIT_0_UDW _MMIO(0xe7c8 + 4) -#define DIS_CHAIN_2XSIMD8 REG_BIT(55 - 32) -#define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32) -#define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32) -#define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32) -#define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32) - -#define GEN7_ROW_CHICKEN2_GT2 _MMIO(0xf4f4) -#define DOP_CLOCK_GATING_DISABLE (1 << 0) -#define PUSH_CONSTANT_DEREF_DISABLE (1 << 8) -#define GEN11_TDL_CLOCK_GATING_FIX_DISABLE (1 << 1) - -#define GEN9_ROW_CHICKEN4 _MMIO(0xe48c) -#define GEN12_DISABLE_GRF_CLEAR REG_BIT(13) -#define GEN12_DISABLE_TDL_PUSH REG_BIT(9) -#define GEN11_DIS_PICK_2ND_EU REG_BIT(7) -#define GEN12_DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX REG_BIT(4) - -#define HSW_ROW_CHICKEN3 _MMIO(0xe49c) -#define HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE (1 << 6) - -#define HALF_SLICE_CHICKEN2 _MMIO(0xe180) -#define GEN8_ST_PO_DISABLE (1 << 13) - -#define HALF_SLICE_CHICKEN3 _MMIO(0xe184) -#define HSW_SAMPLE_C_PERFORMANCE (1 << 9) -#define GEN8_CENTROID_PIXEL_OPT_DIS (1 << 8) -#define GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC (1 << 5) -#define GEN8_SAMPLER_POWER_BYPASS_DIS (1 << 1) - -#define GEN9_HALF_SLICE_CHICKEN7 _MMIO(0xe194) -#define DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA REG_BIT(15) -#define GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR REG_BIT(8) -#define GEN9_ENABLE_YV12_BUGFIX REG_BIT(4) -#define GEN9_ENABLE_GPGPU_PREEMPTION REG_BIT(2) - /* Audio */ #define G4X_AUD_VID_DID _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x62020) #define INTEL_AUDIO_DEVCL 0x808629FB @@ -10840,149 +7811,6 @@ enum skl_power_gate { PORTTC1_PLL_ENABLE, \ PORTTC2_PLL_ENABLE) -#define _MG_REFCLKIN_CTL_PORT1 0x16892C -#define _MG_REFCLKIN_CTL_PORT2 0x16992C -#define _MG_REFCLKIN_CTL_PORT3 0x16A92C -#define _MG_REFCLKIN_CTL_PORT4 0x16B92C -#define MG_REFCLKIN_CTL_OD_2_MUX(x) ((x) << 8) -#define MG_REFCLKIN_CTL_OD_2_MUX_MASK (0x7 << 8) -#define MG_REFCLKIN_CTL(tc_port) _MMIO_PORT((tc_port), \ - _MG_REFCLKIN_CTL_PORT1, \ - _MG_REFCLKIN_CTL_PORT2) - -#define _MG_CLKTOP2_CORECLKCTL1_PORT1 0x1688D8 -#define _MG_CLKTOP2_CORECLKCTL1_PORT2 0x1698D8 -#define _MG_CLKTOP2_CORECLKCTL1_PORT3 0x16A8D8 -#define _MG_CLKTOP2_CORECLKCTL1_PORT4 0x16B8D8 -#define MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO(x) ((x) << 16) -#define MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO_MASK (0xff << 16) -#define MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(x) ((x) << 8) -#define MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK (0xff << 8) -#define MG_CLKTOP2_CORECLKCTL1(tc_port) _MMIO_PORT((tc_port), \ - _MG_CLKTOP2_CORECLKCTL1_PORT1, \ - _MG_CLKTOP2_CORECLKCTL1_PORT2) - -#define _MG_CLKTOP2_HSCLKCTL_PORT1 0x1688D4 -#define _MG_CLKTOP2_HSCLKCTL_PORT2 0x1698D4 -#define _MG_CLKTOP2_HSCLKCTL_PORT3 0x16A8D4 -#define _MG_CLKTOP2_HSCLKCTL_PORT4 0x16B8D4 -#define MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(x) ((x) << 16) -#define MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK (0x1 << 16) -#define MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(x) ((x) << 14) -#define MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK (0x3 << 14) -#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK (0x3 << 12) -#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2 (0 << 12) -#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_3 (1 << 12) -#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_5 (2 << 12) -#define MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_7 (3 << 12) -#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(x) ((x) << 8) -#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT 8 -#define MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK (0xf << 8) -#define MG_CLKTOP2_HSCLKCTL(tc_port) _MMIO_PORT((tc_port), \ - _MG_CLKTOP2_HSCLKCTL_PORT1, \ - _MG_CLKTOP2_HSCLKCTL_PORT2) - -#define _MG_PLL_DIV0_PORT1 0x168A00 -#define _MG_PLL_DIV0_PORT2 0x169A00 -#define _MG_PLL_DIV0_PORT3 0x16AA00 -#define _MG_PLL_DIV0_PORT4 0x16BA00 -#define MG_PLL_DIV0_FRACNEN_H (1 << 30) -#define MG_PLL_DIV0_FBDIV_FRAC_MASK (0x3fffff << 8) -#define MG_PLL_DIV0_FBDIV_FRAC_SHIFT 8 -#define MG_PLL_DIV0_FBDIV_FRAC(x) ((x) << 8) -#define MG_PLL_DIV0_FBDIV_INT_MASK (0xff << 0) -#define MG_PLL_DIV0_FBDIV_INT(x) ((x) << 0) -#define MG_PLL_DIV0(tc_port) _MMIO_PORT((tc_port), _MG_PLL_DIV0_PORT1, \ - _MG_PLL_DIV0_PORT2) - -#define _MG_PLL_DIV1_PORT1 0x168A04 -#define _MG_PLL_DIV1_PORT2 0x169A04 -#define _MG_PLL_DIV1_PORT3 0x16AA04 -#define _MG_PLL_DIV1_PORT4 0x16BA04 -#define MG_PLL_DIV1_IREF_NDIVRATIO(x) ((x) << 16) -#define MG_PLL_DIV1_DITHER_DIV_1 (0 << 12) -#define MG_PLL_DIV1_DITHER_DIV_2 (1 << 12) -#define MG_PLL_DIV1_DITHER_DIV_4 (2 << 12) -#define MG_PLL_DIV1_DITHER_DIV_8 (3 << 12) -#define MG_PLL_DIV1_NDIVRATIO(x) ((x) << 4) -#define MG_PLL_DIV1_FBPREDIV_MASK (0xf << 0) -#define MG_PLL_DIV1_FBPREDIV(x) ((x) << 0) -#define MG_PLL_DIV1(tc_port) _MMIO_PORT((tc_port), _MG_PLL_DIV1_PORT1, \ - _MG_PLL_DIV1_PORT2) - -#define _MG_PLL_LF_PORT1 0x168A08 -#define _MG_PLL_LF_PORT2 0x169A08 -#define _MG_PLL_LF_PORT3 0x16AA08 -#define _MG_PLL_LF_PORT4 0x16BA08 -#define MG_PLL_LF_TDCTARGETCNT(x) ((x) << 24) -#define MG_PLL_LF_AFCCNTSEL_256 (0 << 20) -#define MG_PLL_LF_AFCCNTSEL_512 (1 << 20) -#define MG_PLL_LF_GAINCTRL(x) ((x) << 16) -#define MG_PLL_LF_INT_COEFF(x) ((x) << 8) -#define MG_PLL_LF_PROP_COEFF(x) ((x) << 0) -#define MG_PLL_LF(tc_port) _MMIO_PORT((tc_port), _MG_PLL_LF_PORT1, \ - _MG_PLL_LF_PORT2) - -#define _MG_PLL_FRAC_LOCK_PORT1 0x168A0C -#define _MG_PLL_FRAC_LOCK_PORT2 0x169A0C -#define _MG_PLL_FRAC_LOCK_PORT3 0x16AA0C -#define _MG_PLL_FRAC_LOCK_PORT4 0x16BA0C -#define MG_PLL_FRAC_LOCK_TRUELOCK_CRIT_32 (1 << 18) -#define MG_PLL_FRAC_LOCK_EARLYLOCK_CRIT_32 (1 << 16) -#define MG_PLL_FRAC_LOCK_LOCKTHRESH(x) ((x) << 11) -#define MG_PLL_FRAC_LOCK_DCODITHEREN (1 << 10) -#define MG_PLL_FRAC_LOCK_FEEDFWRDCAL_EN (1 << 8) -#define MG_PLL_FRAC_LOCK_FEEDFWRDGAIN(x) ((x) << 0) -#define MG_PLL_FRAC_LOCK(tc_port) _MMIO_PORT((tc_port), \ - _MG_PLL_FRAC_LOCK_PORT1, \ - _MG_PLL_FRAC_LOCK_PORT2) - -#define _MG_PLL_SSC_PORT1 0x168A10 -#define _MG_PLL_SSC_PORT2 0x169A10 -#define _MG_PLL_SSC_PORT3 0x16AA10 -#define _MG_PLL_SSC_PORT4 0x16BA10 -#define MG_PLL_SSC_EN (1 << 28) -#define MG_PLL_SSC_TYPE(x) ((x) << 26) -#define MG_PLL_SSC_STEPLENGTH(x) ((x) << 16) -#define MG_PLL_SSC_STEPNUM(x) ((x) << 10) -#define MG_PLL_SSC_FLLEN (1 << 9) -#define MG_PLL_SSC_STEPSIZE(x) ((x) << 0) -#define MG_PLL_SSC(tc_port) _MMIO_PORT((tc_port), _MG_PLL_SSC_PORT1, \ - _MG_PLL_SSC_PORT2) - -#define _MG_PLL_BIAS_PORT1 0x168A14 -#define _MG_PLL_BIAS_PORT2 0x169A14 -#define _MG_PLL_BIAS_PORT3 0x16AA14 -#define _MG_PLL_BIAS_PORT4 0x16BA14 -#define MG_PLL_BIAS_BIAS_GB_SEL(x) ((x) << 30) -#define MG_PLL_BIAS_BIAS_GB_SEL_MASK (0x3 << 30) -#define MG_PLL_BIAS_INIT_DCOAMP(x) ((x) << 24) -#define MG_PLL_BIAS_INIT_DCOAMP_MASK (0x3f << 24) -#define MG_PLL_BIAS_BIAS_BONUS(x) ((x) << 16) -#define MG_PLL_BIAS_BIAS_BONUS_MASK (0xff << 16) -#define MG_PLL_BIAS_BIASCAL_EN (1 << 15) -#define MG_PLL_BIAS_CTRIM(x) ((x) << 8) -#define MG_PLL_BIAS_CTRIM_MASK (0x1f << 8) -#define MG_PLL_BIAS_VREF_RDAC(x) ((x) << 5) -#define MG_PLL_BIAS_VREF_RDAC_MASK (0x7 << 5) -#define MG_PLL_BIAS_IREFTRIM(x) ((x) << 0) -#define MG_PLL_BIAS_IREFTRIM_MASK (0x1f << 0) -#define MG_PLL_BIAS(tc_port) _MMIO_PORT((tc_port), _MG_PLL_BIAS_PORT1, \ - _MG_PLL_BIAS_PORT2) - -#define _MG_PLL_TDC_COLDST_BIAS_PORT1 0x168A18 -#define _MG_PLL_TDC_COLDST_BIAS_PORT2 0x169A18 -#define _MG_PLL_TDC_COLDST_BIAS_PORT3 0x16AA18 -#define _MG_PLL_TDC_COLDST_BIAS_PORT4 0x16BA18 -#define MG_PLL_TDC_COLDST_IREFINT_EN (1 << 27) -#define MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(x) ((x) << 17) -#define MG_PLL_TDC_COLDST_COLDSTART (1 << 16) -#define MG_PLL_TDC_TDCOVCCORR_EN (1 << 2) -#define MG_PLL_TDC_TDCSEL(x) ((x) << 0) -#define MG_PLL_TDC_COLDST_BIAS(tc_port) _MMIO_PORT((tc_port), \ - _MG_PLL_TDC_COLDST_BIAS_PORT1, \ - _MG_PLL_TDC_COLDST_BIAS_PORT2) - #define _ICL_DPLL0_CFGCR0 0x164000 #define _ICL_DPLL1_CFGCR0 0x164080 #define ICL_DPLL_CFGCR0(pll) _MMIO_PLL(pll, _ICL_DPLL0_CFGCR0, \ @@ -11039,6 +7867,12 @@ enum skl_power_gate { #define RKL_DPLL_CFGCR0(pll) _MMIO_PLL(pll, _TGL_DPLL0_CFGCR0, \ _TGL_DPLL1_CFGCR0) +#define _TGL_DPLL0_DIV0 0x164B00 +#define _TGL_DPLL1_DIV0 0x164C00 +#define TGL_DPLL0_DIV0(pll) _MMIO_PLL(pll, _TGL_DPLL0_DIV0, _TGL_DPLL1_DIV0) +#define TGL_DPLL0_DIV0_AFC_STARTUP_MASK REG_GENMASK(27, 25) +#define TGL_DPLL0_DIV0_AFC_STARTUP(val) REG_FIELD_PREP(TGL_DPLL0_DIV0_AFC_STARTUP_MASK, (val)) + #define _TGL_DPLL0_CFGCR1 0x164288 #define _TGL_DPLL1_CFGCR1 0x164290 #define _TGL_TBTPLL_CFGCR1 0x1642A0 @@ -11085,7 +7919,15 @@ enum skl_power_gate { #define _DKL_PHY6_BASE 0x16D000 /* DEKEL PHY MMIO Address = Phy base + (internal address & ~index_mask) */ +#define _DKL_PCS_DW5 0x14 +#define DKL_PCS_DW5(tc_port) _MMIO(_PORT(tc_port, _DKL_PHY1_BASE, \ + _DKL_PHY2_BASE) + \ + _DKL_PCS_DW5) +#define DKL_PCS_DW5_CORE_SOFTRESET REG_BIT(11) + #define _DKL_PLL_DIV0 0x200 +#define DKL_PLL_DIV0_AFC_STARTUP_MASK REG_GENMASK(27, 25) +#define DKL_PLL_DIV0_AFC_STARTUP(val) REG_FIELD_PREP(DKL_PLL_DIV0_AFC_STARTUP_MASK, (val)) #define DKL_PLL_DIV0_INTEG_COEFF(x) ((x) << 16) #define DKL_PLL_DIV0_INTEG_COEFF_MASK (0x1F << 16) #define DKL_PLL_DIV0_PROP_COEFF(x) ((x) << 12) @@ -11095,6 +7937,10 @@ enum skl_power_gate { #define DKL_PLL_DIV0_FBPREDIV_MASK (0xF << DKL_PLL_DIV0_FBPREDIV_SHIFT) #define DKL_PLL_DIV0_FBDIV_INT(x) ((x) << 0) #define DKL_PLL_DIV0_FBDIV_INT_MASK (0xFF << 0) +#define DKL_PLL_DIV0_MASK (DKL_PLL_DIV0_INTEG_COEFF_MASK | \ + DKL_PLL_DIV0_PROP_COEFF_MASK | \ + DKL_PLL_DIV0_FBPREDIV_MASK | \ + DKL_PLL_DIV0_FBDIV_INT_MASK) #define DKL_PLL_DIV0(tc_port) _MMIO(_PORT(tc_port, _DKL_PHY1_BASE, \ _DKL_PHY2_BASE) + \ _DKL_PLL_DIV0) @@ -11268,93 +8114,7 @@ enum skl_power_gate { #define DC_STATE_DEBUG_MASK_CORES (1 << 0) #define DC_STATE_DEBUG_MASK_MEMORY_UP (1 << 1) -#define BXT_D_CR_DRP0_DUNIT8 0x1000 -#define BXT_D_CR_DRP0_DUNIT9 0x1200 -#define BXT_D_CR_DRP0_DUNIT_START 8 -#define BXT_D_CR_DRP0_DUNIT_END 11 -#define BXT_D_CR_DRP0_DUNIT(x) _MMIO(MCHBAR_MIRROR_BASE_SNB + \ - _PICK_EVEN((x) - 8, BXT_D_CR_DRP0_DUNIT8,\ - BXT_D_CR_DRP0_DUNIT9)) -#define BXT_DRAM_RANK_MASK 0x3 -#define BXT_DRAM_RANK_SINGLE 0x1 -#define BXT_DRAM_RANK_DUAL 0x3 -#define BXT_DRAM_WIDTH_MASK (0x3 << 4) -#define BXT_DRAM_WIDTH_SHIFT 4 -#define BXT_DRAM_WIDTH_X8 (0x0 << 4) -#define BXT_DRAM_WIDTH_X16 (0x1 << 4) -#define BXT_DRAM_WIDTH_X32 (0x2 << 4) -#define BXT_DRAM_WIDTH_X64 (0x3 << 4) -#define BXT_DRAM_SIZE_MASK (0x7 << 6) -#define BXT_DRAM_SIZE_SHIFT 6 -#define BXT_DRAM_SIZE_4GBIT (0x0 << 6) -#define BXT_DRAM_SIZE_6GBIT (0x1 << 6) -#define BXT_DRAM_SIZE_8GBIT (0x2 << 6) -#define BXT_DRAM_SIZE_12GBIT (0x3 << 6) -#define BXT_DRAM_SIZE_16GBIT (0x4 << 6) -#define BXT_DRAM_TYPE_MASK (0x7 << 22) -#define BXT_DRAM_TYPE_SHIFT 22 -#define BXT_DRAM_TYPE_DDR3 (0x0 << 22) -#define BXT_DRAM_TYPE_LPDDR3 (0x1 << 22) -#define BXT_DRAM_TYPE_LPDDR4 (0x2 << 22) -#define BXT_DRAM_TYPE_DDR4 (0x4 << 22) - -#define SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5E04) -#define DG1_GEAR_TYPE REG_BIT(16) - -#define SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5000) -#define SKL_DRAM_DDR_TYPE_MASK (0x3 << 0) -#define SKL_DRAM_DDR_TYPE_DDR4 (0 << 0) -#define SKL_DRAM_DDR_TYPE_DDR3 (1 << 0) -#define SKL_DRAM_DDR_TYPE_LPDDR3 (2 << 0) -#define SKL_DRAM_DDR_TYPE_LPDDR4 (3 << 0) - -#define SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C) -#define SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5010) -#define SKL_DRAM_S_SHIFT 16 -#define SKL_DRAM_SIZE_MASK 0x3F -#define SKL_DRAM_WIDTH_MASK (0x3 << 8) -#define SKL_DRAM_WIDTH_SHIFT 8 -#define SKL_DRAM_WIDTH_X8 (0x0 << 8) -#define SKL_DRAM_WIDTH_X16 (0x1 << 8) -#define SKL_DRAM_WIDTH_X32 (0x2 << 8) -#define SKL_DRAM_RANK_MASK (0x1 << 10) -#define SKL_DRAM_RANK_SHIFT 10 -#define SKL_DRAM_RANK_1 (0x0 << 10) -#define SKL_DRAM_RANK_2 (0x1 << 10) -#define SKL_DRAM_RANK_MASK (0x1 << 10) -#define ICL_DRAM_SIZE_MASK 0x7F -#define ICL_DRAM_WIDTH_MASK (0x3 << 7) -#define ICL_DRAM_WIDTH_SHIFT 7 -#define ICL_DRAM_WIDTH_X8 (0x0 << 7) -#define ICL_DRAM_WIDTH_X16 (0x1 << 7) -#define ICL_DRAM_WIDTH_X32 (0x2 << 7) -#define ICL_DRAM_RANK_MASK (0x3 << 9) -#define ICL_DRAM_RANK_SHIFT 9 -#define ICL_DRAM_RANK_1 (0x0 << 9) -#define ICL_DRAM_RANK_2 (0x1 << 9) -#define ICL_DRAM_RANK_3 (0x2 << 9) -#define ICL_DRAM_RANK_4 (0x3 << 9) - -#define SA_PERF_STATUS_0_0_0_MCHBAR_PC _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5918) -#define DG1_QCLK_RATIO_MASK REG_GENMASK(9, 2) -#define DG1_QCLK_REFERENCE REG_BIT(10) - -#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4000) -#define DG1_DRAM_T_RDPRE_MASK REG_GENMASK(16, 11) -#define DG1_DRAM_T_RP_MASK REG_GENMASK(6, 0) -#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4004) -#define DG1_DRAM_T_RCD_MASK REG_GENMASK(15, 9) -#define DG1_DRAM_T_RAS_MASK REG_GENMASK(8, 1) - -/* - * Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register, - * since on HSW we can't write to it using intel_uncore_write. - */ -#define D_COMP_HSW _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5F0C) #define D_COMP_BDW _MMIO(0x138144) -#define D_COMP_RCOMP_IN_PROGRESS (1 << 9) -#define D_COMP_COMP_FORCE (1 << 8) -#define D_COMP_COMP_DISABLE (1 << 0) /* Pipe WM_LINETIME - watermark line time */ #define _WM_LINETIME_A 0x45270 @@ -11644,93 +8404,6 @@ enum skl_power_gate { #define CGM_PIPE_GAMMA(pipe, i, w) _MMIO(_PIPE(pipe, _CGM_PIPE_A_GAMMA, _CGM_PIPE_B_GAMMA) + (i) * 8 + (w) * 4) #define CGM_PIPE_MODE(pipe) _MMIO_PIPE(pipe, _CGM_PIPE_A_MODE, _CGM_PIPE_B_MODE) -/* MIPI DSI registers */ - -#define _MIPI_PORT(port, a, c) (((port) == PORT_A) ? a : c) /* ports A and C only */ -#define _MMIO_MIPI(port, a, c) _MMIO(_MIPI_PORT(port, a, c)) - -/* Gen11 DSI */ -#define _MMIO_DSI(tc, dsi0, dsi1) _MMIO_TRANS((tc) - TRANSCODER_DSI_0, \ - dsi0, dsi1) - -#define MIPIO_TXESC_CLK_DIV1 _MMIO(0x160004) -#define GLK_TX_ESC_CLK_DIV1_MASK 0x3FF -#define MIPIO_TXESC_CLK_DIV2 _MMIO(0x160008) -#define GLK_TX_ESC_CLK_DIV2_MASK 0x3FF - -#define _ICL_DSI_ESC_CLK_DIV0 0x6b090 -#define _ICL_DSI_ESC_CLK_DIV1 0x6b890 -#define ICL_DSI_ESC_CLK_DIV(port) _MMIO_PORT((port), \ - _ICL_DSI_ESC_CLK_DIV0, \ - _ICL_DSI_ESC_CLK_DIV1) -#define _ICL_DPHY_ESC_CLK_DIV0 0x162190 -#define _ICL_DPHY_ESC_CLK_DIV1 0x6C190 -#define ICL_DPHY_ESC_CLK_DIV(port) _MMIO_PORT((port), \ - _ICL_DPHY_ESC_CLK_DIV0, \ - _ICL_DPHY_ESC_CLK_DIV1) -#define ICL_BYTE_CLK_PER_ESC_CLK_MASK (0x1f << 16) -#define ICL_BYTE_CLK_PER_ESC_CLK_SHIFT 16 -#define ICL_ESC_CLK_DIV_MASK 0x1ff -#define ICL_ESC_CLK_DIV_SHIFT 0 -#define DSI_MAX_ESC_CLK 20000 /* in KHz */ - -#define _ADL_MIPIO_REG 0x180 -#define ADL_MIPIO_DW(port, dw) _MMIO(_ICL_COMBOPHY(port) + _ADL_MIPIO_REG + 4 * (dw)) -#define TX_ESC_CLK_DIV_PHY_SEL REGBIT(16) -#define TX_ESC_CLK_DIV_PHY_MASK REG_GENMASK(23, 16) -#define TX_ESC_CLK_DIV_PHY REG_FIELD_PREP(TX_ESC_CLK_DIV_PHY_MASK, 0x7f) - -#define _DSI_CMD_FRMCTL_0 0x6b034 -#define _DSI_CMD_FRMCTL_1 0x6b834 -#define DSI_CMD_FRMCTL(port) _MMIO_PORT(port, \ - _DSI_CMD_FRMCTL_0,\ - _DSI_CMD_FRMCTL_1) -#define DSI_FRAME_UPDATE_REQUEST (1 << 31) -#define DSI_PERIODIC_FRAME_UPDATE_ENABLE (1 << 29) -#define DSI_NULL_PACKET_ENABLE (1 << 28) -#define DSI_FRAME_IN_PROGRESS (1 << 0) - -#define _DSI_INTR_MASK_REG_0 0x6b070 -#define _DSI_INTR_MASK_REG_1 0x6b870 -#define DSI_INTR_MASK_REG(port) _MMIO_PORT(port, \ - _DSI_INTR_MASK_REG_0,\ - _DSI_INTR_MASK_REG_1) - -#define _DSI_INTR_IDENT_REG_0 0x6b074 -#define _DSI_INTR_IDENT_REG_1 0x6b874 -#define DSI_INTR_IDENT_REG(port) _MMIO_PORT(port, \ - _DSI_INTR_IDENT_REG_0,\ - _DSI_INTR_IDENT_REG_1) -#define DSI_TE_EVENT (1 << 31) -#define DSI_RX_DATA_OR_BTA_TERMINATED (1 << 30) -#define DSI_TX_DATA (1 << 29) -#define DSI_ULPS_ENTRY_DONE (1 << 28) -#define DSI_NON_TE_TRIGGER_RECEIVED (1 << 27) -#define DSI_HOST_CHKSUM_ERROR (1 << 26) -#define DSI_HOST_MULTI_ECC_ERROR (1 << 25) -#define DSI_HOST_SINGL_ECC_ERROR (1 << 24) -#define DSI_HOST_CONTENTION_DETECTED (1 << 23) -#define DSI_HOST_FALSE_CONTROL_ERROR (1 << 22) -#define DSI_HOST_TIMEOUT_ERROR (1 << 21) -#define DSI_HOST_LOW_POWER_TX_SYNC_ERROR (1 << 20) -#define DSI_HOST_ESCAPE_MODE_ENTRY_ERROR (1 << 19) -#define DSI_FRAME_UPDATE_DONE (1 << 16) -#define DSI_PROTOCOL_VIOLATION_REPORTED (1 << 15) -#define DSI_INVALID_TX_LENGTH (1 << 13) -#define DSI_INVALID_VC (1 << 12) -#define DSI_INVALID_DATA_TYPE (1 << 11) -#define DSI_PERIPHERAL_CHKSUM_ERROR (1 << 10) -#define DSI_PERIPHERAL_MULTI_ECC_ERROR (1 << 9) -#define DSI_PERIPHERAL_SINGLE_ECC_ERROR (1 << 8) -#define DSI_PERIPHERAL_CONTENTION_DETECTED (1 << 7) -#define DSI_PERIPHERAL_FALSE_CTRL_ERROR (1 << 6) -#define DSI_PERIPHERAL_TIMEOUT_ERROR (1 << 5) -#define DSI_PERIPHERAL_LP_TX_SYNC_ERROR (1 << 4) -#define DSI_PERIPHERAL_ESC_MODE_ENTRY_CMD_ERR (1 << 3) -#define DSI_EOT_SYNC_ERROR (1 << 2) -#define DSI_SOT_SYNC_ERROR (1 << 1) -#define DSI_SOT_ERROR (1 << 0) - /* Gen4+ Timestamp and Pipe Frame time stamp registers */ #define GEN4_TIMESTAMP _MMIO(0x2358) #define ILK_TIMESTAMP_HI _MMIO(0x70070) @@ -11746,143 +8419,6 @@ enum skl_power_gate { #define PIPE_FRMTMSTMP(pipe) \ _MMIO_PIPE2(pipe, _PIPE_FRMTMSTMP_A) -/* BXT MIPI clock controls */ -#define BXT_MAX_VAR_OUTPUT_KHZ 39500 - -#define BXT_MIPI_CLOCK_CTL _MMIO(0x46090) -#define BXT_MIPI1_DIV_SHIFT 26 -#define BXT_MIPI2_DIV_SHIFT 10 -#define BXT_MIPI_DIV_SHIFT(port) \ - _MIPI_PORT(port, BXT_MIPI1_DIV_SHIFT, \ - BXT_MIPI2_DIV_SHIFT) - -/* TX control divider to select actual TX clock output from (8x/var) */ -#define BXT_MIPI1_TX_ESCLK_SHIFT 26 -#define BXT_MIPI2_TX_ESCLK_SHIFT 10 -#define BXT_MIPI_TX_ESCLK_SHIFT(port) \ - _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_SHIFT, \ - BXT_MIPI2_TX_ESCLK_SHIFT) -#define BXT_MIPI1_TX_ESCLK_FIXDIV_MASK (0x3F << 26) -#define BXT_MIPI2_TX_ESCLK_FIXDIV_MASK (0x3F << 10) -#define BXT_MIPI_TX_ESCLK_FIXDIV_MASK(port) \ - _MIPI_PORT(port, BXT_MIPI1_TX_ESCLK_FIXDIV_MASK, \ - BXT_MIPI2_TX_ESCLK_FIXDIV_MASK) -#define BXT_MIPI_TX_ESCLK_DIVIDER(port, val) \ - (((val) & 0x3F) << BXT_MIPI_TX_ESCLK_SHIFT(port)) -/* RX upper control divider to select actual RX clock output from 8x */ -#define BXT_MIPI1_RX_ESCLK_UPPER_SHIFT 21 -#define BXT_MIPI2_RX_ESCLK_UPPER_SHIFT 5 -#define BXT_MIPI_RX_ESCLK_UPPER_SHIFT(port) \ - _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_UPPER_SHIFT, \ - BXT_MIPI2_RX_ESCLK_UPPER_SHIFT) -#define BXT_MIPI1_RX_ESCLK_UPPER_FIXDIV_MASK (3 << 21) -#define BXT_MIPI2_RX_ESCLK_UPPER_FIXDIV_MASK (3 << 5) -#define BXT_MIPI_RX_ESCLK_UPPER_FIXDIV_MASK(port) \ - _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_UPPER_FIXDIV_MASK, \ - BXT_MIPI2_RX_ESCLK_UPPER_FIXDIV_MASK) -#define BXT_MIPI_RX_ESCLK_UPPER_DIVIDER(port, val) \ - (((val) & 3) << BXT_MIPI_RX_ESCLK_UPPER_SHIFT(port)) -/* 8/3X divider to select the actual 8/3X clock output from 8x */ -#define BXT_MIPI1_8X_BY3_SHIFT 19 -#define BXT_MIPI2_8X_BY3_SHIFT 3 -#define BXT_MIPI_8X_BY3_SHIFT(port) \ - _MIPI_PORT(port, BXT_MIPI1_8X_BY3_SHIFT, \ - BXT_MIPI2_8X_BY3_SHIFT) -#define BXT_MIPI1_8X_BY3_DIVIDER_MASK (3 << 19) -#define BXT_MIPI2_8X_BY3_DIVIDER_MASK (3 << 3) -#define BXT_MIPI_8X_BY3_DIVIDER_MASK(port) \ - _MIPI_PORT(port, BXT_MIPI1_8X_BY3_DIVIDER_MASK, \ - BXT_MIPI2_8X_BY3_DIVIDER_MASK) -#define BXT_MIPI_8X_BY3_DIVIDER(port, val) \ - (((val) & 3) << BXT_MIPI_8X_BY3_SHIFT(port)) -/* RX lower control divider to select actual RX clock output from 8x */ -#define BXT_MIPI1_RX_ESCLK_LOWER_SHIFT 16 -#define BXT_MIPI2_RX_ESCLK_LOWER_SHIFT 0 -#define BXT_MIPI_RX_ESCLK_LOWER_SHIFT(port) \ - _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_LOWER_SHIFT, \ - BXT_MIPI2_RX_ESCLK_LOWER_SHIFT) -#define BXT_MIPI1_RX_ESCLK_LOWER_FIXDIV_MASK (3 << 16) -#define BXT_MIPI2_RX_ESCLK_LOWER_FIXDIV_MASK (3 << 0) -#define BXT_MIPI_RX_ESCLK_LOWER_FIXDIV_MASK(port) \ - _MIPI_PORT(port, BXT_MIPI1_RX_ESCLK_LOWER_FIXDIV_MASK, \ - BXT_MIPI2_RX_ESCLK_LOWER_FIXDIV_MASK) -#define BXT_MIPI_RX_ESCLK_LOWER_DIVIDER(port, val) \ - (((val) & 3) << BXT_MIPI_RX_ESCLK_LOWER_SHIFT(port)) - -#define RX_DIVIDER_BIT_1_2 0x3 -#define RX_DIVIDER_BIT_3_4 0xC - -/* BXT MIPI mode configure */ -#define _BXT_MIPIA_TRANS_HACTIVE 0x6B0F8 -#define _BXT_MIPIC_TRANS_HACTIVE 0x6B8F8 -#define BXT_MIPI_TRANS_HACTIVE(tc) _MMIO_MIPI(tc, \ - _BXT_MIPIA_TRANS_HACTIVE, _BXT_MIPIC_TRANS_HACTIVE) - -#define _BXT_MIPIA_TRANS_VACTIVE 0x6B0FC -#define _BXT_MIPIC_TRANS_VACTIVE 0x6B8FC -#define BXT_MIPI_TRANS_VACTIVE(tc) _MMIO_MIPI(tc, \ - _BXT_MIPIA_TRANS_VACTIVE, _BXT_MIPIC_TRANS_VACTIVE) - -#define _BXT_MIPIA_TRANS_VTOTAL 0x6B100 -#define _BXT_MIPIC_TRANS_VTOTAL 0x6B900 -#define BXT_MIPI_TRANS_VTOTAL(tc) _MMIO_MIPI(tc, \ - _BXT_MIPIA_TRANS_VTOTAL, _BXT_MIPIC_TRANS_VTOTAL) - -#define BXT_DSI_PLL_CTL _MMIO(0x161000) -#define BXT_DSI_PLL_PVD_RATIO_SHIFT 16 -#define BXT_DSI_PLL_PVD_RATIO_MASK (3 << BXT_DSI_PLL_PVD_RATIO_SHIFT) -#define BXT_DSI_PLL_PVD_RATIO_1 (1 << BXT_DSI_PLL_PVD_RATIO_SHIFT) -#define BXT_DSIC_16X_BY1 (0 << 10) -#define BXT_DSIC_16X_BY2 (1 << 10) -#define BXT_DSIC_16X_BY3 (2 << 10) -#define BXT_DSIC_16X_BY4 (3 << 10) -#define BXT_DSIC_16X_MASK (3 << 10) -#define BXT_DSIA_16X_BY1 (0 << 8) -#define BXT_DSIA_16X_BY2 (1 << 8) -#define BXT_DSIA_16X_BY3 (2 << 8) -#define BXT_DSIA_16X_BY4 (3 << 8) -#define BXT_DSIA_16X_MASK (3 << 8) -#define BXT_DSI_FREQ_SEL_SHIFT 8 -#define BXT_DSI_FREQ_SEL_MASK (0xF << BXT_DSI_FREQ_SEL_SHIFT) - -#define BXT_DSI_PLL_RATIO_MAX 0x7D -#define BXT_DSI_PLL_RATIO_MIN 0x22 -#define GLK_DSI_PLL_RATIO_MAX 0x6F -#define GLK_DSI_PLL_RATIO_MIN 0x22 -#define BXT_DSI_PLL_RATIO_MASK 0xFF -#define BXT_REF_CLOCK_KHZ 19200 - -#define BXT_DSI_PLL_ENABLE _MMIO(0x46080) -#define BXT_DSI_PLL_DO_ENABLE (1 << 31) -#define BXT_DSI_PLL_LOCKED (1 << 30) - -#define _MIPIA_PORT_CTRL (VLV_DISPLAY_BASE + 0x61190) -#define _MIPIC_PORT_CTRL (VLV_DISPLAY_BASE + 0x61700) -#define MIPI_PORT_CTRL(port) _MMIO_MIPI(port, _MIPIA_PORT_CTRL, _MIPIC_PORT_CTRL) - - /* BXT port control */ -#define _BXT_MIPIA_PORT_CTRL 0x6B0C0 -#define _BXT_MIPIC_PORT_CTRL 0x6B8C0 -#define BXT_MIPI_PORT_CTRL(tc) _MMIO_MIPI(tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL) - -/* ICL DSI MODE control */ -#define _ICL_DSI_IO_MODECTL_0 0x6B094 -#define _ICL_DSI_IO_MODECTL_1 0x6B894 -#define ICL_DSI_IO_MODECTL(port) _MMIO_PORT(port, \ - _ICL_DSI_IO_MODECTL_0, \ - _ICL_DSI_IO_MODECTL_1) -#define COMBO_PHY_MODE_DSI (1 << 0) - -/* TGL DSI Chicken register */ -#define _TGL_DSI_CHKN_REG_0 0x6B0C0 -#define _TGL_DSI_CHKN_REG_1 0x6B8C0 -#define TGL_DSI_CHKN_REG(port) _MMIO_PORT(port, \ - _TGL_DSI_CHKN_REG_0, \ - _TGL_DSI_CHKN_REG_1) -#define TGL_DSI_CHKN_LSHS_GB_MASK REG_GENMASK(15, 12) -#define TGL_DSI_CHKN_LSHS_GB(byte_clocks) REG_FIELD_PREP(TGL_DSI_CHKN_LSHS_GB_MASK, \ - (byte_clocks)) - /* Display Stream Splitter Control */ #define DSS_CTL1 _MMIO(0x67400) #define SPLITTER_ENABLE (1 << 31) @@ -11921,718 +8457,6 @@ enum skl_power_gate { _ICL_PIPE_DSS_CTL2_PB, \ _ICL_PIPE_DSS_CTL2_PC) -#define BXT_P_DSI_REGULATOR_CFG _MMIO(0x160020) -#define STAP_SELECT (1 << 0) - -#define BXT_P_DSI_REGULATOR_TX_CTRL _MMIO(0x160054) -#define HS_IO_CTRL_SELECT (1 << 0) - -#define DPI_ENABLE (1 << 31) /* A + C */ -#define MIPIA_MIPI4DPHY_DELAY_COUNT_SHIFT 27 -#define MIPIA_MIPI4DPHY_DELAY_COUNT_MASK (0xf << 27) -#define DUAL_LINK_MODE_SHIFT 26 -#define DUAL_LINK_MODE_MASK (1 << 26) -#define DUAL_LINK_MODE_FRONT_BACK (0 << 26) -#define DUAL_LINK_MODE_PIXEL_ALTERNATIVE (1 << 26) -#define DITHERING_ENABLE (1 << 25) /* A + C */ -#define FLOPPED_HSTX (1 << 23) -#define DE_INVERT (1 << 19) /* XXX */ -#define MIPIA_FLISDSI_DELAY_COUNT_SHIFT 18 -#define MIPIA_FLISDSI_DELAY_COUNT_MASK (0xf << 18) -#define AFE_LATCHOUT (1 << 17) -#define LP_OUTPUT_HOLD (1 << 16) -#define MIPIC_FLISDSI_DELAY_COUNT_HIGH_SHIFT 15 -#define MIPIC_FLISDSI_DELAY_COUNT_HIGH_MASK (1 << 15) -#define MIPIC_MIPI4DPHY_DELAY_COUNT_SHIFT 11 -#define MIPIC_MIPI4DPHY_DELAY_COUNT_MASK (0xf << 11) -#define CSB_SHIFT 9 -#define CSB_MASK (3 << 9) -#define CSB_20MHZ (0 << 9) -#define CSB_10MHZ (1 << 9) -#define CSB_40MHZ (2 << 9) -#define BANDGAP_MASK (1 << 8) -#define BANDGAP_PNW_CIRCUIT (0 << 8) -#define BANDGAP_LNC_CIRCUIT (1 << 8) -#define MIPIC_FLISDSI_DELAY_COUNT_LOW_SHIFT 5 -#define MIPIC_FLISDSI_DELAY_COUNT_LOW_MASK (7 << 5) -#define TEARING_EFFECT_DELAY (1 << 4) /* A + C */ -#define TEARING_EFFECT_SHIFT 2 /* A + C */ -#define TEARING_EFFECT_MASK (3 << 2) -#define TEARING_EFFECT_OFF (0 << 2) -#define TEARING_EFFECT_DSI (1 << 2) -#define TEARING_EFFECT_GPIO (2 << 2) -#define LANE_CONFIGURATION_SHIFT 0 -#define LANE_CONFIGURATION_MASK (3 << 0) -#define LANE_CONFIGURATION_4LANE (0 << 0) -#define LANE_CONFIGURATION_DUAL_LINK_A (1 << 0) -#define LANE_CONFIGURATION_DUAL_LINK_B (2 << 0) - -#define _MIPIA_TEARING_CTRL (VLV_DISPLAY_BASE + 0x61194) -#define _MIPIC_TEARING_CTRL (VLV_DISPLAY_BASE + 0x61704) -#define MIPI_TEARING_CTRL(port) _MMIO_MIPI(port, _MIPIA_TEARING_CTRL, _MIPIC_TEARING_CTRL) -#define TEARING_EFFECT_DELAY_SHIFT 0 -#define TEARING_EFFECT_DELAY_MASK (0xffff << 0) - -/* XXX: all bits reserved */ -#define _MIPIA_AUTOPWG (VLV_DISPLAY_BASE + 0x611a0) - -/* MIPI DSI Controller and D-PHY registers */ - -#define _MIPIA_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb000) -#define _MIPIC_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb800) -#define MIPI_DEVICE_READY(port) _MMIO_MIPI(port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY) -#define BUS_POSSESSION (1 << 3) /* set to give bus to receiver */ -#define ULPS_STATE_MASK (3 << 1) -#define ULPS_STATE_ENTER (2 << 1) -#define ULPS_STATE_EXIT (1 << 1) -#define ULPS_STATE_NORMAL_OPERATION (0 << 1) -#define DEVICE_READY (1 << 0) - -#define _MIPIA_INTR_STAT (dev_priv->mipi_mmio_base + 0xb004) -#define _MIPIC_INTR_STAT (dev_priv->mipi_mmio_base + 0xb804) -#define MIPI_INTR_STAT(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT) -#define _MIPIA_INTR_EN (dev_priv->mipi_mmio_base + 0xb008) -#define _MIPIC_INTR_EN (dev_priv->mipi_mmio_base + 0xb808) -#define MIPI_INTR_EN(port) _MMIO_MIPI(port, _MIPIA_INTR_EN, _MIPIC_INTR_EN) -#define TEARING_EFFECT (1 << 31) -#define SPL_PKT_SENT_INTERRUPT (1 << 30) -#define GEN_READ_DATA_AVAIL (1 << 29) -#define LP_GENERIC_WR_FIFO_FULL (1 << 28) -#define HS_GENERIC_WR_FIFO_FULL (1 << 27) -#define RX_PROT_VIOLATION (1 << 26) -#define RX_INVALID_TX_LENGTH (1 << 25) -#define ACK_WITH_NO_ERROR (1 << 24) -#define TURN_AROUND_ACK_TIMEOUT (1 << 23) -#define LP_RX_TIMEOUT (1 << 22) -#define HS_TX_TIMEOUT (1 << 21) -#define DPI_FIFO_UNDERRUN (1 << 20) -#define LOW_CONTENTION (1 << 19) -#define HIGH_CONTENTION (1 << 18) -#define TXDSI_VC_ID_INVALID (1 << 17) -#define TXDSI_DATA_TYPE_NOT_RECOGNISED (1 << 16) -#define TXCHECKSUM_ERROR (1 << 15) -#define TXECC_MULTIBIT_ERROR (1 << 14) -#define TXECC_SINGLE_BIT_ERROR (1 << 13) -#define TXFALSE_CONTROL_ERROR (1 << 12) -#define RXDSI_VC_ID_INVALID (1 << 11) -#define RXDSI_DATA_TYPE_NOT_REGOGNISED (1 << 10) -#define RXCHECKSUM_ERROR (1 << 9) -#define RXECC_MULTIBIT_ERROR (1 << 8) -#define RXECC_SINGLE_BIT_ERROR (1 << 7) -#define RXFALSE_CONTROL_ERROR (1 << 6) -#define RXHS_RECEIVE_TIMEOUT_ERROR (1 << 5) -#define RX_LP_TX_SYNC_ERROR (1 << 4) -#define RXEXCAPE_MODE_ENTRY_ERROR (1 << 3) -#define RXEOT_SYNC_ERROR (1 << 2) -#define RXSOT_SYNC_ERROR (1 << 1) -#define RXSOT_ERROR (1 << 0) - -#define _MIPIA_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb00c) -#define _MIPIC_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb80c) -#define MIPI_DSI_FUNC_PRG(port) _MMIO_MIPI(port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG) -#define CMD_MODE_DATA_WIDTH_MASK (7 << 13) -#define CMD_MODE_NOT_SUPPORTED (0 << 13) -#define CMD_MODE_DATA_WIDTH_16_BIT (1 << 13) -#define CMD_MODE_DATA_WIDTH_9_BIT (2 << 13) -#define CMD_MODE_DATA_WIDTH_8_BIT (3 << 13) -#define CMD_MODE_DATA_WIDTH_OPTION1 (4 << 13) -#define CMD_MODE_DATA_WIDTH_OPTION2 (5 << 13) -#define VID_MODE_FORMAT_MASK (0xf << 7) -#define VID_MODE_NOT_SUPPORTED (0 << 7) -#define VID_MODE_FORMAT_RGB565 (1 << 7) -#define VID_MODE_FORMAT_RGB666_PACKED (2 << 7) -#define VID_MODE_FORMAT_RGB666 (3 << 7) -#define VID_MODE_FORMAT_RGB888 (4 << 7) -#define CMD_MODE_CHANNEL_NUMBER_SHIFT 5 -#define CMD_MODE_CHANNEL_NUMBER_MASK (3 << 5) -#define VID_MODE_CHANNEL_NUMBER_SHIFT 3 -#define VID_MODE_CHANNEL_NUMBER_MASK (3 << 3) -#define DATA_LANES_PRG_REG_SHIFT 0 -#define DATA_LANES_PRG_REG_MASK (7 << 0) - -#define _MIPIA_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb010) -#define _MIPIC_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb810) -#define MIPI_HS_TX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT) -#define HIGH_SPEED_TX_TIMEOUT_COUNTER_MASK 0xffffff - -#define _MIPIA_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb014) -#define _MIPIC_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb814) -#define MIPI_LP_RX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT) -#define LOW_POWER_RX_TIMEOUT_COUNTER_MASK 0xffffff - -#define _MIPIA_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb018) -#define _MIPIC_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb818) -#define MIPI_TURN_AROUND_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT) -#define TURN_AROUND_TIMEOUT_MASK 0x3f - -#define _MIPIA_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb01c) -#define _MIPIC_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb81c) -#define MIPI_DEVICE_RESET_TIMER(port) _MMIO_MIPI(port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER) -#define DEVICE_RESET_TIMER_MASK 0xffff - -#define _MIPIA_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb020) -#define _MIPIC_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb820) -#define MIPI_DPI_RESOLUTION(port) _MMIO_MIPI(port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION) -#define VERTICAL_ADDRESS_SHIFT 16 -#define VERTICAL_ADDRESS_MASK (0xffff << 16) -#define HORIZONTAL_ADDRESS_SHIFT 0 -#define HORIZONTAL_ADDRESS_MASK 0xffff - -#define _MIPIA_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb024) -#define _MIPIC_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb824) -#define MIPI_DBI_FIFO_THROTTLE(port) _MMIO_MIPI(port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE) -#define DBI_FIFO_EMPTY_HALF (0 << 0) -#define DBI_FIFO_EMPTY_QUARTER (1 << 0) -#define DBI_FIFO_EMPTY_7_LOCATIONS (2 << 0) - -/* regs below are bits 15:0 */ -#define _MIPIA_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb028) -#define _MIPIC_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb828) -#define MIPI_HSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT) - -#define _MIPIA_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb02c) -#define _MIPIC_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb82c) -#define MIPI_HBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT) - -#define _MIPIA_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb030) -#define _MIPIC_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb830) -#define MIPI_HFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT) - -#define _MIPIA_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb034) -#define _MIPIC_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb834) -#define MIPI_HACTIVE_AREA_COUNT(port) _MMIO_MIPI(port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT) - -#define _MIPIA_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb038) -#define _MIPIC_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb838) -#define MIPI_VSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT) - -#define _MIPIA_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb03c) -#define _MIPIC_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb83c) -#define MIPI_VBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT) - -#define _MIPIA_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb040) -#define _MIPIC_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb840) -#define MIPI_VFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT) - -#define _MIPIA_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb044) -#define _MIPIC_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb844) -#define MIPI_HIGH_LOW_SWITCH_COUNT(port) _MMIO_MIPI(port, _MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT) - -/* regs above are bits 15:0 */ - -#define _MIPIA_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb048) -#define _MIPIC_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb848) -#define MIPI_DPI_CONTROL(port) _MMIO_MIPI(port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL) -#define DPI_LP_MODE (1 << 6) -#define BACKLIGHT_OFF (1 << 5) -#define BACKLIGHT_ON (1 << 4) -#define COLOR_MODE_OFF (1 << 3) -#define COLOR_MODE_ON (1 << 2) -#define TURN_ON (1 << 1) -#define SHUTDOWN (1 << 0) - -#define _MIPIA_DPI_DATA (dev_priv->mipi_mmio_base + 0xb04c) -#define _MIPIC_DPI_DATA (dev_priv->mipi_mmio_base + 0xb84c) -#define MIPI_DPI_DATA(port) _MMIO_MIPI(port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA) -#define COMMAND_BYTE_SHIFT 0 -#define COMMAND_BYTE_MASK (0x3f << 0) - -#define _MIPIA_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb050) -#define _MIPIC_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb850) -#define MIPI_INIT_COUNT(port) _MMIO_MIPI(port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT) -#define MASTER_INIT_TIMER_SHIFT 0 -#define MASTER_INIT_TIMER_MASK (0xffff << 0) - -#define _MIPIA_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb054) -#define _MIPIC_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb854) -#define MIPI_MAX_RETURN_PKT_SIZE(port) _MMIO_MIPI(port, \ - _MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE) -#define MAX_RETURN_PKT_SIZE_SHIFT 0 -#define MAX_RETURN_PKT_SIZE_MASK (0x3ff << 0) - -#define _MIPIA_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb058) -#define _MIPIC_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb858) -#define MIPI_VIDEO_MODE_FORMAT(port) _MMIO_MIPI(port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT) -#define RANDOM_DPI_DISPLAY_RESOLUTION (1 << 4) -#define DISABLE_VIDEO_BTA (1 << 3) -#define IP_TG_CONFIG (1 << 2) -#define VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE (1 << 0) -#define VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS (2 << 0) -#define VIDEO_MODE_BURST (3 << 0) - -#define _MIPIA_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb05c) -#define _MIPIC_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb85c) -#define MIPI_EOT_DISABLE(port) _MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE) -#define BXT_DEFEATURE_DPI_FIFO_CTR (1 << 9) -#define BXT_DPHY_DEFEATURE_EN (1 << 8) -#define LP_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 7) -#define HS_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 6) -#define LOW_CONTENTION_RECOVERY_DISABLE (1 << 5) -#define HIGH_CONTENTION_RECOVERY_DISABLE (1 << 4) -#define TXDSI_TYPE_NOT_RECOGNISED_ERROR_RECOVERY_DISABLE (1 << 3) -#define TXECC_MULTIBIT_ERROR_RECOVERY_DISABLE (1 << 2) -#define CLOCKSTOP (1 << 1) -#define EOT_DISABLE (1 << 0) - -#define _MIPIA_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb060) -#define _MIPIC_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb860) -#define MIPI_LP_BYTECLK(port) _MMIO_MIPI(port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK) -#define LP_BYTECLK_SHIFT 0 -#define LP_BYTECLK_MASK (0xffff << 0) - -#define _MIPIA_TLPX_TIME_COUNT (dev_priv->mipi_mmio_base + 0xb0a4) -#define _MIPIC_TLPX_TIME_COUNT (dev_priv->mipi_mmio_base + 0xb8a4) -#define MIPI_TLPX_TIME_COUNT(port) _MMIO_MIPI(port, _MIPIA_TLPX_TIME_COUNT, _MIPIC_TLPX_TIME_COUNT) - -#define _MIPIA_CLK_LANE_TIMING (dev_priv->mipi_mmio_base + 0xb098) -#define _MIPIC_CLK_LANE_TIMING (dev_priv->mipi_mmio_base + 0xb898) -#define MIPI_CLK_LANE_TIMING(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_TIMING, _MIPIC_CLK_LANE_TIMING) - -/* bits 31:0 */ -#define _MIPIA_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb064) -#define _MIPIC_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb864) -#define MIPI_LP_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA) - -/* bits 31:0 */ -#define _MIPIA_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb068) -#define _MIPIC_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb868) -#define MIPI_HS_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA) - -#define _MIPIA_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb06c) -#define _MIPIC_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb86c) -#define MIPI_LP_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL) -#define _MIPIA_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb070) -#define _MIPIC_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb870) -#define MIPI_HS_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL) -#define LONG_PACKET_WORD_COUNT_SHIFT 8 -#define LONG_PACKET_WORD_COUNT_MASK (0xffff << 8) -#define SHORT_PACKET_PARAM_SHIFT 8 -#define SHORT_PACKET_PARAM_MASK (0xffff << 8) -#define VIRTUAL_CHANNEL_SHIFT 6 -#define VIRTUAL_CHANNEL_MASK (3 << 6) -#define DATA_TYPE_SHIFT 0 -#define DATA_TYPE_MASK (0x3f << 0) -/* data type values, see include/video/mipi_display.h */ - -#define _MIPIA_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb074) -#define _MIPIC_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb874) -#define MIPI_GEN_FIFO_STAT(port) _MMIO_MIPI(port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT) -#define DPI_FIFO_EMPTY (1 << 28) -#define DBI_FIFO_EMPTY (1 << 27) -#define LP_CTRL_FIFO_EMPTY (1 << 26) -#define LP_CTRL_FIFO_HALF_EMPTY (1 << 25) -#define LP_CTRL_FIFO_FULL (1 << 24) -#define HS_CTRL_FIFO_EMPTY (1 << 18) -#define HS_CTRL_FIFO_HALF_EMPTY (1 << 17) -#define HS_CTRL_FIFO_FULL (1 << 16) -#define LP_DATA_FIFO_EMPTY (1 << 10) -#define LP_DATA_FIFO_HALF_EMPTY (1 << 9) -#define LP_DATA_FIFO_FULL (1 << 8) -#define HS_DATA_FIFO_EMPTY (1 << 2) -#define HS_DATA_FIFO_HALF_EMPTY (1 << 1) -#define HS_DATA_FIFO_FULL (1 << 0) - -#define _MIPIA_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb078) -#define _MIPIC_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb878) -#define MIPI_HS_LP_DBI_ENABLE(port) _MMIO_MIPI(port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE) -#define DBI_HS_LP_MODE_MASK (1 << 0) -#define DBI_LP_MODE (1 << 0) -#define DBI_HS_MODE (0 << 0) - -#define _MIPIA_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb080) -#define _MIPIC_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb880) -#define MIPI_DPHY_PARAM(port) _MMIO_MIPI(port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM) -#define EXIT_ZERO_COUNT_SHIFT 24 -#define EXIT_ZERO_COUNT_MASK (0x3f << 24) -#define TRAIL_COUNT_SHIFT 16 -#define TRAIL_COUNT_MASK (0x1f << 16) -#define CLK_ZERO_COUNT_SHIFT 8 -#define CLK_ZERO_COUNT_MASK (0xff << 8) -#define PREPARE_COUNT_SHIFT 0 -#define PREPARE_COUNT_MASK (0x3f << 0) - -#define _ICL_DSI_T_INIT_MASTER_0 0x6b088 -#define _ICL_DSI_T_INIT_MASTER_1 0x6b888 -#define ICL_DSI_T_INIT_MASTER(port) _MMIO_PORT(port, \ - _ICL_DSI_T_INIT_MASTER_0,\ - _ICL_DSI_T_INIT_MASTER_1) - -#define _DPHY_CLK_TIMING_PARAM_0 0x162180 -#define _DPHY_CLK_TIMING_PARAM_1 0x6c180 -#define DPHY_CLK_TIMING_PARAM(port) _MMIO_PORT(port, \ - _DPHY_CLK_TIMING_PARAM_0,\ - _DPHY_CLK_TIMING_PARAM_1) -#define _DSI_CLK_TIMING_PARAM_0 0x6b080 -#define _DSI_CLK_TIMING_PARAM_1 0x6b880 -#define DSI_CLK_TIMING_PARAM(port) _MMIO_PORT(port, \ - _DSI_CLK_TIMING_PARAM_0,\ - _DSI_CLK_TIMING_PARAM_1) -#define CLK_PREPARE_OVERRIDE (1 << 31) -#define CLK_PREPARE(x) ((x) << 28) -#define CLK_PREPARE_MASK (0x7 << 28) -#define CLK_PREPARE_SHIFT 28 -#define CLK_ZERO_OVERRIDE (1 << 27) -#define CLK_ZERO(x) ((x) << 20) -#define CLK_ZERO_MASK (0xf << 20) -#define CLK_ZERO_SHIFT 20 -#define CLK_PRE_OVERRIDE (1 << 19) -#define CLK_PRE(x) ((x) << 16) -#define CLK_PRE_MASK (0x3 << 16) -#define CLK_PRE_SHIFT 16 -#define CLK_POST_OVERRIDE (1 << 15) -#define CLK_POST(x) ((x) << 8) -#define CLK_POST_MASK (0x7 << 8) -#define CLK_POST_SHIFT 8 -#define CLK_TRAIL_OVERRIDE (1 << 7) -#define CLK_TRAIL(x) ((x) << 0) -#define CLK_TRAIL_MASK (0xf << 0) -#define CLK_TRAIL_SHIFT 0 - -#define _DPHY_DATA_TIMING_PARAM_0 0x162184 -#define _DPHY_DATA_TIMING_PARAM_1 0x6c184 -#define DPHY_DATA_TIMING_PARAM(port) _MMIO_PORT(port, \ - _DPHY_DATA_TIMING_PARAM_0,\ - _DPHY_DATA_TIMING_PARAM_1) -#define _DSI_DATA_TIMING_PARAM_0 0x6B084 -#define _DSI_DATA_TIMING_PARAM_1 0x6B884 -#define DSI_DATA_TIMING_PARAM(port) _MMIO_PORT(port, \ - _DSI_DATA_TIMING_PARAM_0,\ - _DSI_DATA_TIMING_PARAM_1) -#define HS_PREPARE_OVERRIDE (1 << 31) -#define HS_PREPARE(x) ((x) << 24) -#define HS_PREPARE_MASK (0x7 << 24) -#define HS_PREPARE_SHIFT 24 -#define HS_ZERO_OVERRIDE (1 << 23) -#define HS_ZERO(x) ((x) << 16) -#define HS_ZERO_MASK (0xf << 16) -#define HS_ZERO_SHIFT 16 -#define HS_TRAIL_OVERRIDE (1 << 15) -#define HS_TRAIL(x) ((x) << 8) -#define HS_TRAIL_MASK (0x7 << 8) -#define HS_TRAIL_SHIFT 8 -#define HS_EXIT_OVERRIDE (1 << 7) -#define HS_EXIT(x) ((x) << 0) -#define HS_EXIT_MASK (0x7 << 0) -#define HS_EXIT_SHIFT 0 - -#define _DPHY_TA_TIMING_PARAM_0 0x162188 -#define _DPHY_TA_TIMING_PARAM_1 0x6c188 -#define DPHY_TA_TIMING_PARAM(port) _MMIO_PORT(port, \ - _DPHY_TA_TIMING_PARAM_0,\ - _DPHY_TA_TIMING_PARAM_1) -#define _DSI_TA_TIMING_PARAM_0 0x6b098 -#define _DSI_TA_TIMING_PARAM_1 0x6b898 -#define DSI_TA_TIMING_PARAM(port) _MMIO_PORT(port, \ - _DSI_TA_TIMING_PARAM_0,\ - _DSI_TA_TIMING_PARAM_1) -#define TA_SURE_OVERRIDE (1 << 31) -#define TA_SURE(x) ((x) << 16) -#define TA_SURE_MASK (0x1f << 16) -#define TA_SURE_SHIFT 16 -#define TA_GO_OVERRIDE (1 << 15) -#define TA_GO(x) ((x) << 8) -#define TA_GO_MASK (0xf << 8) -#define TA_GO_SHIFT 8 -#define TA_GET_OVERRIDE (1 << 7) -#define TA_GET(x) ((x) << 0) -#define TA_GET_MASK (0xf << 0) -#define TA_GET_SHIFT 0 - -/* DSI transcoder configuration */ -#define _DSI_TRANS_FUNC_CONF_0 0x6b030 -#define _DSI_TRANS_FUNC_CONF_1 0x6b830 -#define DSI_TRANS_FUNC_CONF(tc) _MMIO_DSI(tc, \ - _DSI_TRANS_FUNC_CONF_0,\ - _DSI_TRANS_FUNC_CONF_1) -#define OP_MODE_MASK (0x3 << 28) -#define OP_MODE_SHIFT 28 -#define CMD_MODE_NO_GATE (0x0 << 28) -#define CMD_MODE_TE_GATE (0x1 << 28) -#define VIDEO_MODE_SYNC_EVENT (0x2 << 28) -#define VIDEO_MODE_SYNC_PULSE (0x3 << 28) -#define TE_SOURCE_GPIO (1 << 27) -#define LINK_READY (1 << 20) -#define PIX_FMT_MASK (0x3 << 16) -#define PIX_FMT_SHIFT 16 -#define PIX_FMT_RGB565 (0x0 << 16) -#define PIX_FMT_RGB666_PACKED (0x1 << 16) -#define PIX_FMT_RGB666_LOOSE (0x2 << 16) -#define PIX_FMT_RGB888 (0x3 << 16) -#define PIX_FMT_RGB101010 (0x4 << 16) -#define PIX_FMT_RGB121212 (0x5 << 16) -#define PIX_FMT_COMPRESSED (0x6 << 16) -#define BGR_TRANSMISSION (1 << 15) -#define PIX_VIRT_CHAN(x) ((x) << 12) -#define PIX_VIRT_CHAN_MASK (0x3 << 12) -#define PIX_VIRT_CHAN_SHIFT 12 -#define PIX_BUF_THRESHOLD_MASK (0x3 << 10) -#define PIX_BUF_THRESHOLD_SHIFT 10 -#define PIX_BUF_THRESHOLD_1_4 (0x0 << 10) -#define PIX_BUF_THRESHOLD_1_2 (0x1 << 10) -#define PIX_BUF_THRESHOLD_3_4 (0x2 << 10) -#define PIX_BUF_THRESHOLD_FULL (0x3 << 10) -#define CONTINUOUS_CLK_MASK (0x3 << 8) -#define CONTINUOUS_CLK_SHIFT 8 -#define CLK_ENTER_LP_AFTER_DATA (0x0 << 8) -#define CLK_HS_OR_LP (0x2 << 8) -#define CLK_HS_CONTINUOUS (0x3 << 8) -#define LINK_CALIBRATION_MASK (0x3 << 4) -#define LINK_CALIBRATION_SHIFT 4 -#define CALIBRATION_DISABLED (0x0 << 4) -#define CALIBRATION_ENABLED_INITIAL_ONLY (0x2 << 4) -#define CALIBRATION_ENABLED_INITIAL_PERIODIC (0x3 << 4) -#define BLANKING_PACKET_ENABLE (1 << 2) -#define S3D_ORIENTATION_LANDSCAPE (1 << 1) -#define EOTP_DISABLED (1 << 0) - -#define _DSI_CMD_RXCTL_0 0x6b0d4 -#define _DSI_CMD_RXCTL_1 0x6b8d4 -#define DSI_CMD_RXCTL(tc) _MMIO_DSI(tc, \ - _DSI_CMD_RXCTL_0,\ - _DSI_CMD_RXCTL_1) -#define READ_UNLOADS_DW (1 << 16) -#define RECEIVED_UNASSIGNED_TRIGGER (1 << 15) -#define RECEIVED_ACKNOWLEDGE_TRIGGER (1 << 14) -#define RECEIVED_TEAR_EFFECT_TRIGGER (1 << 13) -#define RECEIVED_RESET_TRIGGER (1 << 12) -#define RECEIVED_PAYLOAD_WAS_LOST (1 << 11) -#define RECEIVED_CRC_WAS_LOST (1 << 10) -#define NUMBER_RX_PLOAD_DW_MASK (0xff << 0) -#define NUMBER_RX_PLOAD_DW_SHIFT 0 - -#define _DSI_CMD_TXCTL_0 0x6b0d0 -#define _DSI_CMD_TXCTL_1 0x6b8d0 -#define DSI_CMD_TXCTL(tc) _MMIO_DSI(tc, \ - _DSI_CMD_TXCTL_0,\ - _DSI_CMD_TXCTL_1) -#define KEEP_LINK_IN_HS (1 << 24) -#define FREE_HEADER_CREDIT_MASK (0x1f << 8) -#define FREE_HEADER_CREDIT_SHIFT 0x8 -#define FREE_PLOAD_CREDIT_MASK (0xff << 0) -#define FREE_PLOAD_CREDIT_SHIFT 0 -#define MAX_HEADER_CREDIT 0x10 -#define MAX_PLOAD_CREDIT 0x40 - -#define _DSI_CMD_TXHDR_0 0x6b100 -#define _DSI_CMD_TXHDR_1 0x6b900 -#define DSI_CMD_TXHDR(tc) _MMIO_DSI(tc, \ - _DSI_CMD_TXHDR_0,\ - _DSI_CMD_TXHDR_1) -#define PAYLOAD_PRESENT (1 << 31) -#define LP_DATA_TRANSFER (1 << 30) -#define VBLANK_FENCE (1 << 29) -#define PARAM_WC_MASK (0xffff << 8) -#define PARAM_WC_LOWER_SHIFT 8 -#define PARAM_WC_UPPER_SHIFT 16 -#define VC_MASK (0x3 << 6) -#define VC_SHIFT 6 -#define DT_MASK (0x3f << 0) -#define DT_SHIFT 0 - -#define _DSI_CMD_TXPYLD_0 0x6b104 -#define _DSI_CMD_TXPYLD_1 0x6b904 -#define DSI_CMD_TXPYLD(tc) _MMIO_DSI(tc, \ - _DSI_CMD_TXPYLD_0,\ - _DSI_CMD_TXPYLD_1) - -#define _DSI_LP_MSG_0 0x6b0d8 -#define _DSI_LP_MSG_1 0x6b8d8 -#define DSI_LP_MSG(tc) _MMIO_DSI(tc, \ - _DSI_LP_MSG_0,\ - _DSI_LP_MSG_1) -#define LPTX_IN_PROGRESS (1 << 17) -#define LINK_IN_ULPS (1 << 16) -#define LINK_ULPS_TYPE_LP11 (1 << 8) -#define LINK_ENTER_ULPS (1 << 0) - -/* DSI timeout registers */ -#define _DSI_HSTX_TO_0 0x6b044 -#define _DSI_HSTX_TO_1 0x6b844 -#define DSI_HSTX_TO(tc) _MMIO_DSI(tc, \ - _DSI_HSTX_TO_0,\ - _DSI_HSTX_TO_1) -#define HSTX_TIMEOUT_VALUE_MASK (0xffff << 16) -#define HSTX_TIMEOUT_VALUE_SHIFT 16 -#define HSTX_TIMEOUT_VALUE(x) ((x) << 16) -#define HSTX_TIMED_OUT (1 << 0) - -#define _DSI_LPRX_HOST_TO_0 0x6b048 -#define _DSI_LPRX_HOST_TO_1 0x6b848 -#define DSI_LPRX_HOST_TO(tc) _MMIO_DSI(tc, \ - _DSI_LPRX_HOST_TO_0,\ - _DSI_LPRX_HOST_TO_1) -#define LPRX_TIMED_OUT (1 << 16) -#define LPRX_TIMEOUT_VALUE_MASK (0xffff << 0) -#define LPRX_TIMEOUT_VALUE_SHIFT 0 -#define LPRX_TIMEOUT_VALUE(x) ((x) << 0) - -#define _DSI_PWAIT_TO_0 0x6b040 -#define _DSI_PWAIT_TO_1 0x6b840 -#define DSI_PWAIT_TO(tc) _MMIO_DSI(tc, \ - _DSI_PWAIT_TO_0,\ - _DSI_PWAIT_TO_1) -#define PRESET_TIMEOUT_VALUE_MASK (0xffff << 16) -#define PRESET_TIMEOUT_VALUE_SHIFT 16 -#define PRESET_TIMEOUT_VALUE(x) ((x) << 16) -#define PRESPONSE_TIMEOUT_VALUE_MASK (0xffff << 0) -#define PRESPONSE_TIMEOUT_VALUE_SHIFT 0 -#define PRESPONSE_TIMEOUT_VALUE(x) ((x) << 0) - -#define _DSI_TA_TO_0 0x6b04c -#define _DSI_TA_TO_1 0x6b84c -#define DSI_TA_TO(tc) _MMIO_DSI(tc, \ - _DSI_TA_TO_0,\ - _DSI_TA_TO_1) -#define TA_TIMED_OUT (1 << 16) -#define TA_TIMEOUT_VALUE_MASK (0xffff << 0) -#define TA_TIMEOUT_VALUE_SHIFT 0 -#define TA_TIMEOUT_VALUE(x) ((x) << 0) - -/* bits 31:0 */ -#define _MIPIA_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb084) -#define _MIPIC_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb884) -#define MIPI_DBI_BW_CTRL(port) _MMIO_MIPI(port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL) - -#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb088) -#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb888) -#define MIPI_CLK_LANE_SWITCH_TIME_CNT(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT) -#define LP_HS_SSW_CNT_SHIFT 16 -#define LP_HS_SSW_CNT_MASK (0xffff << 16) -#define HS_LP_PWR_SW_CNT_SHIFT 0 -#define HS_LP_PWR_SW_CNT_MASK (0xffff << 0) - -#define _MIPIA_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb08c) -#define _MIPIC_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb88c) -#define MIPI_STOP_STATE_STALL(port) _MMIO_MIPI(port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL) -#define STOP_STATE_STALL_COUNTER_SHIFT 0 -#define STOP_STATE_STALL_COUNTER_MASK (0xff << 0) - -#define _MIPIA_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb090) -#define _MIPIC_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb890) -#define MIPI_INTR_STAT_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1) -#define _MIPIA_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb094) -#define _MIPIC_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb894) -#define MIPI_INTR_EN_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1) -#define RX_CONTENTION_DETECTED (1 << 0) - -/* XXX: only pipe A ?!? */ -#define MIPIA_DBI_TYPEC_CTRL (dev_priv->mipi_mmio_base + 0xb100) -#define DBI_TYPEC_ENABLE (1 << 31) -#define DBI_TYPEC_WIP (1 << 30) -#define DBI_TYPEC_OPTION_SHIFT 28 -#define DBI_TYPEC_OPTION_MASK (3 << 28) -#define DBI_TYPEC_FREQ_SHIFT 24 -#define DBI_TYPEC_FREQ_MASK (0xf << 24) -#define DBI_TYPEC_OVERRIDE (1 << 8) -#define DBI_TYPEC_OVERRIDE_COUNTER_SHIFT 0 -#define DBI_TYPEC_OVERRIDE_COUNTER_MASK (0xff << 0) - - -/* MIPI adapter registers */ - -#define _MIPIA_CTRL (dev_priv->mipi_mmio_base + 0xb104) -#define _MIPIC_CTRL (dev_priv->mipi_mmio_base + 0xb904) -#define MIPI_CTRL(port) _MMIO_MIPI(port, _MIPIA_CTRL, _MIPIC_CTRL) -#define ESCAPE_CLOCK_DIVIDER_SHIFT 5 /* A only */ -#define ESCAPE_CLOCK_DIVIDER_MASK (3 << 5) -#define ESCAPE_CLOCK_DIVIDER_1 (0 << 5) -#define ESCAPE_CLOCK_DIVIDER_2 (1 << 5) -#define ESCAPE_CLOCK_DIVIDER_4 (2 << 5) -#define READ_REQUEST_PRIORITY_SHIFT 3 -#define READ_REQUEST_PRIORITY_MASK (3 << 3) -#define READ_REQUEST_PRIORITY_LOW (0 << 3) -#define READ_REQUEST_PRIORITY_HIGH (3 << 3) -#define RGB_FLIP_TO_BGR (1 << 2) - -#define BXT_PIPE_SELECT_SHIFT 7 -#define BXT_PIPE_SELECT_MASK (7 << 7) -#define BXT_PIPE_SELECT(pipe) ((pipe) << 7) -#define GLK_PHY_STATUS_PORT_READY (1 << 31) /* RO */ -#define GLK_ULPS_NOT_ACTIVE (1 << 30) /* RO */ -#define GLK_MIPIIO_RESET_RELEASED (1 << 28) -#define GLK_CLOCK_LANE_STOP_STATE (1 << 27) /* RO */ -#define GLK_DATA_LANE_STOP_STATE (1 << 26) /* RO */ -#define GLK_LP_WAKE (1 << 22) -#define GLK_LP11_LOW_PWR_MODE (1 << 21) -#define GLK_LP00_LOW_PWR_MODE (1 << 20) -#define GLK_FIREWALL_ENABLE (1 << 16) -#define BXT_PIXEL_OVERLAP_CNT_MASK (0xf << 10) -#define BXT_PIXEL_OVERLAP_CNT_SHIFT 10 -#define BXT_DSC_ENABLE (1 << 3) -#define BXT_RGB_FLIP (1 << 2) -#define GLK_MIPIIO_PORT_POWERED (1 << 1) /* RO */ -#define GLK_MIPIIO_ENABLE (1 << 0) - -#define _MIPIA_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb108) -#define _MIPIC_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb908) -#define MIPI_DATA_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS) -#define DATA_MEM_ADDRESS_SHIFT 5 -#define DATA_MEM_ADDRESS_MASK (0x7ffffff << 5) -#define DATA_VALID (1 << 0) - -#define _MIPIA_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb10c) -#define _MIPIC_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb90c) -#define MIPI_DATA_LENGTH(port) _MMIO_MIPI(port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH) -#define DATA_LENGTH_SHIFT 0 -#define DATA_LENGTH_MASK (0xfffff << 0) - -#define _MIPIA_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb110) -#define _MIPIC_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb910) -#define MIPI_COMMAND_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS) -#define COMMAND_MEM_ADDRESS_SHIFT 5 -#define COMMAND_MEM_ADDRESS_MASK (0x7ffffff << 5) -#define AUTO_PWG_ENABLE (1 << 2) -#define MEMORY_WRITE_DATA_FROM_PIPE_RENDERING (1 << 1) -#define COMMAND_VALID (1 << 0) - -#define _MIPIA_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb114) -#define _MIPIC_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb914) -#define MIPI_COMMAND_LENGTH(port) _MMIO_MIPI(port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH) -#define COMMAND_LENGTH_SHIFT(n) (8 * (n)) /* n: 0...3 */ -#define COMMAND_LENGTH_MASK(n) (0xff << (8 * (n))) - -#define _MIPIA_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb118) -#define _MIPIC_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb918) -#define MIPI_READ_DATA_RETURN(port, n) _MMIO(_MIPI(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) + 4 * (n)) /* n: 0...7 */ - -#define _MIPIA_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb138) -#define _MIPIC_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb938) -#define MIPI_READ_DATA_VALID(port) _MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID) -#define READ_DATA_VALID(n) (1 << (n)) - -/* MOCS (Memory Object Control State) registers */ -#define GEN9_LNCFCMOCS(i) _MMIO(0xb020 + (i) * 4) /* L3 Cache Control */ -#define GEN9_LNCFCMOCS_REG_COUNT 32 - -#define __GEN9_RCS0_MOCS0 0xc800 -#define GEN9_GFX_MOCS(i) _MMIO(__GEN9_RCS0_MOCS0 + (i) * 4) -#define __GEN9_VCS0_MOCS0 0xc900 -#define GEN9_MFX0_MOCS(i) _MMIO(__GEN9_VCS0_MOCS0 + (i) * 4) -#define __GEN9_VCS1_MOCS0 0xca00 -#define GEN9_MFX1_MOCS(i) _MMIO(__GEN9_VCS1_MOCS0 + (i) * 4) -#define __GEN9_VECS0_MOCS0 0xcb00 -#define GEN9_VEBOX_MOCS(i) _MMIO(__GEN9_VECS0_MOCS0 + (i) * 4) -#define __GEN9_BCS0_MOCS0 0xcc00 -#define GEN9_BLT_MOCS(i) _MMIO(__GEN9_BCS0_MOCS0 + (i) * 4) -#define __GEN11_VCS2_MOCS0 0x10000 -#define GEN11_MFX2_MOCS(i) _MMIO(__GEN11_VCS2_MOCS0 + (i) * 4) - -#define GEN9_SCRATCH_LNCF1 _MMIO(0xb008) -#define GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE REG_BIT(0) - -#define GEN9_SCRATCH1 _MMIO(0xb11c) -#define EVICTION_PERF_FIX_ENABLE REG_BIT(8) - -#define GEN10_SCRATCH_LNCF2 _MMIO(0xb0a0) -#define PMFLUSHDONE_LNICRSDROP (1 << 20) -#define PMFLUSH_GAPL3UNBLOCK (1 << 21) -#define PMFLUSHDONE_LNEBLK (1 << 22) - -#define XEHP_L3NODEARBCFG _MMIO(0xb0b4) -#define XEHP_LNESPARE REG_BIT(19) - -#define GEN12_GLOBAL_MOCS(i) _MMIO(0x4000 + (i) * 4) /* Global MOCS regs */ - #define GEN12_GSMBASE _MMIO(0x108100) #define GEN12_DSMBASE _MMIO(0x1080C0) @@ -12641,6 +8465,9 @@ enum skl_power_gate { #define SGGI_DIS REG_BIT(15) #define SGR_DIS REG_BIT(13) +#define XEHPSDV_FLAT_CCS_BASE_ADDR _MMIO(0x4910) +#define XEHPSDV_CCS_BASE_SHIFT 8 + /* gamt regs */ #define GEN8_L3_LRA_1_GPGPU _MMIO(0x4dd4) #define GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW 0x67F1427F /* max/min for LRA1/2 */ @@ -12654,8 +8481,10 @@ enum skl_power_gate { #define _ICL_PHY_MISC_A 0x64C00 #define _ICL_PHY_MISC_B 0x64C04 -#define ICL_PHY_MISC(port) _MMIO_PORT(port, _ICL_PHY_MISC_A, \ - _ICL_PHY_MISC_B) +#define _DG2_PHY_MISC_TC1 0x64C14 /* TC1="PHY E" but offset as if "PHY F" */ +#define ICL_PHY_MISC(port) _MMIO_PORT(port, _ICL_PHY_MISC_A, _ICL_PHY_MISC_B) +#define DG2_PHY_MISC(port) ((port) == PHY_E ? _MMIO(_DG2_PHY_MISC_TC1) : \ + ICL_PHY_MISC(port)) #define ICL_PHY_MISC_MUX_DDID (1 << 28) #define ICL_PHY_MISC_DE_IO_COMP_PWR_DOWN (1 << 23) #define DG2_PHY_DP_TX_ACK_MASK REG_GENMASK(23, 20) @@ -12995,6 +8824,14 @@ enum skl_power_gate { #define TCSS_DDI_STATUS_HPD_LIVE_STATUS_TBT REG_BIT(1) #define TCSS_DDI_STATUS_HPD_LIVE_STATUS_ALT REG_BIT(0) +#define PRIMARY_SPI_TRIGGER _MMIO(0x102040) +#define PRIMARY_SPI_ADDRESS _MMIO(0x102080) +#define PRIMARY_SPI_REGIONID _MMIO(0x102084) +#define SPI_STATIC_REGIONS _MMIO(0x102090) +#define OPTIONROM_SPI_REGIONID_MASK REG_GENMASK(7, 0) +#define OROM_OFFSET _MMIO(0x1020c0) +#define OROM_OFFSET_MASK REG_GENMASK(20, 16) + /* This register controls the Display State Buffer (DSB) engines. */ #define _DSBSL_INSTANCE_BASE 0x70B00 #define DSBSL_INSTANCE(pipe, id) (_DSBSL_INSTANCE_BASE + \ @@ -13005,19 +8842,14 @@ enum skl_power_gate { #define DSB_ENABLE (1 << 31) #define DSB_STATUS (1 << 0) -#define TGL_ROOT_DEVICE_ID 0x9A00 -#define TGL_ROOT_DEVICE_MASK 0xFF00 -#define TGL_ROOT_DEVICE_SKU_MASK 0xF -#define TGL_ROOT_DEVICE_SKU_ULX 0x2 -#define TGL_ROOT_DEVICE_SKU_ULT 0x4 - #define CLKREQ_POLICY _MMIO(0x101038) #define CLKREQ_POLICY_MEM_UP_OVRD REG_BIT(1) #define CLKGATE_DIS_MISC _MMIO(0x46534) #define CLKGATE_DIS_MISC_DMASC_GATING_DIS REG_BIT(21) -#define SLICE_COMMON_ECO_CHICKEN1 _MMIO(0x731C) -#define MSC_MSAA_REODER_BUF_BYPASS_DISABLE REG_BIT(14) +#define GEN12_CULLBIT1 _MMIO(0x6100) +#define GEN12_CULLBIT2 _MMIO(0x7030) +#define GEN12_STATE_ACK_DEBUG _MMIO(0x20BC) #endif /* _I915_REG_H_ */ diff --git a/drivers/gpu/drm/i915/i915_reg_defs.h b/drivers/gpu/drm/i915/i915_reg_defs.h new file mode 100644 index 000000000000..d78d78fce431 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_reg_defs.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __I915_REG_DEFS__ +#define __I915_REG_DEFS__ + +#include <linux/bitfield.h> +#include <linux/bits.h> + +/** + * REG_BIT() - Prepare a u32 bit value + * @__n: 0-based bit number + * + * Local wrapper for BIT() to force u32, with compile time checks. + * + * @return: Value with bit @__n set. + */ +#define REG_BIT(__n) \ + ((u32)(BIT(__n) + \ + BUILD_BUG_ON_ZERO(__is_constexpr(__n) && \ + ((__n) < 0 || (__n) > 31)))) + +/** + * REG_GENMASK() - Prepare a continuous u32 bitmask + * @__high: 0-based high bit + * @__low: 0-based low bit + * + * Local wrapper for GENMASK() to force u32, with compile time checks. + * + * @return: Continuous bitmask from @__high to @__low, inclusive. + */ +#define REG_GENMASK(__high, __low) \ + ((u32)(GENMASK(__high, __low) + \ + BUILD_BUG_ON_ZERO(__is_constexpr(__high) && \ + __is_constexpr(__low) && \ + ((__low) < 0 || (__high) > 31 || (__low) > (__high))))) + +/** + * REG_GENMASK64() - Prepare a continuous u64 bitmask + * @__high: 0-based high bit + * @__low: 0-based low bit + * + * Local wrapper for GENMASK_ULL() to force u64, with compile time checks. + * + * @return: Continuous bitmask from @__high to @__low, inclusive. + */ +#define REG_GENMASK64(__high, __low) \ + ((u64)(GENMASK_ULL(__high, __low) + \ + BUILD_BUG_ON_ZERO(__is_constexpr(__high) && \ + __is_constexpr(__low) && \ + ((__low) < 0 || (__high) > 63 || (__low) > (__high))))) + +/* + * Local integer constant expression version of is_power_of_2(). + */ +#define IS_POWER_OF_2(__x) ((__x) && (((__x) & ((__x) - 1)) == 0)) + +/** + * REG_FIELD_PREP() - Prepare a u32 bitfield value + * @__mask: shifted mask defining the field's length and position + * @__val: value to put in the field + * + * Local copy of FIELD_PREP() to generate an integer constant expression, force + * u32 and for consistency with REG_FIELD_GET(), REG_BIT() and REG_GENMASK(). + * + * @return: @__val masked and shifted into the field defined by @__mask. + */ +#define REG_FIELD_PREP(__mask, __val) \ + ((u32)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) + \ + BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) + \ + BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U32_MAX) + \ + BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \ + BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0)))) + +/** + * REG_FIELD_GET() - Extract a u32 bitfield value + * @__mask: shifted mask defining the field's length and position + * @__val: value to extract the bitfield value from + * + * Local wrapper for FIELD_GET() to force u32 and for consistency with + * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK(). + * + * @return: Masked and shifted value of the field defined by @__mask in @__val. + */ +#define REG_FIELD_GET(__mask, __val) ((u32)FIELD_GET(__mask, __val)) + +/** + * REG_FIELD_GET64() - Extract a u64 bitfield value + * @__mask: shifted mask defining the field's length and position + * @__val: value to extract the bitfield value from + * + * Local wrapper for FIELD_GET() to force u64 and for consistency with + * REG_GENMASK64(). + * + * @return: Masked and shifted value of the field defined by @__mask in @__val. + */ +#define REG_FIELD_GET64(__mask, __val) ((u64)FIELD_GET(__mask, __val)) + +typedef struct { + u32 reg; +} i915_reg_t; + +#define _MMIO(r) ((const i915_reg_t){ .reg = (r) }) + +#define INVALID_MMIO_REG _MMIO(0) + +static __always_inline u32 i915_mmio_reg_offset(i915_reg_t reg) +{ + return reg.reg; +} + +static inline bool i915_mmio_reg_equal(i915_reg_t a, i915_reg_t b) +{ + return i915_mmio_reg_offset(a) == i915_mmio_reg_offset(b); +} + +static inline bool i915_mmio_reg_valid(i915_reg_t reg) +{ + return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG); +} + +#define VLV_DISPLAY_BASE 0x180000 + +#define GEN12_SFC_DONE_MAX 4 + +#endif /* __I915_REG_DEFS__ */ diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 76cf5ac91e94..582770360ad1 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -36,6 +36,7 @@ #include "gt/intel_context.h" #include "gt/intel_engine.h" #include "gt/intel_engine_heartbeat.h" +#include "gt/intel_engine_regs.h" #include "gt/intel_gpu_commands.h" #include "gt/intel_reset.h" #include "gt/intel_ring.h" @@ -43,6 +44,7 @@ #include "i915_active.h" #include "i915_deps.h" +#include "i915_driver.h" #include "i915_drv.h" #include "i915_trace.h" #include "intel_pm.h" @@ -116,8 +118,10 @@ static void i915_fence_release(struct dma_fence *fence) rq->guc_prio != GUC_PRIO_FINI); i915_request_free_capture_list(fetch_and_zero(&rq->capture_list)); - if (i915_vma_snapshot_present(&rq->batch_snapshot)) - i915_vma_snapshot_put_onstack(&rq->batch_snapshot); + if (rq->batch_res) { + i915_vma_resource_put(rq->batch_res); + rq->batch_res = NULL; + } /* * The request is put onto a RCU freelist (i.e. the address @@ -308,7 +312,7 @@ void i915_request_free_capture_list(struct i915_capture_list *capture) while (capture) { struct i915_capture_list *next = capture->next; - i915_vma_snapshot_put(capture->vma_snapshot); + i915_vma_resource_put(capture->vma_res); kfree(capture); capture = next; } @@ -854,7 +858,7 @@ static void __i915_request_ctor(void *arg) i915_sw_fence_init(&rq->semaphore, semaphore_notify); clear_capture_list(rq); - rq->batch_snapshot.present = false; + rq->batch_res = NULL; init_llist_head(&rq->execute_cb); } @@ -960,7 +964,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) __rq_init_watchdog(rq); assert_capture_list_is_null(rq); GEM_BUG_ON(!llist_empty(&rq->execute_cb)); - GEM_BUG_ON(i915_vma_snapshot_present(&rq->batch_snapshot)); + GEM_BUG_ON(rq->batch_res); /* * Reserve space in the ring buffer for all the commands required to diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index 170ee78c2858..28b1f9db5487 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -40,7 +40,7 @@ #include "i915_scheduler.h" #include "i915_selftest.h" #include "i915_sw_fence.h" -#include "i915_vma_snapshot.h" +#include "i915_vma_resource.h" #include <uapi/drm/i915_drm.h> @@ -52,7 +52,7 @@ struct i915_request; #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) struct i915_capture_list { - struct i915_vma_snapshot *vma_snapshot; + struct i915_vma_resource *vma_res; struct i915_capture_list *next; }; @@ -300,7 +300,7 @@ struct i915_request { /** Batch buffer pointer for selftest internal use. */ I915_SELFTEST_DECLARE(struct i915_vma *batch); - struct i915_vma_snapshot batch_snapshot; + struct i915_vma_resource *batch_res; #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) /** diff --git a/drivers/gpu/drm/i915/i915_scatterlist.c b/drivers/gpu/drm/i915/i915_scatterlist.c index 41f2adb6a583..159571b9bd24 100644 --- a/drivers/gpu/drm/i915/i915_scatterlist.c +++ b/drivers/gpu/drm/i915/i915_scatterlist.c @@ -5,10 +5,9 @@ */ #include "i915_scatterlist.h" - -#include "i915_buddy.h" #include "i915_ttm_buddy_manager.h" +#include <drm/drm_buddy.h> #include <drm/drm_mm.h> #include <linux/slab.h> @@ -153,9 +152,9 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); const u64 size = res->num_pages << PAGE_SHIFT; const u64 max_segment = rounddown(UINT_MAX, PAGE_SIZE); - struct i915_buddy_mm *mm = bman_res->mm; + struct drm_buddy *mm = bman_res->mm; struct list_head *blocks = &bman_res->blocks; - struct i915_buddy_block *block; + struct drm_buddy_block *block; struct i915_refct_sgt *rsgt; struct scatterlist *sg; struct sg_table *st; @@ -181,8 +180,8 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, list_for_each_entry(block, blocks, link) { u64 block_size, offset; - block_size = min_t(u64, size, i915_buddy_block_size(mm, block)); - offset = i915_buddy_block_offset(block); + block_size = min_t(u64, size, drm_buddy_block_size(mm, block)); + offset = drm_buddy_block_offset(block); while (block_size) { u64 len; diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index f7b55f34dba8..889f5b7dc78e 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -32,6 +32,7 @@ #include "i915_drv.h" #include "i915_reg.h" #include "i915_suspend.h" +#include "intel_pci_config.h" static void intel_save_swf(struct drm_i915_private *dev_priv) { diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index fae4d1f4f275..a4d1759375b9 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -30,6 +30,7 @@ #include <linux/stat.h> #include <linux/sysfs.h> +#include "gt/intel_gt_regs.h" #include "gt/intel_rc6.h" #include "gt/intel_rps.h" #include "gt/sysfs_engines.h" diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index d59fbb019032..129f668f21ff 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -8,16 +8,20 @@ #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> +#include <drm/drm_buddy.h> + #include "i915_ttm_buddy_manager.h" -#include "i915_buddy.h" #include "i915_gem.h" struct i915_ttm_buddy_manager { struct ttm_resource_manager manager; - struct i915_buddy_mm mm; + struct drm_buddy mm; struct list_head reserved; struct mutex lock; + unsigned long visible_size; + unsigned long visible_avail; + unsigned long visible_reserved; u64 default_page_size; }; @@ -34,14 +38,15 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); struct i915_ttm_buddy_resource *bman_res; - struct i915_buddy_mm *mm = &bman->mm; - unsigned long n_pages; - unsigned int min_order; + struct drm_buddy *mm = &bman->mm; + unsigned long n_pages, lpfn; u64 min_page_size; u64 size; int err; - GEM_BUG_ON(place->fpfn || place->lpfn); + lpfn = place->lpfn; + if (!lpfn) + lpfn = man->size; bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL); if (!bman_res) @@ -51,6 +56,12 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, INIT_LIST_HEAD(&bman_res->blocks); bman_res->mm = mm; + if (place->flags & TTM_PL_FLAG_TOPDOWN) + bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; + + if (place->fpfn || lpfn != man->size) + bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION; + GEM_BUG_ON(!bman_res->base.num_pages); size = bman_res->base.num_pages << PAGE_SHIFT; @@ -59,56 +70,93 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, min_page_size = bo->page_alignment << PAGE_SHIFT; GEM_BUG_ON(min_page_size < mm->chunk_size); - min_order = ilog2(min_page_size) - ilog2(mm->chunk_size); + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + unsigned long pages; + size = roundup_pow_of_two(size); - min_order = ilog2(size) - ilog2(mm->chunk_size); + min_page_size = size; + + pages = size >> ilog2(mm->chunk_size); + if (pages > lpfn) + lpfn = pages; } - if (size > mm->size) { + if (size > lpfn << PAGE_SHIFT) { err = -E2BIG; goto err_free_res; } n_pages = size >> ilog2(mm->chunk_size); - do { - struct i915_buddy_block *block; - unsigned int order; + mutex_lock(&bman->lock); + if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) { + mutex_unlock(&bman->lock); + err = -ENOSPC; + goto err_free_res; + } - order = fls(n_pages) - 1; - GEM_BUG_ON(order > mm->max_order); - GEM_BUG_ON(order < min_order); + err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, + (u64)lpfn << PAGE_SHIFT, + (u64)n_pages << PAGE_SHIFT, + min_page_size, + &bman_res->blocks, + bman_res->flags); + mutex_unlock(&bman->lock); + if (unlikely(err)) + goto err_free_blocks; - do { - mutex_lock(&bman->lock); - block = i915_buddy_alloc(mm, order); - mutex_unlock(&bman->lock); - if (!IS_ERR(block)) - break; + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + u64 original_size = (u64)bman_res->base.num_pages << PAGE_SHIFT; - if (order-- == min_order) { - err = -ENOSPC; - goto err_free_blocks; - } - } while (1); + mutex_lock(&bman->lock); + drm_buddy_block_trim(mm, + original_size, + &bman_res->blocks); + mutex_unlock(&bman->lock); + } - n_pages -= BIT(order); + if (lpfn <= bman->visible_size) { + bman_res->used_visible_size = bman_res->base.num_pages; + } else { + struct drm_buddy_block *block; - list_add_tail(&block->link, &bman_res->blocks); + list_for_each_entry(block, &bman_res->blocks, link) { + unsigned long start = + drm_buddy_block_offset(block) >> PAGE_SHIFT; - if (!n_pages) - break; - } while (1); + if (start < bman->visible_size) { + unsigned long end = start + + (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); + + bman_res->used_visible_size += + min(end, bman->visible_size) - start; + } + } + } + + if (bman_res->used_visible_size) { + mutex_lock(&bman->lock); + bman->visible_avail -= bman_res->used_visible_size; + mutex_unlock(&bman->lock); + } + + if (place->lpfn - place->fpfn == n_pages) + bman_res->base.start = place->fpfn; + else if (lpfn <= bman->visible_size) + bman_res->base.start = 0; + else + bman_res->base.start = bman->visible_size; *res = &bman_res->base; return 0; err_free_blocks: mutex_lock(&bman->lock); - i915_buddy_free_list(mm, &bman_res->blocks); + drm_buddy_free_list(mm, &bman_res->blocks); mutex_unlock(&bman->lock); err_free_res: + ttm_resource_fini(man, &bman_res->base); kfree(bman_res); return err; } @@ -120,9 +168,11 @@ static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man, struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); mutex_lock(&bman->lock); - i915_buddy_free_list(&bman->mm, &bman_res->blocks); + drm_buddy_free_list(&bman->mm, &bman_res->blocks); + bman->visible_avail += bman_res->used_visible_size; mutex_unlock(&bman->lock); + ttm_resource_fini(man, res); kfree(bman_res); } @@ -130,17 +180,23 @@ static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man, struct drm_printer *printer) { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); - struct i915_buddy_block *block; + struct drm_buddy_block *block; mutex_lock(&bman->lock); drm_printf(printer, "default_page_size: %lluKiB\n", bman->default_page_size >> 10); + drm_printf(printer, "visible_avail: %lluMiB\n", + (u64)bman->visible_avail << PAGE_SHIFT >> 20); + drm_printf(printer, "visible_size: %lluMiB\n", + (u64)bman->visible_size << PAGE_SHIFT >> 20); + drm_printf(printer, "visible_reserved: %lluMiB\n", + (u64)bman->visible_reserved << PAGE_SHIFT >> 20); - i915_buddy_print(&bman->mm, printer); + drm_buddy_print(&bman->mm, printer); drm_printf(printer, "reserved:\n"); list_for_each_entry(block, &bman->reserved, link) - i915_buddy_block_print(&bman->mm, block, printer); + drm_buddy_block_print(&bman->mm, block, printer); mutex_unlock(&bman->lock); } @@ -156,6 +212,7 @@ static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = { * @type: Memory type we want to manage * @use_tt: Set use_tt for the manager * @size: The size in bytes to manage + * @visible_size: The CPU visible size in bytes to manage * @default_page_size: The default minimum page size in bytes for allocations, * this must be at least as large as @chunk_size, and can be overridden by * setting the BO page_alignment, to be larger or smaller as needed. @@ -179,7 +236,7 @@ static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = { */ int i915_ttm_buddy_man_init(struct ttm_device *bdev, unsigned int type, bool use_tt, - u64 size, u64 default_page_size, + u64 size, u64 visible_size, u64 default_page_size, u64 chunk_size) { struct ttm_resource_manager *man; @@ -190,7 +247,7 @@ int i915_ttm_buddy_man_init(struct ttm_device *bdev, if (!bman) return -ENOMEM; - err = i915_buddy_init(&bman->mm, size, chunk_size); + err = drm_buddy_init(&bman->mm, size, chunk_size); if (err) goto err_free_bman; @@ -198,11 +255,13 @@ int i915_ttm_buddy_man_init(struct ttm_device *bdev, INIT_LIST_HEAD(&bman->reserved); GEM_BUG_ON(default_page_size < chunk_size); bman->default_page_size = default_page_size; + bman->visible_size = visible_size >> PAGE_SHIFT; + bman->visible_avail = bman->visible_size; man = &bman->manager; man->use_tt = use_tt; man->func = &i915_ttm_buddy_manager_func; - ttm_resource_manager_init(man, bman->mm.size >> PAGE_SHIFT); + ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT); ttm_resource_manager_set_used(man, true); ttm_set_driver_manager(bdev, type, man); @@ -228,7 +287,7 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type) { struct ttm_resource_manager *man = ttm_manager_type(bdev, type); struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); - struct i915_buddy_mm *mm = &bman->mm; + struct drm_buddy *mm = &bman->mm; int ret; ttm_resource_manager_set_used(man, false); @@ -240,8 +299,10 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type) ttm_set_driver_manager(bdev, type, NULL); mutex_lock(&bman->lock); - i915_buddy_free_list(mm, &bman->reserved); - i915_buddy_fini(mm); + drm_buddy_free_list(mm, &bman->reserved); + drm_buddy_fini(mm); + bman->visible_avail += bman->visible_reserved; + WARN_ON_ONCE(bman->visible_avail != bman->visible_size); mutex_unlock(&bman->lock); ttm_resource_manager_cleanup(man); @@ -264,13 +325,50 @@ int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, u64 start, u64 size) { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); - struct i915_buddy_mm *mm = &bman->mm; + struct drm_buddy *mm = &bman->mm; + unsigned long fpfn = start >> PAGE_SHIFT; + unsigned long flags = 0; int ret; + flags |= DRM_BUDDY_RANGE_ALLOCATION; + mutex_lock(&bman->lock); - ret = i915_buddy_alloc_range(mm, &bman->reserved, start, size); + ret = drm_buddy_alloc_blocks(mm, start, + start + size, + size, mm->chunk_size, + &bman->reserved, + flags); + + if (fpfn < bman->visible_size) { + unsigned long lpfn = fpfn + (size >> PAGE_SHIFT); + unsigned long visible = min(lpfn, bman->visible_size) - fpfn; + + bman->visible_reserved += visible; + bman->visible_avail -= visible; + } mutex_unlock(&bman->lock); return ret; } +/** + * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion + * in pages. + * @man: The buddy allocator ttm manager + */ +u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man) +{ + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); + + return bman->visible_size; +} + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man, + u64 size) +{ + struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); + + bman->visible_size = size; +} +#endif diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h index 0722d33f3e14..52d9586d242c 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h @@ -13,13 +13,16 @@ struct ttm_device; struct ttm_resource_manager; -struct i915_buddy_mm; +struct drm_buddy; /** * struct i915_ttm_buddy_resource * * @base: struct ttm_resource base class we extend * @blocks: the list of struct i915_buddy_block for this resource/allocation + * @flags: DRM_BUDDY_*_ALLOCATION flags + * @used_visible_size: How much of this resource, if any, uses the CPU visible + * portion, in pages. * @mm: the struct i915_buddy_mm for this resource * * Extends the struct ttm_resource to manage an address space allocation with @@ -28,7 +31,9 @@ struct i915_buddy_mm; struct i915_ttm_buddy_resource { struct ttm_resource base; struct list_head blocks; - struct i915_buddy_mm *mm; + unsigned long flags; + unsigned long used_visible_size; + struct drm_buddy *mm; }; /** @@ -46,11 +51,19 @@ to_ttm_buddy_resource(struct ttm_resource *res) int i915_ttm_buddy_man_init(struct ttm_device *bdev, unsigned type, bool use_tt, - u64 size, u64 default_page_size, u64 chunk_size); + u64 size, u64 visible_size, + u64 default_page_size, u64 chunk_size); int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type); int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, u64 start, u64 size); +u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man); + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man, + u64 size); +#endif + #endif diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h index 7a5925072466..bfafd0afd117 100644 --- a/drivers/gpu/drm/i915/i915_utils.h +++ b/drivers/gpu/drm/i915/i915_utils.h @@ -37,21 +37,6 @@ struct timer_list; #define FDO_BUG_URL "https://gitlab.freedesktop.org/drm/intel/-/wikis/How-to-file-i915-bugs" -#undef WARN_ON -/* Many gcc seem to no see through this and fall over :( */ -#if 0 -#define WARN_ON(x) ({ \ - bool __i915_warn_cond = (x); \ - if (__builtin_constant_p(__i915_warn_cond)) \ - BUILD_BUG_ON(__i915_warn_cond); \ - WARN(__i915_warn_cond, "WARN_ON(" #x ")"); }) -#else -#define WARN_ON(x) WARN((x), "%s", "WARN_ON(" __stringify(x) ")") -#endif - -#undef WARN_ON_ONCE -#define WARN_ON_ONCE(x) WARN_ONCE((x), "%s", "WARN_ON_ONCE(" __stringify(x) ")") - #define MISSING_CASE(x) WARN(1, "Missing case (%s == %ld)\n", \ __stringify(x), (long)(x)) diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c index 31a105bc1792..c97323973f9b 100644 --- a/drivers/gpu/drm/i915/i915_vgpu.c +++ b/drivers/gpu/drm/i915/i915_vgpu.c @@ -197,7 +197,7 @@ static int vgt_balloon_space(struct i915_ggtt *ggtt, drm_info(&dev_priv->drm, "balloon space: range [ 0x%lx - 0x%lx ] %lu KiB.\n", start, end, size / 1024); - ret = i915_gem_gtt_reserve(&ggtt->vm, node, + ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, node, size, start, I915_COLOR_UNEVICTABLE, 0); if (!ret) diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index c0d6d5526abe..94fcdb7bd21d 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -26,17 +26,30 @@ #include <drm/drm_gem.h> #include "display/intel_frontbuffer.h" - #include "gem/i915_gem_lmem.h" +#include "gem/i915_gem_tiling.h" #include "gt/intel_engine.h" #include "gt/intel_engine_heartbeat.h" #include "gt/intel_gt.h" #include "gt/intel_gt_requests.h" #include "i915_drv.h" +#include "i915_gem_evict.h" #include "i915_sw_fence_work.h" #include "i915_trace.h" #include "i915_vma.h" +#include "i915_vma_resource.h" + +static inline void assert_vma_held_evict(const struct i915_vma *vma) +{ + /* + * We may be forced to unbind when the vm is dead, to clean it up. + * This is the only exception to the requirement of the object lock + * being held. + */ + if (atomic_read(&vma->vm->open)) + assert_object_held_shared(vma->obj); +} static struct kmem_cache *slab_vmas; @@ -284,7 +297,7 @@ struct i915_vma_work { struct dma_fence_work base; struct i915_address_space *vm; struct i915_vm_pt_stash stash; - struct i915_vma *vma; + struct i915_vma_resource *vma_res; struct drm_i915_gem_object *pinned; struct i915_sw_dma_fence_cb cb; enum i915_cache_level cache_level; @@ -294,23 +307,24 @@ struct i915_vma_work { static void __vma_bind(struct dma_fence_work *work) { struct i915_vma_work *vw = container_of(work, typeof(*vw), base); - struct i915_vma *vma = vw->vma; + struct i915_vma_resource *vma_res = vw->vma_res; + + vma_res->ops->bind_vma(vma_res->vm, &vw->stash, + vma_res, vw->cache_level, vw->flags); - vma->ops->bind_vma(vw->vm, &vw->stash, - vma, vw->cache_level, vw->flags); } static void __vma_release(struct dma_fence_work *work) { struct i915_vma_work *vw = container_of(work, typeof(*vw), base); - if (vw->pinned) { - __i915_gem_object_unpin_pages(vw->pinned); + if (vw->pinned) i915_gem_object_put(vw->pinned); - } i915_vm_free_pt_stash(vw->vm, &vw->stash); i915_vm_put(vw->vm); + if (vw->vma_res) + i915_vma_resource_put(vw->vma_res); } static const struct dma_fence_work_ops bind_ops = { @@ -374,12 +388,27 @@ static int i915_vma_verify_bind_complete(struct i915_vma *vma) #define i915_vma_verify_bind_complete(_vma) 0 #endif +I915_SELFTEST_EXPORT void +i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res, + struct i915_vma *vma) +{ + struct drm_i915_gem_object *obj = vma->obj; + + i915_vma_resource_init(vma_res, vma->vm, vma->pages, &vma->page_sizes, + obj->mm.rsgt, i915_gem_object_is_readonly(obj), + i915_gem_object_is_lmem(obj), obj->mm.region, + vma->ops, vma->private, vma->node.start, + vma->node.size, vma->size); +} + /** * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. * @vma: VMA to map * @cache_level: mapping cache level * @flags: flags like global or local mapping * @work: preallocated worker for allocating and binding the PTE + * @vma_res: pointer to a preallocated vma resource. The resource is either + * consumed or freed. * * DMA addresses are taken from the scatter-gather table of this object (or of * this VMA in case of non-default GGTT views) and PTE entries set up. @@ -388,10 +417,12 @@ static int i915_vma_verify_bind_complete(struct i915_vma *vma) int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags, - struct i915_vma_work *work) + struct i915_vma_work *work, + struct i915_vma_resource *vma_res) { u32 bind_flags; u32 vma_flags; + int ret; lockdep_assert_held(&vma->vm->mutex); GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); @@ -399,11 +430,15 @@ int i915_vma_bind(struct i915_vma *vma, if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start, vma->node.size, - vma->vm->total))) + vma->vm->total))) { + i915_vma_resource_free(vma_res); return -ENODEV; + } - if (GEM_DEBUG_WARN_ON(!flags)) + if (GEM_DEBUG_WARN_ON(!flags)) { + i915_vma_resource_free(vma_res); return -EINVAL; + } bind_flags = flags; bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; @@ -412,16 +447,44 @@ int i915_vma_bind(struct i915_vma *vma, vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; bind_flags &= ~vma_flags; - if (bind_flags == 0) + if (bind_flags == 0) { + i915_vma_resource_free(vma_res); return 0; + } GEM_BUG_ON(!atomic_read(&vma->pages_count)); + /* Wait for or await async unbinds touching our range */ + if (work && bind_flags & vma->vm->bind_async_flags) + ret = i915_vma_resource_bind_dep_await(vma->vm, + &work->base.chain, + vma->node.start, + vma->node.size, + true, + GFP_NOWAIT | + __GFP_RETRY_MAYFAIL | + __GFP_NOWARN); + else + ret = i915_vma_resource_bind_dep_sync(vma->vm, vma->node.start, + vma->node.size, true); + if (ret) { + i915_vma_resource_free(vma_res); + return ret; + } + + if (vma->resource || !vma_res) { + /* Rebinding with an additional I915_VMA_*_BIND */ + GEM_WARN_ON(!vma_flags); + i915_vma_resource_free(vma_res); + } else { + i915_vma_resource_init_from_vma(vma_res, vma); + vma->resource = vma_res; + } trace_i915_vma_bind(vma, bind_flags); if (work && bind_flags & vma->vm->bind_async_flags) { struct dma_fence *prev; - work->vma = vma; + work->vma_res = i915_vma_resource_get(vma->resource); work->cache_level = cache_level; work->flags = bind_flags; @@ -444,17 +507,25 @@ int i915_vma_bind(struct i915_vma *vma, work->base.dma.error = 0; /* enable the queue_work() */ - __i915_gem_object_pin_pages(vma->obj); - work->pinned = i915_gem_object_get(vma->obj); + /* + * If we don't have the refcounted pages list, keep a reference + * on the object to avoid waiting for the async bind to + * complete in the object destruction path. + */ + if (!work->vma_res->bi.pages_rsgt) + work->pinned = i915_gem_object_get(vma->obj); } else { if (vma->obj) { - int ret; - ret = i915_gem_object_wait_moving_fence(vma->obj, true); - if (ret) + if (ret) { + i915_vma_resource_free(vma->resource); + vma->resource = NULL; + return ret; + } } - vma->ops->bind_vma(vma->vm, NULL, vma, cache_level, bind_flags); + vma->ops->bind_vma(vma->vm, NULL, vma->resource, cache_level, + bind_flags); } if (vma->obj) @@ -469,6 +540,9 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) void __iomem *ptr; int err; + if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY)) + return IO_ERR_PTR(-EINVAL); + if (!i915_gem_object_is_lmem(vma->obj)) { if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) { err = -ENODEV; @@ -654,7 +728,8 @@ bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color) * 0 on success, negative error code otherwise. */ static int -i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) +i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, + u64 size, u64 alignment, u64 flags) { unsigned long color; u64 start, end; @@ -685,6 +760,14 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE); GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); + alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj)); + /* + * for compact-pt we round up the reservation to prevent + * any smaller pages being used within the same PDE + */ + if (NEEDS_COMPACT_PT(vma->vm->i915)) + size = round_up(size, alignment); + /* If binding the object/GGTT view requires more space than the entire * aperture has, reject it early before evicting everything in a vain * attempt to find space. @@ -697,6 +780,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) } color = 0; + if (i915_vm_has_cache_coloring(vma->vm)) color = vma->obj->cache_level; @@ -706,7 +790,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) range_overflows(offset, size, end)) return -EINVAL; - ret = i915_gem_gtt_reserve(vma->vm, &vma->node, + ret = i915_gem_gtt_reserve(vma->vm, ww, &vma->node, size, offset, color, flags); if (ret) @@ -745,7 +829,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) size = round_up(size, I915_GTT_PAGE_SIZE_2M); } - ret = i915_gem_gtt_insert(vma->vm, &vma->node, + ret = i915_gem_gtt_insert(vma->vm, ww, &vma->node, size, alignment, color, start, end, flags); if (ret) @@ -779,9 +863,17 @@ i915_vma_detach(struct i915_vma *vma) static bool try_qad_pin(struct i915_vma *vma, unsigned int flags) { unsigned int bound; - bool pinned = true; bound = atomic_read(&vma->flags); + + if (flags & PIN_VALIDATE) { + flags &= I915_VMA_BIND_MASK; + + return (flags & bound) == flags; + } + + /* with the lock mandatory for unbind, we don't race here */ + flags &= I915_VMA_BIND_MASK; do { if (unlikely(flags & ~bound)) return false; @@ -789,34 +881,10 @@ static bool try_qad_pin(struct i915_vma *vma, unsigned int flags) if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))) return false; - if (!(bound & I915_VMA_PIN_MASK)) - goto unpinned; - GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0); } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1)); return true; - -unpinned: - /* - * If pin_count==0, but we are bound, check under the lock to avoid - * racing with a concurrent i915_vma_unbind(). - */ - mutex_lock(&vma->vm->mutex); - do { - if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))) { - pinned = false; - break; - } - - if (unlikely(flags & ~bound)) { - pinned = false; - break; - } - } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1)); - mutex_unlock(&vma->vm->mutex); - - return pinned; } static struct scatterlist * @@ -912,30 +980,39 @@ err_st_alloc: } static struct scatterlist * -remap_pages(struct drm_i915_gem_object *obj, - unsigned int offset, unsigned int alignment_pad, - unsigned int width, unsigned int height, - unsigned int src_stride, unsigned int dst_stride, - struct sg_table *st, struct scatterlist *sg) +add_padding_pages(unsigned int count, + struct sg_table *st, struct scatterlist *sg) +{ + st->nents++; + + /* + * The DE ignores the PTEs for the padding tiles, the sg entry + * here is just a convenience to indicate how many padding PTEs + * to insert at this spot. + */ + sg_set_page(sg, NULL, count * I915_GTT_PAGE_SIZE, 0); + sg_dma_address(sg) = 0; + sg_dma_len(sg) = count * I915_GTT_PAGE_SIZE; + sg = sg_next(sg); + + return sg; +} + +static struct scatterlist * +remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj, + unsigned int offset, unsigned int alignment_pad, + unsigned int width, unsigned int height, + unsigned int src_stride, unsigned int dst_stride, + struct sg_table *st, struct scatterlist *sg, + unsigned int *gtt_offset) { unsigned int row; if (!width || !height) return sg; - if (alignment_pad) { - st->nents++; - - /* - * The DE ignores the PTEs for the padding tiles, the sg entry - * here is just a convenience to indicate how many padding PTEs - * to insert at this spot. - */ - sg_set_page(sg, NULL, alignment_pad * 4096, 0); - sg_dma_address(sg) = 0; - sg_dma_len(sg) = alignment_pad * 4096; - sg = sg_next(sg); - } + if (alignment_pad) + sg = add_padding_pages(alignment_pad, st, sg); for (row = 0; row < height; row++) { unsigned int left = width * I915_GTT_PAGE_SIZE; @@ -972,18 +1049,98 @@ remap_pages(struct drm_i915_gem_object *obj, if (!left) continue; + sg = add_padding_pages(left >> PAGE_SHIFT, st, sg); + } + + *gtt_offset += alignment_pad + dst_stride * height; + + return sg; +} + +static struct scatterlist * +remap_contiguous_pages(struct drm_i915_gem_object *obj, + unsigned int obj_offset, + unsigned int count, + struct sg_table *st, struct scatterlist *sg) +{ + struct scatterlist *iter; + unsigned int offset; + + iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset); + GEM_BUG_ON(!iter); + + do { + unsigned int len; + + len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT), + count << PAGE_SHIFT); + sg_set_page(sg, NULL, len, 0); + sg_dma_address(sg) = + sg_dma_address(iter) + (offset << PAGE_SHIFT); + sg_dma_len(sg) = len; + st->nents++; + count -= len >> PAGE_SHIFT; + if (count == 0) + return sg; - /* - * The DE ignores the PTEs for the padding tiles, the sg entry - * here is just a conenience to indicate how many padding PTEs - * to insert at this spot. - */ - sg_set_page(sg, NULL, left, 0); - sg_dma_address(sg) = 0; - sg_dma_len(sg) = left; - sg = sg_next(sg); - } + sg = __sg_next(sg); + iter = __sg_next(iter); + offset = 0; + } while (1); +} + +static struct scatterlist * +remap_linear_color_plane_pages(struct drm_i915_gem_object *obj, + unsigned int obj_offset, unsigned int alignment_pad, + unsigned int size, + struct sg_table *st, struct scatterlist *sg, + unsigned int *gtt_offset) +{ + if (!size) + return sg; + + if (alignment_pad) + sg = add_padding_pages(alignment_pad, st, sg); + + sg = remap_contiguous_pages(obj, obj_offset, size, st, sg); + sg = sg_next(sg); + + *gtt_offset += alignment_pad + size; + + return sg; +} + +static struct scatterlist * +remap_color_plane_pages(const struct intel_remapped_info *rem_info, + struct drm_i915_gem_object *obj, + int color_plane, + struct sg_table *st, struct scatterlist *sg, + unsigned int *gtt_offset) +{ + unsigned int alignment_pad = 0; + + if (rem_info->plane_alignment) + alignment_pad = ALIGN(*gtt_offset, rem_info->plane_alignment) - *gtt_offset; + + if (rem_info->plane[color_plane].linear) + sg = remap_linear_color_plane_pages(obj, + rem_info->plane[color_plane].offset, + alignment_pad, + rem_info->plane[color_plane].size, + st, sg, + gtt_offset); + + else + sg = remap_tiled_color_plane_pages(obj, + rem_info->plane[color_plane].offset, + alignment_pad, + rem_info->plane[color_plane].width, + rem_info->plane[color_plane].height, + rem_info->plane[color_plane].src_stride, + rem_info->plane[color_plane].dst_stride, + st, sg, + gtt_offset); return sg; } @@ -1012,21 +1169,8 @@ intel_remap_pages(struct intel_remapped_info *rem_info, st->nents = 0; sg = st->sgl; - for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) { - unsigned int alignment_pad = 0; - - if (rem_info->plane_alignment) - alignment_pad = ALIGN(gtt_offset, rem_info->plane_alignment) - gtt_offset; - - sg = remap_pages(obj, - rem_info->plane[i].offset, alignment_pad, - rem_info->plane[i].width, rem_info->plane[i].height, - rem_info->plane[i].src_stride, rem_info->plane[i].dst_stride, - st, sg); - - gtt_offset += alignment_pad + - rem_info->plane[i].dst_stride * rem_info->plane[i].height; - } + for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) + sg = remap_color_plane_pages(rem_info, obj, i, st, sg, >t_offset); i915_sg_trim(st); @@ -1048,9 +1192,8 @@ intel_partial_pages(const struct i915_ggtt_view *view, struct drm_i915_gem_object *obj) { struct sg_table *st; - struct scatterlist *sg, *iter; + struct scatterlist *sg; unsigned int count = view->partial.size; - unsigned int offset; int ret = -ENOMEM; st = kmalloc(sizeof(*st), GFP_KERNEL); @@ -1061,34 +1204,14 @@ intel_partial_pages(const struct i915_ggtt_view *view, if (ret) goto err_sg_alloc; - iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset); - GEM_BUG_ON(!iter); - - sg = st->sgl; st->nents = 0; - do { - unsigned int len; - len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT), - count << PAGE_SHIFT); - sg_set_page(sg, NULL, len, 0); - sg_dma_address(sg) = - sg_dma_address(iter) + (offset << PAGE_SHIFT); - sg_dma_len(sg) = len; + sg = remap_contiguous_pages(obj, view->partial.offset, count, st, st->sgl); - st->nents++; - count -= len >> PAGE_SHIFT; - if (count == 0) { - sg_mark_end(sg); - i915_sg_trim(st); /* Drop any unused tail entries. */ + sg_mark_end(sg); + i915_sg_trim(st); /* Drop any unused tail entries. */ - return st; - } - - sg = __sg_next(sg); - iter = __sg_next(iter); - offset = 0; - } while (1); + return st; err_sg_alloc: kfree(st); @@ -1100,7 +1223,6 @@ static int __i915_vma_get_pages(struct i915_vma *vma) { struct sg_table *pages; - int ret; /* * The vma->pages are only valid within the lifespan of the borrowed @@ -1133,18 +1255,16 @@ __i915_vma_get_pages(struct i915_vma *vma) break; } - ret = 0; if (IS_ERR(pages)) { - ret = PTR_ERR(pages); - pages = NULL; drm_err(&vma->vm->i915->drm, - "Failed to get pages for VMA view type %u (%d)!\n", - vma->ggtt_view.type, ret); + "Failed to get pages for VMA view type %u (%ld)!\n", + vma->ggtt_view.type, PTR_ERR(pages)); + return PTR_ERR(pages); } vma->pages = pages; - return ret; + return 0; } I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma) @@ -1176,25 +1296,14 @@ err_unpin: static void __vma_put_pages(struct i915_vma *vma, unsigned int count) { /* We allocate under vma_get_pages, so beware the shrinker */ - struct sg_table *pages = READ_ONCE(vma->pages); - GEM_BUG_ON(atomic_read(&vma->pages_count) < count); if (atomic_sub_return(count, &vma->pages_count) == 0) { - /* - * The atomic_sub_return is a read barrier for the READ_ONCE of - * vma->pages above. - * - * READ_ONCE is safe because this is either called from the same - * function (i915_vma_pin_ww), or guarded by vma->vm->mutex. - * - * TODO: We're leaving vma->pages dangling, until vma->obj->resv - * lock is required. - */ - if (pages != vma->obj->mm.pages) { - sg_free_table(pages); - kfree(pages); + if (vma->pages != vma->obj->mm.pages) { + sg_free_table(vma->pages); + kfree(vma->pages); } + vma->pages = NULL; i915_gem_object_unpin_pages(vma->obj); } @@ -1227,6 +1336,7 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, { struct i915_vma_work *work = NULL; struct dma_fence *moving = NULL; + struct i915_vma_resource *vma_res = NULL; intel_wakeref_t wakeref = 0; unsigned int bound; int err; @@ -1240,7 +1350,7 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL))); /* First try and grab the pin without rebinding the vma */ - if (try_qad_pin(vma, flags & I915_VMA_BIND_MASK)) + if (try_qad_pin(vma, flags)) return 0; err = i915_vma_get_pages(vma); @@ -1281,6 +1391,12 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, } } + vma_res = i915_vma_resource_alloc(); + if (IS_ERR(vma_res)) { + err = PTR_ERR(vma_res); + goto err_fence; + } + /* * Differentiate between user/kernel vma inside the aliasing-ppgtt. * @@ -1301,7 +1417,7 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, err = mutex_lock_interruptible_nested(&vma->vm->mutex, !(flags & PIN_GLOBAL)); if (err) - goto err_fence; + goto err_vma_res; /* No more allocations allowed now we hold vm->mutex */ @@ -1322,7 +1438,8 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, } if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) { - __i915_vma_pin(vma); + if (!(flags & PIN_VALIDATE)) + __i915_vma_pin(vma); goto err_unlock; } @@ -1331,7 +1448,7 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, goto err_unlock; if (!(bound & I915_VMA_BIND_MASK)) { - err = i915_vma_insert(vma, size, alignment, flags); + err = i915_vma_insert(vma, ww, size, alignment, flags); if (err) goto err_active; @@ -1342,7 +1459,8 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, GEM_BUG_ON(!vma->pages); err = i915_vma_bind(vma, vma->obj->cache_level, - flags, work); + flags, work, vma_res); + vma_res = NULL; if (err) goto err_remove; @@ -1351,8 +1469,10 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count); list_move_tail(&vma->vm_link, &vma->vm->bound_list); - __i915_vma_pin(vma); - GEM_BUG_ON(!i915_vma_is_pinned(vma)); + if (!(flags & PIN_VALIDATE)) { + __i915_vma_pin(vma); + GEM_BUG_ON(!i915_vma_is_pinned(vma)); + } GEM_BUG_ON(!i915_vma_is_bound(vma, flags)); GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); @@ -1365,6 +1485,8 @@ err_active: i915_active_release(&vma->active); err_unlock: mutex_unlock(&vma->vm->mutex); +err_vma_res: + i915_vma_resource_free(vma_res); err_fence: if (work) dma_fence_work_commit_imm(&work->base); @@ -1411,7 +1533,12 @@ static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, /* Unlike i915_vma_pin, we don't take no for an answer! */ flush_idle_contexts(vm->gt); if (mutex_lock_interruptible(&vm->mutex) == 0) { - i915_gem_evict_vm(vm); + /* + * We pass NULL ww here, as we don't want to unbind + * locked objects when called from execbuf when pinning + * is removed. This would probably regress badly. + */ + i915_gem_evict_vm(vm, NULL); mutex_unlock(&vm->mutex); } } while (1); @@ -1494,15 +1621,27 @@ void i915_vma_reopen(struct i915_vma *vma) void i915_vma_release(struct kref *ref) { struct i915_vma *vma = container_of(ref, typeof(*vma), ref); + + i915_vm_put(vma->vm); + i915_active_fini(&vma->active); + GEM_WARN_ON(vma->resource); + i915_vma_free(vma); +} + +static void force_unbind(struct i915_vma *vma) +{ + if (!drm_mm_node_allocated(&vma->node)) + return; + + atomic_and(~I915_VMA_PIN_MASK, &vma->flags); + WARN_ON(__i915_vma_unbind(vma)); + GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); +} + +static void release_references(struct i915_vma *vma) +{ struct drm_i915_gem_object *obj = vma->obj; - if (drm_mm_node_allocated(&vma->node)) { - mutex_lock(&vma->vm->mutex); - atomic_and(~I915_VMA_PIN_MASK, &vma->flags); - WARN_ON(__i915_vma_unbind(vma)); - mutex_unlock(&vma->vm->mutex); - GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); - } GEM_BUG_ON(i915_vma_is_active(vma)); spin_lock(&obj->vma.lock); @@ -1512,10 +1651,49 @@ void i915_vma_release(struct kref *ref) spin_unlock(&obj->vma.lock); __i915_vma_remove_closed(vma); - i915_vm_put(vma->vm); - i915_active_fini(&vma->active); - i915_vma_free(vma); + __i915_vma_put(vma); +} + +/** + * i915_vma_destroy_locked - Remove all weak reference to the vma and put + * the initial reference. + * + * This function should be called when it's decided the vma isn't needed + * anymore. The caller must assure that it doesn't race with another lookup + * plus destroy, typically by taking an appropriate reference. + * + * Current callsites are + * - __i915_gem_object_pages_fini() + * - __i915_vm_close() - Blocks the above function by taking a reference on + * the object. + * - __i915_vma_parked() - Blocks the above functions by taking an open-count on + * the vm and a reference on the object. + * + * Because of locks taken during destruction, a vma is also guaranteed to + * stay alive while the following locks are held if it was looked up while + * holding one of the locks: + * - vm->mutex + * - obj->vma.lock + * - gt->closed_lock + * + * A vma user can also temporarily keep the vma alive while holding a vma + * reference. + */ +void i915_vma_destroy_locked(struct i915_vma *vma) +{ + lockdep_assert_held(&vma->vm->mutex); + + force_unbind(vma); + release_references(vma); +} + +void i915_vma_destroy(struct i915_vma *vma) +{ + mutex_lock(&vma->vm->mutex); + force_unbind(vma); + mutex_unlock(&vma->vm->mutex); + release_references(vma); } void i915_vma_parked(struct intel_gt *gt) @@ -1547,8 +1725,16 @@ void i915_vma_parked(struct intel_gt *gt) struct drm_i915_gem_object *obj = vma->obj; struct i915_address_space *vm = vma->vm; - INIT_LIST_HEAD(&vma->closed_link); - __i915_vma_put(vma); + if (i915_gem_object_trylock(obj, NULL)) { + INIT_LIST_HEAD(&vma->closed_link); + i915_vma_destroy(vma); + i915_gem_object_unlock(obj); + } else { + /* back you go.. */ + spin_lock_irq(>->closed_lock); + list_add(&vma->closed_link, >->closed_vma); + spin_unlock_irq(>->closed_lock); + } i915_gem_object_put(obj); i915_vm_close(vm); @@ -1599,8 +1785,6 @@ static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request * { int err; - GEM_BUG_ON(!i915_vma_is_pinned(vma)); - /* Wait for the vma to be bound before we start! */ err = __i915_request_await_bind(rq, vma); if (err) @@ -1619,6 +1803,8 @@ int _i915_vma_move_to_active(struct i915_vma *vma, assert_object_held(obj); + GEM_BUG_ON(!vma->pages); + err = __i915_vma_move_to_active(vma, rq); if (unlikely(err)) return err; @@ -1661,9 +1847,13 @@ int _i915_vma_move_to_active(struct i915_vma *vma, return 0; } -void __i915_vma_evict(struct i915_vma *vma) +struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async) { + struct i915_vma_resource *vma_res = vma->resource; + struct dma_fence *unbind_fence; + GEM_BUG_ON(i915_vma_is_pinned(vma)); + assert_vma_held_evict(vma); if (i915_vma_is_map_and_fenceable(vma)) { /* Force a pagefault for domain tracking on next user access */ @@ -1693,15 +1883,36 @@ void __i915_vma_evict(struct i915_vma *vma) GEM_BUG_ON(vma->fence); GEM_BUG_ON(i915_vma_has_userfault(vma)); - if (likely(atomic_read(&vma->vm->open))) { - trace_i915_vma_unbind(vma); - vma->ops->unbind_vma(vma->vm, vma); - } + /* Object backend must be async capable. */ + GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt); + + /* If vm is not open, unbind is a nop. */ + vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) && + atomic_read(&vma->vm->open); + trace_i915_vma_unbind(vma); + + unbind_fence = i915_vma_resource_unbind(vma_res); + vma->resource = NULL; + atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE), &vma->flags); i915_vma_detach(vma); + + if (!async && unbind_fence) { + dma_fence_wait(unbind_fence, false); + dma_fence_put(unbind_fence); + unbind_fence = NULL; + } + + /* + * Binding itself may not have completed until the unbind fence signals, + * so don't drop the pages until that happens, unless the resource is + * async_capable. + */ + vma_unbind_pages(vma); + return unbind_fence; } int __i915_vma_unbind(struct i915_vma *vma) @@ -1709,6 +1920,7 @@ int __i915_vma_unbind(struct i915_vma *vma) int ret; lockdep_assert_held(&vma->vm->mutex); + assert_vma_held_evict(vma); if (!drm_mm_node_allocated(&vma->node)) return 0; @@ -1728,18 +1940,55 @@ int __i915_vma_unbind(struct i915_vma *vma) return ret; GEM_BUG_ON(i915_vma_is_active(vma)); - __i915_vma_evict(vma); + __i915_vma_evict(vma, false); drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */ return 0; } +static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma) +{ + struct dma_fence *fence; + + lockdep_assert_held(&vma->vm->mutex); + + if (!drm_mm_node_allocated(&vma->node)) + return NULL; + + if (i915_vma_is_pinned(vma) || + &vma->obj->mm.rsgt->table != vma->resource->bi.pages) + return ERR_PTR(-EAGAIN); + + /* + * We probably need to replace this with awaiting the fences of the + * object's dma_resv when the vma active goes away. When doing that + * we need to be careful to not add the vma_resource unbind fence + * immediately to the object's dma_resv, because then unbinding + * the next vma from the object, in case there are many, will + * actually await the unbinding of the previous vmas, which is + * undesirable. + */ + if (i915_sw_fence_await_active(&vma->resource->chain, &vma->active, + I915_ACTIVE_AWAIT_EXCL | + I915_ACTIVE_AWAIT_ACTIVE) < 0) { + return ERR_PTR(-EBUSY); + } + + fence = __i915_vma_evict(vma, true); + + drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */ + + return fence; +} + int i915_vma_unbind(struct i915_vma *vma) { struct i915_address_space *vm = vma->vm; intel_wakeref_t wakeref = 0; int err; + assert_object_held_shared(vma->obj); + /* Optimistic wait before taking the mutex */ err = i915_vma_sync(vma); if (err) @@ -1770,6 +2019,79 @@ out_rpm: return err; } +int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm) +{ + struct drm_i915_gem_object *obj = vma->obj; + struct i915_address_space *vm = vma->vm; + intel_wakeref_t wakeref = 0; + struct dma_fence *fence; + int err; + + /* + * We need the dma-resv lock since we add the + * unbind fence to the dma-resv object. + */ + assert_object_held(obj); + + if (!drm_mm_node_allocated(&vma->node)) + return 0; + + if (i915_vma_is_pinned(vma)) { + vma_print_allocator(vma, "is pinned"); + return -EAGAIN; + } + + if (!obj->mm.rsgt) + return -EBUSY; + + err = dma_resv_reserve_shared(obj->base.resv, 1); + if (err) + return -EBUSY; + + /* + * It would be great if we could grab this wakeref from the + * async unbind work if needed, but we can't because it uses + * kmalloc and it's in the dma-fence signalling critical path. + */ + if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) + wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); + + if (trylock_vm && !mutex_trylock(&vm->mutex)) { + err = -EBUSY; + goto out_rpm; + } else if (!trylock_vm) { + err = mutex_lock_interruptible_nested(&vm->mutex, !wakeref); + if (err) + goto out_rpm; + } + + fence = __i915_vma_unbind_async(vma); + mutex_unlock(&vm->mutex); + if (IS_ERR_OR_NULL(fence)) { + err = PTR_ERR_OR_ZERO(fence); + goto out_rpm; + } + + dma_resv_add_shared_fence(obj->base.resv, fence); + dma_fence_put(fence); + +out_rpm: + if (wakeref) + intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref); + return err; +} + +int i915_vma_unbind_unlocked(struct i915_vma *vma) +{ + int err; + + i915_gem_object_lock(vma->obj, NULL); + err = i915_vma_unbind(vma); + i915_gem_object_unlock(vma->obj); + + return err; +} + struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma) { i915_gem_object_make_unshrinkable(vma->obj); diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h index 32719431b3df..67ae7341c7e0 100644 --- a/drivers/gpu/drm/i915/i915_vma.h +++ b/drivers/gpu/drm/i915/i915_vma.h @@ -37,6 +37,7 @@ #include "i915_active.h" #include "i915_request.h" +#include "i915_vma_resource.h" #include "i915_vma_types.h" struct i915_vma * @@ -204,16 +205,19 @@ struct i915_vma_work *i915_vma_work(void); int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags, - struct i915_vma_work *work); + struct i915_vma_work *work, + struct i915_vma_resource *vma_res); bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color); bool i915_vma_misplaced(const struct i915_vma *vma, u64 size, u64 alignment, u64 flags); void __i915_vma_set_map_and_fenceable(struct i915_vma *vma); void i915_vma_revoke_mmap(struct i915_vma *vma); -void __i915_vma_evict(struct i915_vma *vma); +struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async); int __i915_vma_unbind(struct i915_vma *vma); int __must_check i915_vma_unbind(struct i915_vma *vma); +int __must_check i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm); +int __must_check i915_vma_unbind_unlocked(struct i915_vma *vma); void i915_vma_unlink_ctx(struct i915_vma *vma); void i915_vma_close(struct i915_vma *vma); void i915_vma_reopen(struct i915_vma *vma); @@ -232,6 +236,9 @@ static inline void __i915_vma_put(struct i915_vma *vma) kref_put(&vma->ref, i915_vma_release); } +void i915_vma_destroy_locked(struct i915_vma *vma); +void i915_vma_destroy(struct i915_vma *vma); + #define assert_vma_held(vma) dma_resv_assert_held((vma)->obj->base.resv) static inline void i915_vma_lock(struct i915_vma *vma) @@ -337,12 +344,6 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); */ void i915_vma_unpin_iomap(struct i915_vma *vma); -static inline struct page *i915_vma_first_page(struct i915_vma *vma) -{ - GEM_BUG_ON(!vma->pages); - return sg_page(vma->pages->sgl); -} - /** * i915_vma_pin_fence - pin fencing state * @vma: vma to pin fencing for @@ -428,6 +429,26 @@ static inline int i915_vma_sync(struct i915_vma *vma) return i915_active_wait(&vma->active); } +/** + * i915_vma_get_current_resource - Get the current resource of the vma + * @vma: The vma to get the current resource from. + * + * It's illegal to call this function if the vma is not bound. + * + * Return: A refcounted pointer to the current vma resource + * of the vma, assuming the vma is bound. + */ +static inline struct i915_vma_resource * +i915_vma_get_current_resource(struct i915_vma *vma) +{ + return i915_vma_resource_get(vma->resource); +} + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +void i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res, + struct i915_vma *vma); +#endif + void i915_vma_module_exit(void); int i915_vma_module_init(void); diff --git a/drivers/gpu/drm/i915/i915_vma_resource.c b/drivers/gpu/drm/i915/i915_vma_resource.c new file mode 100644 index 000000000000..57ae92ba8af1 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_vma_resource.c @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include <linux/interval_tree_generic.h> +#include <linux/sched/mm.h> + +#include "i915_sw_fence.h" +#include "i915_vma_resource.h" +#include "i915_drv.h" +#include "intel_memory_region.h" + +#include "gt/intel_gtt.h" + +static struct kmem_cache *slab_vma_resources; + +/** + * DOC: + * We use a per-vm interval tree to keep track of vma_resources + * scheduled for unbind but not yet unbound. The tree is protected by + * the vm mutex, and nodes are removed just after the unbind fence signals. + * The removal takes the vm mutex from a kernel thread which we need to + * keep in mind so that we don't grab the mutex and try to wait for all + * pending unbinds to complete, because that will temporaryily block many + * of the workqueue threads, and people will get angry. + * + * We should consider using a single ordered fence per VM instead but that + * requires ordering the unbinds and might introduce unnecessary waiting + * for unrelated unbinds. Amount of code will probably be roughly the same + * due to the simplicity of using the interval tree interface. + * + * Another drawback of this interval tree is that the complexity of insertion + * and removal of fences increases as O(ln(pending_unbinds)) instead of + * O(1) for a single fence without interval tree. + */ +#define VMA_RES_START(_node) ((_node)->start) +#define VMA_RES_LAST(_node) ((_node)->start + (_node)->node_size - 1) +INTERVAL_TREE_DEFINE(struct i915_vma_resource, rb, + u64, __subtree_last, + VMA_RES_START, VMA_RES_LAST, static, vma_res_itree); + +/* Callbacks for the unbind dma-fence. */ + +/** + * i915_vma_resource_alloc - Allocate a vma resource + * + * Return: A pointer to a cleared struct i915_vma_resource or + * a -ENOMEM error pointer if allocation fails. + */ +struct i915_vma_resource *i915_vma_resource_alloc(void) +{ + struct i915_vma_resource *vma_res = + kmem_cache_zalloc(slab_vma_resources, GFP_KERNEL); + + return vma_res ? vma_res : ERR_PTR(-ENOMEM); +} + +/** + * i915_vma_resource_free - Free a vma resource + * @vma_res: The vma resource to free. + */ +void i915_vma_resource_free(struct i915_vma_resource *vma_res) +{ + if (vma_res) + kmem_cache_free(slab_vma_resources, vma_res); +} + +static const char *get_driver_name(struct dma_fence *fence) +{ + return "vma unbind fence"; +} + +static const char *get_timeline_name(struct dma_fence *fence) +{ + return "unbound"; +} + +static void unbind_fence_free_rcu(struct rcu_head *head) +{ + struct i915_vma_resource *vma_res = + container_of(head, typeof(*vma_res), unbind_fence.rcu); + + i915_vma_resource_free(vma_res); +} + +static void unbind_fence_release(struct dma_fence *fence) +{ + struct i915_vma_resource *vma_res = + container_of(fence, typeof(*vma_res), unbind_fence); + + i915_sw_fence_fini(&vma_res->chain); + + call_rcu(&fence->rcu, unbind_fence_free_rcu); +} + +static struct dma_fence_ops unbind_fence_ops = { + .get_driver_name = get_driver_name, + .get_timeline_name = get_timeline_name, + .release = unbind_fence_release, +}; + +static void __i915_vma_resource_unhold(struct i915_vma_resource *vma_res) +{ + struct i915_address_space *vm; + + if (!refcount_dec_and_test(&vma_res->hold_count)) + return; + + dma_fence_signal(&vma_res->unbind_fence); + + vm = vma_res->vm; + if (vma_res->wakeref) + intel_runtime_pm_put(&vm->i915->runtime_pm, vma_res->wakeref); + + vma_res->vm = NULL; + if (!RB_EMPTY_NODE(&vma_res->rb)) { + mutex_lock(&vm->mutex); + vma_res_itree_remove(vma_res, &vm->pending_unbind); + mutex_unlock(&vm->mutex); + } + + if (vma_res->bi.pages_rsgt) + i915_refct_sgt_put(vma_res->bi.pages_rsgt); +} + +/** + * i915_vma_resource_unhold - Unhold the signaling of the vma resource unbind + * fence. + * @vma_res: The vma resource. + * @lockdep_cookie: The lockdep cookie returned from i915_vma_resource_hold. + * + * The function may leave a dma_fence critical section. + */ +void i915_vma_resource_unhold(struct i915_vma_resource *vma_res, + bool lockdep_cookie) +{ + dma_fence_end_signalling(lockdep_cookie); + + if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { + unsigned long irq_flags; + + /* Inefficient open-coded might_lock_irqsave() */ + spin_lock_irqsave(&vma_res->lock, irq_flags); + spin_unlock_irqrestore(&vma_res->lock, irq_flags); + } + + __i915_vma_resource_unhold(vma_res); +} + +/** + * i915_vma_resource_hold - Hold the signaling of the vma resource unbind fence. + * @vma_res: The vma resource. + * @lockdep_cookie: Pointer to a bool serving as a lockdep cooke that should + * be given as an argument to the pairing i915_vma_resource_unhold. + * + * If returning true, the function enters a dma_fence signalling critical + * section if not in one already. + * + * Return: true if holding successful, false if not. + */ +bool i915_vma_resource_hold(struct i915_vma_resource *vma_res, + bool *lockdep_cookie) +{ + bool held = refcount_inc_not_zero(&vma_res->hold_count); + + if (held) + *lockdep_cookie = dma_fence_begin_signalling(); + + return held; +} + +static void i915_vma_resource_unbind_work(struct work_struct *work) +{ + struct i915_vma_resource *vma_res = + container_of(work, typeof(*vma_res), work); + struct i915_address_space *vm = vma_res->vm; + bool lockdep_cookie; + + lockdep_cookie = dma_fence_begin_signalling(); + if (likely(atomic_read(&vm->open))) + vma_res->ops->unbind_vma(vm, vma_res); + + dma_fence_end_signalling(lockdep_cookie); + __i915_vma_resource_unhold(vma_res); + i915_vma_resource_put(vma_res); +} + +static int +i915_vma_resource_fence_notify(struct i915_sw_fence *fence, + enum i915_sw_fence_notify state) +{ + struct i915_vma_resource *vma_res = + container_of(fence, typeof(*vma_res), chain); + struct dma_fence *unbind_fence = + &vma_res->unbind_fence; + + switch (state) { + case FENCE_COMPLETE: + dma_fence_get(unbind_fence); + if (vma_res->immediate_unbind) { + i915_vma_resource_unbind_work(&vma_res->work); + } else { + INIT_WORK(&vma_res->work, i915_vma_resource_unbind_work); + queue_work(system_unbound_wq, &vma_res->work); + } + break; + case FENCE_FREE: + i915_vma_resource_put(vma_res); + break; + } + + return NOTIFY_DONE; +} + +/** + * i915_vma_resource_unbind - Unbind a vma resource + * @vma_res: The vma resource to unbind. + * + * At this point this function does little more than publish a fence that + * signals immediately unless signaling is held back. + * + * Return: A refcounted pointer to a dma-fence that signals when unbinding is + * complete. + */ +struct dma_fence *i915_vma_resource_unbind(struct i915_vma_resource *vma_res) +{ + struct i915_address_space *vm = vma_res->vm; + + /* Reference for the sw fence */ + i915_vma_resource_get(vma_res); + + /* Caller must already have a wakeref in this case. */ + if (vma_res->needs_wakeref) + vma_res->wakeref = intel_runtime_pm_get_if_in_use(&vm->i915->runtime_pm); + + if (atomic_read(&vma_res->chain.pending) <= 1) { + RB_CLEAR_NODE(&vma_res->rb); + vma_res->immediate_unbind = 1; + } else { + vma_res_itree_insert(vma_res, &vma_res->vm->pending_unbind); + } + + i915_sw_fence_commit(&vma_res->chain); + + return &vma_res->unbind_fence; +} + +/** + * __i915_vma_resource_init - Initialize a vma resource. + * @vma_res: The vma resource to initialize + * + * Initializes the private members of a vma resource. + */ +void __i915_vma_resource_init(struct i915_vma_resource *vma_res) +{ + spin_lock_init(&vma_res->lock); + dma_fence_init(&vma_res->unbind_fence, &unbind_fence_ops, + &vma_res->lock, 0, 0); + refcount_set(&vma_res->hold_count, 1); + i915_sw_fence_init(&vma_res->chain, i915_vma_resource_fence_notify); +} + +static void +i915_vma_resource_color_adjust_range(struct i915_address_space *vm, + u64 *start, + u64 *end) +{ + if (i915_vm_has_cache_coloring(vm)) { + if (*start) + *start -= I915_GTT_PAGE_SIZE; + *end += I915_GTT_PAGE_SIZE; + } +} + +/** + * i915_vma_resource_bind_dep_sync - Wait for / sync all unbinds touching a + * certain vm range. + * @vm: The vm to look at. + * @offset: The range start. + * @size: The range size. + * @intr: Whether to wait interrubtible. + * + * The function needs to be called with the vm lock held. + * + * Return: Zero on success, -ERESTARTSYS if interrupted and @intr==true + */ +int i915_vma_resource_bind_dep_sync(struct i915_address_space *vm, + u64 offset, + u64 size, + bool intr) +{ + struct i915_vma_resource *node; + u64 last = offset + size - 1; + + lockdep_assert_held(&vm->mutex); + might_sleep(); + + i915_vma_resource_color_adjust_range(vm, &offset, &last); + node = vma_res_itree_iter_first(&vm->pending_unbind, offset, last); + while (node) { + int ret = dma_fence_wait(&node->unbind_fence, intr); + + if (ret) + return ret; + + node = vma_res_itree_iter_next(node, offset, last); + } + + return 0; +} + +/** + * i915_vma_resource_bind_dep_sync_all - Wait for / sync all unbinds of a vm, + * releasing the vm lock while waiting. + * @vm: The vm to look at. + * + * The function may not be called with the vm lock held. + * Typically this is called at vm destruction to finish any pending + * unbind operations. The vm mutex is released while waiting to avoid + * stalling kernel workqueues trying to grab the mutex. + */ +void i915_vma_resource_bind_dep_sync_all(struct i915_address_space *vm) +{ + struct i915_vma_resource *node; + struct dma_fence *fence; + + do { + fence = NULL; + mutex_lock(&vm->mutex); + node = vma_res_itree_iter_first(&vm->pending_unbind, 0, + U64_MAX); + if (node) + fence = dma_fence_get_rcu(&node->unbind_fence); + mutex_unlock(&vm->mutex); + + if (fence) { + /* + * The wait makes sure the node eventually removes + * itself from the tree. + */ + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + } while (node); +} + +/** + * i915_vma_resource_bind_dep_await - Have a struct i915_sw_fence await all + * pending unbinds in a certain range of a vm. + * @vm: The vm to look at. + * @sw_fence: The struct i915_sw_fence that will be awaiting the unbinds. + * @offset: The range start. + * @size: The range size. + * @intr: Whether to wait interrubtible. + * @gfp: Allocation mode for memory allocations. + * + * The function makes @sw_fence await all pending unbinds in a certain + * vm range before calling the complete notifier. To be able to await + * each individual unbind, the function needs to allocate memory using + * the @gpf allocation mode. If that fails, the function will instead + * wait for the unbind fence to signal, using @intr to judge whether to + * wait interruptible or not. Note that @gfp should ideally be selected so + * as to avoid any expensive memory allocation stalls and rather fail and + * synchronize itself. For now the vm mutex is required when calling this + * function with means that @gfp can't call into direct reclaim. In reality + * this means that during heavy memory pressure, we will sync in this + * function. + * + * Return: Zero on success, -ERESTARTSYS if interrupted and @intr==true + */ +int i915_vma_resource_bind_dep_await(struct i915_address_space *vm, + struct i915_sw_fence *sw_fence, + u64 offset, + u64 size, + bool intr, + gfp_t gfp) +{ + struct i915_vma_resource *node; + u64 last = offset + size - 1; + + lockdep_assert_held(&vm->mutex); + might_alloc(gfp); + might_sleep(); + + i915_vma_resource_color_adjust_range(vm, &offset, &last); + node = vma_res_itree_iter_first(&vm->pending_unbind, offset, last); + while (node) { + int ret; + + ret = i915_sw_fence_await_dma_fence(sw_fence, + &node->unbind_fence, + 0, gfp); + if (ret < 0) { + ret = dma_fence_wait(&node->unbind_fence, intr); + if (ret) + return ret; + } + + node = vma_res_itree_iter_next(node, offset, last); + } + + return 0; +} + +void i915_vma_resource_module_exit(void) +{ + kmem_cache_destroy(slab_vma_resources); +} + +int __init i915_vma_resource_module_init(void) +{ + slab_vma_resources = KMEM_CACHE(i915_vma_resource, SLAB_HWCACHE_ALIGN); + if (!slab_vma_resources) + return -ENOMEM; + + return 0; +} diff --git a/drivers/gpu/drm/i915/i915_vma_resource.h b/drivers/gpu/drm/i915/i915_vma_resource.h new file mode 100644 index 000000000000..25913913baa6 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_vma_resource.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __I915_VMA_RESOURCE_H__ +#define __I915_VMA_RESOURCE_H__ + +#include <linux/dma-fence.h> +#include <linux/refcount.h> + +#include "i915_gem.h" +#include "i915_scatterlist.h" +#include "i915_sw_fence.h" +#include "intel_runtime_pm.h" + +struct intel_memory_region; + +struct i915_page_sizes { + /** + * The sg mask of the pages sg_table. i.e the mask of + * the lengths for each sg entry. + */ + unsigned int phys; + + /** + * The gtt page sizes we are allowed to use given the + * sg mask and the supported page sizes. This will + * express the smallest unit we can use for the whole + * object, as well as the larger sizes we may be able + * to use opportunistically. + */ + unsigned int sg; +}; + +/** + * struct i915_vma_resource - Snapshotted unbind information. + * @unbind_fence: Fence to mark unbinding complete. Note that this fence + * is not considered published until unbind is scheduled, and as such it + * is illegal to access this fence before scheduled unbind other than + * for refcounting. + * @lock: The @unbind_fence lock. + * @hold_count: Number of holders blocking the fence from finishing. + * The vma itself is keeping a hold, which is released when unbind + * is scheduled. + * @work: Work struct for deferred unbind work. + * @chain: Pointer to struct i915_sw_fence used to await dependencies. + * @rb: Rb node for the vm's pending unbind interval tree. + * @__subtree_last: Interval tree private member. + * @vm: non-refcounted pointer to the vm. This is for internal use only and + * this member is cleared after vm_resource unbind. + * @mr: The memory region of the object pointed to by the vma. + * @ops: Pointer to the backend i915_vma_ops. + * @private: Bind backend private info. + * @start: Offset into the address space of bind range start. + * @node_size: Size of the allocated range manager node. + * @vma_size: Bind size. + * @page_sizes_gtt: Resulting page sizes from the bind operation. + * @bound_flags: Flags indicating binding status. + * @allocated: Backend private data. TODO: Should move into @private. + * @immediate_unbind: Unbind can be done immediately and doesn't need to be + * deferred to a work item awaiting unsignaled fences. This is a hack. + * (dma_fence_work uses a fence flag for this, but this seems slightly + * cleaner). + * + * The lifetime of a struct i915_vma_resource is from a binding request to + * the actual possible asynchronous unbind has completed. + */ +struct i915_vma_resource { + struct dma_fence unbind_fence; + /* See above for description of the lock. */ + spinlock_t lock; + refcount_t hold_count; + struct work_struct work; + struct i915_sw_fence chain; + struct rb_node rb; + u64 __subtree_last; + struct i915_address_space *vm; + intel_wakeref_t wakeref; + + /** + * struct i915_vma_bindinfo - Information needed for async bind + * only but that can be dropped after the bind has taken place. + * Consider making this a separate argument to the bind_vma + * op, coalescing with other arguments like vm, stash, cache_level + * and flags + * @pages: The pages sg-table. + * @page_sizes: Page sizes of the pages. + * @pages_rsgt: Refcounted sg-table when delayed object destruction + * is supported. May be NULL. + * @readonly: Whether the vma should be bound read-only. + * @lmem: Whether the vma points to lmem. + */ + struct i915_vma_bindinfo { + struct sg_table *pages; + struct i915_page_sizes page_sizes; + struct i915_refct_sgt *pages_rsgt; + bool readonly:1; + bool lmem:1; + } bi; + +#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) + struct intel_memory_region *mr; +#endif + const struct i915_vma_ops *ops; + void *private; + u64 start; + u64 node_size; + u64 vma_size; + u32 page_sizes_gtt; + + u32 bound_flags; + bool allocated:1; + bool immediate_unbind:1; + bool needs_wakeref:1; +}; + +bool i915_vma_resource_hold(struct i915_vma_resource *vma_res, + bool *lockdep_cookie); + +void i915_vma_resource_unhold(struct i915_vma_resource *vma_res, + bool lockdep_cookie); + +struct i915_vma_resource *i915_vma_resource_alloc(void); + +void i915_vma_resource_free(struct i915_vma_resource *vma_res); + +struct dma_fence *i915_vma_resource_unbind(struct i915_vma_resource *vma_res); + +void __i915_vma_resource_init(struct i915_vma_resource *vma_res); + +/** + * i915_vma_resource_get - Take a reference on a vma resource + * @vma_res: The vma resource on which to take a reference. + * + * Return: The @vma_res pointer + */ +static inline struct i915_vma_resource +*i915_vma_resource_get(struct i915_vma_resource *vma_res) +{ + dma_fence_get(&vma_res->unbind_fence); + return vma_res; +} + +/** + * i915_vma_resource_put - Release a reference to a struct i915_vma_resource + * @vma_res: The resource + */ +static inline void i915_vma_resource_put(struct i915_vma_resource *vma_res) +{ + dma_fence_put(&vma_res->unbind_fence); +} + +/** + * i915_vma_resource_init - Initialize a vma resource. + * @vma_res: The vma resource to initialize + * @vm: Pointer to the vm. + * @pages: The pages sg-table. + * @page_sizes: Page sizes of the pages. + * @pages_rsgt: Pointer to a struct i915_refct_sgt of an object with + * delayed destruction. + * @readonly: Whether the vma should be bound read-only. + * @lmem: Whether the vma points to lmem. + * @mr: The memory region of the object the vma points to. + * @ops: The backend ops. + * @private: Bind backend private info. + * @start: Offset into the address space of bind range start. + * @node_size: Size of the allocated range manager node. + * @size: Bind size. + * + * Initializes a vma resource allocated using i915_vma_resource_alloc(). + * The reason for having separate allocate and initialize function is that + * initialization may need to be performed from under a lock where + * allocation is not allowed. + */ +static inline void i915_vma_resource_init(struct i915_vma_resource *vma_res, + struct i915_address_space *vm, + struct sg_table *pages, + const struct i915_page_sizes *page_sizes, + struct i915_refct_sgt *pages_rsgt, + bool readonly, + bool lmem, + struct intel_memory_region *mr, + const struct i915_vma_ops *ops, + void *private, + u64 start, + u64 node_size, + u64 size) +{ + __i915_vma_resource_init(vma_res); + vma_res->vm = vm; + vma_res->bi.pages = pages; + vma_res->bi.page_sizes = *page_sizes; + if (pages_rsgt) + vma_res->bi.pages_rsgt = i915_refct_sgt_get(pages_rsgt); + vma_res->bi.readonly = readonly; + vma_res->bi.lmem = lmem; +#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) + vma_res->mr = mr; +#endif + vma_res->ops = ops; + vma_res->private = private; + vma_res->start = start; + vma_res->node_size = node_size; + vma_res->vma_size = size; +} + +static inline void i915_vma_resource_fini(struct i915_vma_resource *vma_res) +{ + GEM_BUG_ON(refcount_read(&vma_res->hold_count) != 1); + if (vma_res->bi.pages_rsgt) + i915_refct_sgt_put(vma_res->bi.pages_rsgt); + i915_sw_fence_fini(&vma_res->chain); +} + +int i915_vma_resource_bind_dep_sync(struct i915_address_space *vm, + u64 first, + u64 last, + bool intr); + +int i915_vma_resource_bind_dep_await(struct i915_address_space *vm, + struct i915_sw_fence *sw_fence, + u64 first, + u64 last, + bool intr, + gfp_t gfp); + +void i915_vma_resource_bind_dep_sync_all(struct i915_address_space *vm); + +void i915_vma_resource_module_exit(void); + +int i915_vma_resource_module_init(void); + +#endif diff --git a/drivers/gpu/drm/i915/i915_vma_snapshot.c b/drivers/gpu/drm/i915/i915_vma_snapshot.c deleted file mode 100644 index 2949ceea9884..000000000000 --- a/drivers/gpu/drm/i915/i915_vma_snapshot.c +++ /dev/null @@ -1,134 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include "i915_vma_snapshot.h" -#include "i915_vma_types.h" -#include "i915_vma.h" - -/** - * i915_vma_snapshot_init - Initialize a struct i915_vma_snapshot from - * a struct i915_vma. - * @vsnap: The i915_vma_snapshot to init. - * @vma: A struct i915_vma used to initialize @vsnap. - * @name: Name associated with the snapshot. The character pointer needs to - * stay alive over the lifitime of the shapsot - */ -void i915_vma_snapshot_init(struct i915_vma_snapshot *vsnap, - struct i915_vma *vma, - const char *name) -{ - if (!i915_vma_is_pinned(vma)) - assert_object_held(vma->obj); - - vsnap->name = name; - vsnap->size = vma->size; - vsnap->obj_size = vma->obj->base.size; - vsnap->gtt_offset = vma->node.start; - vsnap->gtt_size = vma->node.size; - vsnap->page_sizes = vma->page_sizes.gtt; - vsnap->pages = vma->pages; - vsnap->pages_rsgt = NULL; - vsnap->mr = NULL; - if (vma->obj->mm.rsgt) - vsnap->pages_rsgt = i915_refct_sgt_get(vma->obj->mm.rsgt); - vsnap->mr = vma->obj->mm.region; - kref_init(&vsnap->kref); - vsnap->vma_resource = &vma->active; - vsnap->onstack = false; - vsnap->present = true; -} - -/** - * i915_vma_snapshot_init_onstack - Initialize a struct i915_vma_snapshot from - * a struct i915_vma, but avoid kfreeing it on last put. - * @vsnap: The i915_vma_snapshot to init. - * @vma: A struct i915_vma used to initialize @vsnap. - * @name: Name associated with the snapshot. The character pointer needs to - * stay alive over the lifitime of the shapsot - */ -void i915_vma_snapshot_init_onstack(struct i915_vma_snapshot *vsnap, - struct i915_vma *vma, - const char *name) -{ - i915_vma_snapshot_init(vsnap, vma, name); - vsnap->onstack = true; -} - -static void vma_snapshot_release(struct kref *ref) -{ - struct i915_vma_snapshot *vsnap = - container_of(ref, typeof(*vsnap), kref); - - vsnap->present = false; - if (vsnap->pages_rsgt) - i915_refct_sgt_put(vsnap->pages_rsgt); - if (!vsnap->onstack) - kfree(vsnap); -} - -/** - * i915_vma_snapshot_put - Put an i915_vma_snapshot pointer reference - * @vsnap: The pointer reference - */ -void i915_vma_snapshot_put(struct i915_vma_snapshot *vsnap) -{ - kref_put(&vsnap->kref, vma_snapshot_release); -} - -/** - * i915_vma_snapshot_put_onstack - Put an onstcak i915_vma_snapshot pointer - * reference and varify that the structure is released - * @vsnap: The pointer reference - * - * This function is intended to be paired with a i915_vma_init_onstack() - * and should be called before exiting the scope that declared or - * freeing the structure that embedded @vsnap to verify that all references - * have been released. - */ -void i915_vma_snapshot_put_onstack(struct i915_vma_snapshot *vsnap) -{ - if (!kref_put(&vsnap->kref, vma_snapshot_release)) - GEM_BUG_ON(1); -} - -/** - * i915_vma_snapshot_resource_pin - Temporarily block the memory the - * vma snapshot is pointing to from being released. - * @vsnap: The vma snapshot. - * @lockdep_cookie: Pointer to bool needed for lockdep support. This needs - * to be passed to the paired i915_vma_snapshot_resource_unpin. - * - * This function will temporarily try to hold up a fence or similar structure - * and will therefore enter a fence signaling critical section. - * - * Return: true if we succeeded in blocking the memory from being released, - * false otherwise. - */ -bool i915_vma_snapshot_resource_pin(struct i915_vma_snapshot *vsnap, - bool *lockdep_cookie) -{ - bool pinned = i915_active_acquire_if_busy(vsnap->vma_resource); - - if (pinned) - *lockdep_cookie = dma_fence_begin_signalling(); - - return pinned; -} - -/** - * i915_vma_snapshot_resource_unpin - Unblock vma snapshot memory from - * being released. - * @vsnap: The vma snapshot. - * @lockdep_cookie: Cookie returned from matching i915_vma_resource_pin(). - * - * Might leave a fence signalling critical section and signal a fence. - */ -void i915_vma_snapshot_resource_unpin(struct i915_vma_snapshot *vsnap, - bool lockdep_cookie) -{ - dma_fence_end_signalling(lockdep_cookie); - - return i915_active_release(vsnap->vma_resource); -} diff --git a/drivers/gpu/drm/i915/i915_vma_snapshot.h b/drivers/gpu/drm/i915/i915_vma_snapshot.h deleted file mode 100644 index 940581df4622..000000000000 --- a/drivers/gpu/drm/i915/i915_vma_snapshot.h +++ /dev/null @@ -1,112 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2021 Intel Corporation - */ -#ifndef _I915_VMA_SNAPSHOT_H_ -#define _I915_VMA_SNAPSHOT_H_ - -#include <linux/kref.h> -#include <linux/slab.h> -#include <linux/types.h> - -struct i915_active; -struct i915_refct_sgt; -struct i915_vma; -struct intel_memory_region; -struct sg_table; - -/** - * DOC: Simple utilities for snapshotting GPU vma metadata, later used for - * error capture. Vi use a separate header for this to avoid issues due to - * recursive header includes. - */ - -/** - * struct i915_vma_snapshot - Snapshot of vma metadata. - * @size: The vma size in bytes. - * @obj_size: The size of the underlying object in bytes. - * @gtt_offset: The gtt offset the vma is bound to. - * @gtt_size: The size in bytes allocated for the vma in the GTT. - * @pages: The struct sg_table pointing to the pages bound. - * @pages_rsgt: The refcounted sg_table holding the reference for @pages if any. - * @mr: The memory region pointed for the pages bound. - * @kref: Reference for this structure. - * @vma_resource: FIXME: A means to keep the unbind fence from signaling. - * Temporarily while we have only sync unbinds, and still use the vma - * active, we use that. With async unbinding we need a signaling refcount - * for the unbind fence. - * @page_sizes: The vma GTT page sizes information. - * @onstack: Whether the structure shouldn't be freed on final put. - * @present: Whether the structure is present and initialized. - */ -struct i915_vma_snapshot { - const char *name; - size_t size; - size_t obj_size; - size_t gtt_offset; - size_t gtt_size; - struct sg_table *pages; - struct i915_refct_sgt *pages_rsgt; - struct intel_memory_region *mr; - struct kref kref; - struct i915_active *vma_resource; - u32 page_sizes; - bool onstack:1; - bool present:1; -}; - -void i915_vma_snapshot_init(struct i915_vma_snapshot *vsnap, - struct i915_vma *vma, - const char *name); - -void i915_vma_snapshot_init_onstack(struct i915_vma_snapshot *vsnap, - struct i915_vma *vma, - const char *name); - -void i915_vma_snapshot_put(struct i915_vma_snapshot *vsnap); - -void i915_vma_snapshot_put_onstack(struct i915_vma_snapshot *vsnap); - -bool i915_vma_snapshot_resource_pin(struct i915_vma_snapshot *vsnap, - bool *lockdep_cookie); - -void i915_vma_snapshot_resource_unpin(struct i915_vma_snapshot *vsnap, - bool lockdep_cookie); - -/** - * i915_vma_snapshot_alloc - Allocate a struct i915_vma_snapshot - * @gfp: Allocation mode. - * - * Return: A pointer to a struct i915_vma_snapshot if successful. - * NULL otherwise. - */ -static inline struct i915_vma_snapshot *i915_vma_snapshot_alloc(gfp_t gfp) -{ - return kmalloc(sizeof(struct i915_vma_snapshot), gfp); -} - -/** - * i915_vma_snapshot_get - Take a reference on a struct i915_vma_snapshot - * - * Return: A pointer to a struct i915_vma_snapshot. - */ -static inline struct i915_vma_snapshot * -i915_vma_snapshot_get(struct i915_vma_snapshot *vsnap) -{ - kref_get(&vsnap->kref); - return vsnap; -} - -/** - * i915_vma_snapshot_present - Whether a struct i915_vma_snapshot is - * present and initialized. - * - * Return: true if present and initialized; false otherwise. - */ -static inline bool -i915_vma_snapshot_present(const struct i915_vma_snapshot *vsnap) -{ - return vsnap && vsnap->present; -} - -#endif diff --git a/drivers/gpu/drm/i915/i915_vma_types.h b/drivers/gpu/drm/i915/i915_vma_types.h index ca575e129ced..88370dadca82 100644 --- a/drivers/gpu/drm/i915/i915_vma_types.h +++ b/drivers/gpu/drm/i915/i915_vma_types.h @@ -95,6 +95,8 @@ enum i915_cache_level; * */ +struct i915_vma_resource; + struct intel_remapped_plane_info { /* in gtt pages */ u32 offset:31; @@ -247,22 +249,20 @@ struct i915_vma { #define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND) -#define I915_VMA_ALLOC_BIT 12 - -#define I915_VMA_ERROR_BIT 13 +#define I915_VMA_ERROR_BIT 12 #define I915_VMA_ERROR ((int)BIT(I915_VMA_ERROR_BIT)) -#define I915_VMA_GGTT_BIT 14 -#define I915_VMA_CAN_FENCE_BIT 15 -#define I915_VMA_USERFAULT_BIT 16 -#define I915_VMA_GGTT_WRITE_BIT 17 +#define I915_VMA_GGTT_BIT 13 +#define I915_VMA_CAN_FENCE_BIT 14 +#define I915_VMA_USERFAULT_BIT 15 +#define I915_VMA_GGTT_WRITE_BIT 16 #define I915_VMA_GGTT ((int)BIT(I915_VMA_GGTT_BIT)) #define I915_VMA_CAN_FENCE ((int)BIT(I915_VMA_CAN_FENCE_BIT)) #define I915_VMA_USERFAULT ((int)BIT(I915_VMA_USERFAULT_BIT)) #define I915_VMA_GGTT_WRITE ((int)BIT(I915_VMA_GGTT_WRITE_BIT)) -#define I915_VMA_SCANOUT_BIT 18 +#define I915_VMA_SCANOUT_BIT 17 #define I915_VMA_SCANOUT ((int)BIT(I915_VMA_SCANOUT_BIT)) struct i915_active active; @@ -291,6 +291,9 @@ struct i915_vma { struct list_head evict_link; struct list_head closed_link; + + /** The async vma resource. Protected by the vm_mutex */ + struct i915_vma_resource *resource; }; #endif diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index 04fd266d70e2..32c5f10e31db 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -114,7 +114,7 @@ void intel_device_info_print_static(const struct intel_device_info *info, DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG); #undef PRINT_FLAG -#define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, yesno(info->display.name)); +#define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, yesno(info->display.name)) DEV_INFO_DISPLAY_FOR_EACH_FLAG(PRINT_FLAG); #undef PRINT_FLAG } @@ -170,6 +170,14 @@ static const u16 subplatform_portf_ids[] = { INTEL_ICL_PORT_F_IDS(0), }; +static const u16 subplatform_uy_ids[] = { + INTEL_TGL_12_GT2_IDS(0), +}; + +static const u16 subplatform_n_ids[] = { + INTEL_ADLN_IDS(0), +}; + static const u16 subplatform_rpls_ids[] = { INTEL_RPLS_IDS(0), }; @@ -210,30 +218,17 @@ void intel_device_info_subplatform_init(struct drm_i915_private *i915) } else if (find_devid(devid, subplatform_portf_ids, ARRAY_SIZE(subplatform_portf_ids))) { mask = BIT(INTEL_SUBPLATFORM_PORTF); + } else if (find_devid(devid, subplatform_uy_ids, + ARRAY_SIZE(subplatform_uy_ids))) { + mask = BIT(INTEL_SUBPLATFORM_UY); + } else if (find_devid(devid, subplatform_n_ids, + ARRAY_SIZE(subplatform_n_ids))) { + mask = BIT(INTEL_SUBPLATFORM_N); } else if (find_devid(devid, subplatform_rpls_ids, ARRAY_SIZE(subplatform_rpls_ids))) { mask = BIT(INTEL_SUBPLATFORM_RPL_S); } - if (IS_TIGERLAKE(i915)) { - struct pci_dev *root, *pdev = to_pci_dev(i915->drm.dev); - - root = list_first_entry(&pdev->bus->devices, typeof(*root), bus_list); - - drm_WARN_ON(&i915->drm, mask); - drm_WARN_ON(&i915->drm, (root->device & TGL_ROOT_DEVICE_MASK) != - TGL_ROOT_DEVICE_ID); - - switch (root->device & TGL_ROOT_DEVICE_SKU_MASK) { - case TGL_ROOT_DEVICE_SKU_ULX: - mask = BIT(INTEL_SUBPLATFORM_ULX); - break; - case TGL_ROOT_DEVICE_SKU_ULT: - mask = BIT(INTEL_SUBPLATFORM_ULT); - break; - } - } - GEM_BUG_ON(mask & ~INTEL_SUBPLATFORM_MASK); RUNTIME_INFO(i915)->platform_mask[pi] |= mask; @@ -328,6 +323,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) "Display fused off, disabling\n"); info->display.pipe_mask = 0; info->display.cpu_transcoder_mask = 0; + info->display.fbc_mask = 0; } else if (fuse_strap & IVB_PIPE_C_DISABLE) { drm_info(&dev_priv->drm, "PipeC fused off\n"); info->display.pipe_mask &= ~BIT(PIPE_C); @@ -339,6 +335,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) if (dfsm & SKL_DFSM_PIPE_A_DISABLE) { info->display.pipe_mask &= ~BIT(PIPE_A); info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_A); + info->display.fbc_mask &= ~BIT(INTEL_FBC_A); } if (dfsm & SKL_DFSM_PIPE_B_DISABLE) { info->display.pipe_mask &= ~BIT(PIPE_B); @@ -359,7 +356,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) info->display.has_hdcp = 0; if (dfsm & SKL_DFSM_DISPLAY_PM_DISABLE) - info->display.has_fbc = 0; + info->display.fbc_mask = 0; if (DISPLAY_VER(dev_priv) >= 11 && (dfsm & ICL_DFSM_DMC_DISABLE)) info->display.has_dmc = 0; diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index 78597d382445..291215d9da28 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -96,7 +96,7 @@ enum intel_platform { * it is fine for the same bit to be used on multiple parent platforms. */ -#define INTEL_SUBPLATFORM_BITS (2) +#define INTEL_SUBPLATFORM_BITS (3) #define INTEL_SUBPLATFORM_MASK (BIT(INTEL_SUBPLATFORM_BITS) - 1) /* HSW/BDW/SKL/KBL/CFL */ @@ -106,13 +106,20 @@ enum intel_platform { /* ICL */ #define INTEL_SUBPLATFORM_PORTF (0) +/* TGL */ +#define INTEL_SUBPLATFORM_UY (0) + /* DG2 */ #define INTEL_SUBPLATFORM_G10 0 #define INTEL_SUBPLATFORM_G11 1 +#define INTEL_SUBPLATFORM_G12 2 /* ADL-S */ #define INTEL_SUBPLATFORM_RPL_S 0 +/* ADL-P */ +#define INTEL_SUBPLATFORM_N 0 + enum intel_ppgtt_type { INTEL_PPGTT_NONE = I915_GEM_PPGTT_NONE, INTEL_PPGTT_ALIASING = I915_GEM_PPGTT_ALIASING, @@ -127,10 +134,13 @@ enum intel_ppgtt_type { /* Keep has_* in alphabetical order */ \ func(has_64bit_reloc); \ func(has_64k_pages); \ + func(needs_compact_pt); \ func(gpu_reset_clobbers_display); \ func(has_reset_engine); \ + func(has_flat_ccs); \ func(has_global_mocs); \ func(has_gt_uc); \ + func(has_guc_deprivilege); \ func(has_l3_dpf); \ func(has_llc); \ func(has_logical_ring_contexts); \ @@ -156,7 +166,6 @@ enum intel_ppgtt_type { func(has_dp_mst); \ func(has_dsb); \ func(has_dsc); \ - func(has_fbc); \ func(has_fpga_dbg); \ func(has_gmch); \ func(has_hdcp); \ @@ -206,6 +215,7 @@ struct intel_device_info { u8 pipe_mask; u8 cpu_transcoder_mask; + u8 fbc_mask; u8 abox_mask; #define DEFINE_FLAG(name) u8 name:1 diff --git a/drivers/gpu/drm/i915/intel_dram.c b/drivers/gpu/drm/i915/intel_dram.c index 84bb212bae4b..174c95c3e10f 100644 --- a/drivers/gpu/drm/i915/intel_dram.c +++ b/drivers/gpu/drm/i915/intel_dram.c @@ -4,7 +4,9 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "intel_dram.h" +#include "intel_mchbar_regs.h" #include "intel_pcode.h" struct dram_dimm_info { @@ -389,10 +391,8 @@ static int icl_pcode_read_mem_global_info(struct drm_i915_private *dev_priv) u32 val = 0; int ret; - ret = sandybridge_pcode_read(dev_priv, - ICL_PCODE_MEM_SUBSYSYSTEM_INFO | - ICL_PCODE_MEM_SS_READ_GLOBAL_INFO, - &val, NULL); + ret = snb_pcode_read(dev_priv, ICL_PCODE_MEM_SUBSYSYSTEM_INFO | + ICL_PCODE_MEM_SS_READ_GLOBAL_INFO, &val, NULL); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/intel_mchbar_regs.h b/drivers/gpu/drm/i915/intel_mchbar_regs.h new file mode 100644 index 000000000000..2aad2f0cc8db --- /dev/null +++ b/drivers/gpu/drm/i915/intel_mchbar_regs.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_MCHBAR_REGS__ +#define __INTEL_MCHBAR_REGS__ + +#include "i915_reg_defs.h" + +/* + * MCHBAR mirror. + * + * This mirrors the MCHBAR MMIO space whose location is determined by + * device 0 function 0's pci config register 0x44 or 0x48 and matches it in + * every way. It is not accessible from the CP register read instructions. + * + * Starting from Haswell, you can't write registers using the MCHBAR mirror, + * just read. + */ + +#define MCHBAR_MIRROR_BASE 0x10000 +#define MCHBAR_MIRROR_BASE_SNB 0x140000 + +#define CTG_STOLEN_RESERVED _MMIO(MCHBAR_MIRROR_BASE + 0x34) +#define ELK_STOLEN_RESERVED _MMIO(MCHBAR_MIRROR_BASE + 0x48) +#define G4X_STOLEN_RESERVED_ADDR1_MASK (0xFFFF << 16) +#define G4X_STOLEN_RESERVED_ADDR2_MASK (0xFFF << 4) +#define G4X_STOLEN_RESERVED_ENABLE (1 << 0) + +/* Pineview MCH register contains DDR3 setting */ +#define CSHRDDR3CTL _MMIO(MCHBAR_MIRROR_BASE + 0x1a8) +#define CSHRDDR3CTL_DDR3 (1 << 2) + +/* 915-945 and GM965 MCH register controlling DRAM channel access */ +#define DCC _MMIO(MCHBAR_MIRROR_BASE + 0x200) +#define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0) +#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC (1 << 0) +#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0) +#define DCC_ADDRESSING_MODE_MASK (3 << 0) +#define DCC_CHANNEL_XOR_DISABLE (1 << 10) +#define DCC_CHANNEL_XOR_BIT_17 (1 << 9) +#define DCC2 _MMIO(MCHBAR_MIRROR_BASE + 0x204) +#define DCC2_MODIFIED_ENHANCED_DISABLE (1 << 20) + +/* 965 MCH register controlling DRAM channel configuration */ +#define C0DRB3_BW _MMIO(MCHBAR_MIRROR_BASE + 0x206) +#define C1DRB3_BW _MMIO(MCHBAR_MIRROR_BASE + 0x606) + +/* Clocking configuration register */ +#define CLKCFG _MMIO(MCHBAR_MIRROR_BASE + 0xc00) +#define CLKCFG_FSB_400 (0 << 0) /* hrawclk 100 */ +#define CLKCFG_FSB_400_ALT (5 << 0) /* hrawclk 100 */ +#define CLKCFG_FSB_533 (1 << 0) /* hrawclk 133 */ +#define CLKCFG_FSB_667 (3 << 0) /* hrawclk 166 */ +#define CLKCFG_FSB_800 (2 << 0) /* hrawclk 200 */ +#define CLKCFG_FSB_1067 (6 << 0) /* hrawclk 266 */ +#define CLKCFG_FSB_1067_ALT (0 << 0) /* hrawclk 266 */ +#define CLKCFG_FSB_1333 (7 << 0) /* hrawclk 333 */ +#define CLKCFG_FSB_1333_ALT (4 << 0) /* hrawclk 333 */ +#define CLKCFG_FSB_1600_ALT (6 << 0) /* hrawclk 400 */ +#define CLKCFG_FSB_MASK (7 << 0) +#define CLKCFG_MEM_533 (1 << 4) +#define CLKCFG_MEM_667 (2 << 4) +#define CLKCFG_MEM_800 (3 << 4) +#define CLKCFG_MEM_MASK (7 << 4) + +#define HPLLVCO_MOBILE _MMIO(MCHBAR_MIRROR_BASE + 0xc0f) +#define HPLLVCO _MMIO(MCHBAR_MIRROR_BASE + 0xc38) + +#define TSC1 _MMIO(MCHBAR_MIRROR_BASE + 0x1001) +#define TSE (1 << 0) +#define TR1 _MMIO(MCHBAR_MIRROR_BASE + 0x1006) +#define TSFS _MMIO(MCHBAR_MIRROR_BASE + 0x1020) +#define TSFS_SLOPE_MASK 0x0000ff00 +#define TSFS_SLOPE_SHIFT 8 +#define TSFS_INTR_MASK 0x000000ff + +/* Memory latency timer register */ +#define MLTR_ILK _MMIO(MCHBAR_MIRROR_BASE + 0x1222) +/* the unit of memory self-refresh latency time is 0.5us */ +#define MLTR_WM2_MASK REG_GENMASK(13, 8) +#define MLTR_WM1_MASK REG_GENMASK(5, 0) + +#define CSIPLL0 _MMIO(MCHBAR_MIRROR_BASE + 0x2c10) +#define DDRMPLL1 _MMIO(MCHBAR_MIRROR_BASE + 0x2c20) + +#define ILK_GDSR _MMIO(MCHBAR_MIRROR_BASE + 0x2ca4) +#define ILK_GRDOM_FULL (0 << 1) +#define ILK_GRDOM_RENDER (1 << 1) +#define ILK_GRDOM_MEDIA (3 << 1) +#define ILK_GRDOM_MASK (3 << 1) +#define ILK_GRDOM_RESET_ENABLE (1 << 0) + +#define BXT_D_CR_DRP0_DUNIT8 0x1000 +#define BXT_D_CR_DRP0_DUNIT9 0x1200 +#define BXT_D_CR_DRP0_DUNIT_START 8 +#define BXT_D_CR_DRP0_DUNIT_END 11 +#define BXT_D_CR_DRP0_DUNIT(x) _MMIO(MCHBAR_MIRROR_BASE_SNB + \ + _PICK_EVEN((x) - 8, BXT_D_CR_DRP0_DUNIT8,\ + BXT_D_CR_DRP0_DUNIT9)) +#define BXT_DRAM_RANK_MASK 0x3 +#define BXT_DRAM_RANK_SINGLE 0x1 +#define BXT_DRAM_RANK_DUAL 0x3 +#define BXT_DRAM_WIDTH_MASK (0x3 << 4) +#define BXT_DRAM_WIDTH_SHIFT 4 +#define BXT_DRAM_WIDTH_X8 (0x0 << 4) +#define BXT_DRAM_WIDTH_X16 (0x1 << 4) +#define BXT_DRAM_WIDTH_X32 (0x2 << 4) +#define BXT_DRAM_WIDTH_X64 (0x3 << 4) +#define BXT_DRAM_SIZE_MASK (0x7 << 6) +#define BXT_DRAM_SIZE_SHIFT 6 +#define BXT_DRAM_SIZE_4GBIT (0x0 << 6) +#define BXT_DRAM_SIZE_6GBIT (0x1 << 6) +#define BXT_DRAM_SIZE_8GBIT (0x2 << 6) +#define BXT_DRAM_SIZE_12GBIT (0x3 << 6) +#define BXT_DRAM_SIZE_16GBIT (0x4 << 6) +#define BXT_DRAM_TYPE_MASK (0x7 << 22) +#define BXT_DRAM_TYPE_SHIFT 22 +#define BXT_DRAM_TYPE_DDR3 (0x0 << 22) +#define BXT_DRAM_TYPE_LPDDR3 (0x1 << 22) +#define BXT_DRAM_TYPE_LPDDR4 (0x2 << 22) +#define BXT_DRAM_TYPE_DDR4 (0x4 << 22) + +#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4000) +#define DG1_DRAM_T_RDPRE_MASK REG_GENMASK(16, 11) +#define DG1_DRAM_T_RP_MASK REG_GENMASK(6, 0) +#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4004) +#define DG1_DRAM_T_RCD_MASK REG_GENMASK(15, 9) +#define DG1_DRAM_T_RAS_MASK REG_GENMASK(8, 1) + +#define SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5000) +#define SKL_DRAM_DDR_TYPE_MASK (0x3 << 0) +#define SKL_DRAM_DDR_TYPE_DDR4 (0 << 0) +#define SKL_DRAM_DDR_TYPE_DDR3 (1 << 0) +#define SKL_DRAM_DDR_TYPE_LPDDR3 (2 << 0) +#define SKL_DRAM_DDR_TYPE_LPDDR4 (3 << 0) + +/* snb MCH registers for reading the DRAM channel configuration */ +#define MAD_DIMM_C0 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004) +#define MAD_DIMM_C1 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5008) +#define MAD_DIMM_C2 _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C) +#define MAD_DIMM_ECC_MASK (0x3 << 24) +#define MAD_DIMM_ECC_OFF (0x0 << 24) +#define MAD_DIMM_ECC_IO_ON_LOGIC_OFF (0x1 << 24) +#define MAD_DIMM_ECC_IO_OFF_LOGIC_ON (0x2 << 24) +#define MAD_DIMM_ECC_ON (0x3 << 24) +#define MAD_DIMM_ENH_INTERLEAVE (0x1 << 22) +#define MAD_DIMM_RANK_INTERLEAVE (0x1 << 21) +#define MAD_DIMM_B_WIDTH_X16 (0x1 << 20) /* X8 chips if unset */ +#define MAD_DIMM_A_WIDTH_X16 (0x1 << 19) /* X8 chips if unset */ +#define MAD_DIMM_B_DUAL_RANK (0x1 << 18) +#define MAD_DIMM_A_DUAL_RANK (0x1 << 17) +#define MAD_DIMM_A_SELECT (0x1 << 16) +/* DIMM sizes are in multiples of 256mb. */ +#define MAD_DIMM_B_SIZE_SHIFT 8 +#define MAD_DIMM_B_SIZE_MASK (0xff << MAD_DIMM_B_SIZE_SHIFT) +#define MAD_DIMM_A_SIZE_SHIFT 0 +#define MAD_DIMM_A_SIZE_MASK (0xff << MAD_DIMM_A_SIZE_SHIFT) + +#define SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C) +#define SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5010) +#define SKL_DRAM_S_SHIFT 16 +#define SKL_DRAM_SIZE_MASK 0x3F +#define SKL_DRAM_WIDTH_MASK (0x3 << 8) +#define SKL_DRAM_WIDTH_SHIFT 8 +#define SKL_DRAM_WIDTH_X8 (0x0 << 8) +#define SKL_DRAM_WIDTH_X16 (0x1 << 8) +#define SKL_DRAM_WIDTH_X32 (0x2 << 8) +#define SKL_DRAM_RANK_MASK (0x1 << 10) +#define SKL_DRAM_RANK_SHIFT 10 +#define SKL_DRAM_RANK_1 (0x0 << 10) +#define SKL_DRAM_RANK_2 (0x1 << 10) +#define SKL_DRAM_RANK_MASK (0x1 << 10) +#define ICL_DRAM_SIZE_MASK 0x7F +#define ICL_DRAM_WIDTH_MASK (0x3 << 7) +#define ICL_DRAM_WIDTH_SHIFT 7 +#define ICL_DRAM_WIDTH_X8 (0x0 << 7) +#define ICL_DRAM_WIDTH_X16 (0x1 << 7) +#define ICL_DRAM_WIDTH_X32 (0x2 << 7) +#define ICL_DRAM_RANK_MASK (0x3 << 9) +#define ICL_DRAM_RANK_SHIFT 9 +#define ICL_DRAM_RANK_1 (0x0 << 9) +#define ICL_DRAM_RANK_2 (0x1 << 9) +#define ICL_DRAM_RANK_3 (0x2 << 9) +#define ICL_DRAM_RANK_4 (0x3 << 9) + +#define SA_PERF_STATUS_0_0_0_MCHBAR_PC _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5918) +#define DG1_QCLK_RATIO_MASK REG_GENMASK(9, 2) +#define DG1_QCLK_REFERENCE REG_BIT(10) + +#define GEN6_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5948) +#define GEN6_RP_STATE_LIMITS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994) +#define GEN6_RP_STATE_CAP _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998) +#define RP0_CAP_MASK REG_GENMASK(7, 0) +#define RP1_CAP_MASK REG_GENMASK(15, 8) +#define RPN_CAP_MASK REG_GENMASK(23, 16) + +/* snb MCH registers for priority tuning */ +#define MCH_SSKPD _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5d10) +#define SSKPD_NEW_WM0_MASK_HSW REG_GENMASK64(63, 56) +#define SSKPD_WM4_MASK_HSW REG_GENMASK64(40, 32) +#define SSKPD_WM3_MASK_HSW REG_GENMASK64(28, 20) +#define SSKPD_WM2_MASK_HSW REG_GENMASK64(19, 12) +#define SSKPD_WM1_MASK_HSW REG_GENMASK64(11, 4) +#define SSKPD_OLD_WM0_MASK_HSW REG_GENMASK64(3, 0) +#define SSKPD_WM3_MASK_SNB REG_GENMASK(29, 24) +#define SSKPD_WM2_MASK_SNB REG_GENMASK(21, 16) +#define SSKPD_WM1_MASK_SNB REG_GENMASK(13, 8) +#define SSKPD_WM0_MASK_SNB REG_GENMASK(5, 0) + +/* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */ +#define DCLK _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04) +#define SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04) +#define DG1_GEAR_TYPE REG_BIT(16) + +/* + * Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register, + * since on HSW we can't write to it using intel_uncore_write. + */ +#define D_COMP_HSW _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5f0c) +#define D_COMP_RCOMP_IN_PROGRESS (1 << 9) +#define D_COMP_COMP_FORCE (1 << 8) +#define D_COMP_COMP_DISABLE (1 << 0) + +#define BXT_GT_PERF_STATUS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7070) + +#endif /* __INTEL_MCHBAR_REGS */ diff --git a/drivers/gpu/drm/i915/intel_memory_region.c b/drivers/gpu/drm/i915/intel_memory_region.c index c70d7e286a51..1c841f68169a 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.c +++ b/drivers/gpu/drm/i915/intel_memory_region.c @@ -97,10 +97,14 @@ static int iomemtest(struct intel_memory_region *mem, bool test_all, const void *caller) { - resource_size_t last = resource_size(&mem->region) - PAGE_SIZE; - resource_size_t page; + resource_size_t last, page; int err; + if (mem->io_size < PAGE_SIZE) + return 0; + + last = mem->io_size - PAGE_SIZE; + /* * Quick test to check read/write access to the iomap (backing store). * @@ -217,6 +221,7 @@ intel_memory_region_create(struct drm_i915_private *i915, resource_size_t size, resource_size_t min_page_size, resource_size_t io_start, + resource_size_t io_size, u16 type, u16 instance, const struct intel_memory_region_ops *ops) @@ -231,6 +236,7 @@ intel_memory_region_create(struct drm_i915_private *i915, mem->i915 = i915; mem->region = (struct resource)DEFINE_RES_MEM(start, size); mem->io_start = io_start; + mem->io_size = io_size; mem->min_page_size = min_page_size; mem->ops = ops; mem->total = size; diff --git a/drivers/gpu/drm/i915/intel_memory_region.h b/drivers/gpu/drm/i915/intel_memory_region.h index 5625c9c38993..21dcbd620758 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.h +++ b/drivers/gpu/drm/i915/intel_memory_region.h @@ -67,10 +67,8 @@ struct intel_memory_region { struct io_mapping iomap; struct resource region; - /* For fake LMEM */ - struct drm_mm_node fake_mappable; - resource_size_t io_start; + resource_size_t io_size; resource_size_t min_page_size; resource_size_t total; resource_size_t avail; @@ -81,8 +79,6 @@ struct intel_memory_region { char name[16]; bool private; /* not for userspace */ - dma_addr_t remap_addr; - struct { struct mutex lock; /* Protects access to objects */ struct list_head list; @@ -103,6 +99,7 @@ intel_memory_region_create(struct drm_i915_private *i915, resource_size_t size, resource_size_t min_page_size, resource_size_t io_start, + resource_size_t io_size, u16 type, u16 instance, const struct intel_memory_region_ops *ops); diff --git a/drivers/gpu/drm/i915/intel_pch.c b/drivers/gpu/drm/i915/intel_pch.c index fc8a68f3a2ed..4cce044efde2 100644 --- a/drivers/gpu/drm/i915/intel_pch.c +++ b/drivers/gpu/drm/i915/intel_pch.c @@ -130,6 +130,7 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id) case INTEL_PCH_ADP_DEVICE_ID_TYPE: case INTEL_PCH_ADP2_DEVICE_ID_TYPE: case INTEL_PCH_ADP3_DEVICE_ID_TYPE: + case INTEL_PCH_ADP4_DEVICE_ID_TYPE: drm_dbg_kms(&dev_priv->drm, "Found Alder Lake PCH\n"); drm_WARN_ON(&dev_priv->drm, !IS_ALDERLAKE_S(dev_priv) && !IS_ALDERLAKE_P(dev_priv)); diff --git a/drivers/gpu/drm/i915/intel_pch.h b/drivers/gpu/drm/i915/intel_pch.h index 4ba0f1967cca..b7a8cf409d48 100644 --- a/drivers/gpu/drm/i915/intel_pch.h +++ b/drivers/gpu/drm/i915/intel_pch.h @@ -58,6 +58,7 @@ enum intel_pch { #define INTEL_PCH_ADP_DEVICE_ID_TYPE 0x7A80 #define INTEL_PCH_ADP2_DEVICE_ID_TYPE 0x5180 #define INTEL_PCH_ADP3_DEVICE_ID_TYPE 0x7A00 +#define INTEL_PCH_ADP4_DEVICE_ID_TYPE 0x5480 #define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100 #define INTEL_PCH_P3X_DEVICE_ID_TYPE 0x7000 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE 0x2900 /* qemu q35 has 2918 */ diff --git a/drivers/gpu/drm/i915/intel_pci_config.h b/drivers/gpu/drm/i915/intel_pci_config.h new file mode 100644 index 000000000000..12cd9d4f23de --- /dev/null +++ b/drivers/gpu/drm/i915/intel_pci_config.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_PCI_CONFIG_H__ +#define __INTEL_PCI_CONFIG_H__ + +/* BSM in include/drm/i915_drm.h */ + +#define MCHBAR_I915 0x44 +#define MCHBAR_I965 0x48 +#define MCHBAR_SIZE (4 * 4096) + +#define DEVEN 0x54 +#define DEVEN_MCHBAR_EN (1 << 28) + +#define HPLLCC 0xc0 /* 85x only */ +#define GC_CLOCK_CONTROL_MASK (0x7 << 0) +#define GC_CLOCK_133_200 (0 << 0) +#define GC_CLOCK_100_200 (1 << 0) +#define GC_CLOCK_100_133 (2 << 0) +#define GC_CLOCK_133_266 (3 << 0) +#define GC_CLOCK_133_200_2 (4 << 0) +#define GC_CLOCK_133_266_2 (5 << 0) +#define GC_CLOCK_166_266 (6 << 0) +#define GC_CLOCK_166_250 (7 << 0) + +#define I915_GDRST 0xc0 +#define GRDOM_FULL (0 << 2) +#define GRDOM_RENDER (1 << 2) +#define GRDOM_MEDIA (3 << 2) +#define GRDOM_MASK (3 << 2) +#define GRDOM_RESET_STATUS (1 << 1) +#define GRDOM_RESET_ENABLE (1 << 0) + +/* BSpec only has register offset, PCI device and bit found empirically */ +#define I830_CLOCK_GATE 0xc8 /* device 0 */ +#define I830_L2_CACHE_CLOCK_GATE_DISABLE (1 << 2) + +#define GCDGMBUS 0xcc + +#define GCFGC2 0xda +#define GCFGC 0xf0 /* 915+ only */ +#define GC_LOW_FREQUENCY_ENABLE (1 << 7) +#define GC_DISPLAY_CLOCK_190_200_MHZ (0 << 4) +#define GC_DISPLAY_CLOCK_333_320_MHZ (4 << 4) +#define GC_DISPLAY_CLOCK_267_MHZ_PNV (0 << 4) +#define GC_DISPLAY_CLOCK_333_MHZ_PNV (1 << 4) +#define GC_DISPLAY_CLOCK_444_MHZ_PNV (2 << 4) +#define GC_DISPLAY_CLOCK_200_MHZ_PNV (5 << 4) +#define GC_DISPLAY_CLOCK_133_MHZ_PNV (6 << 4) +#define GC_DISPLAY_CLOCK_167_MHZ_PNV (7 << 4) +#define GC_DISPLAY_CLOCK_MASK (7 << 4) +#define GM45_GC_RENDER_CLOCK_MASK (0xf << 0) +#define GM45_GC_RENDER_CLOCK_266_MHZ (8 << 0) +#define GM45_GC_RENDER_CLOCK_320_MHZ (9 << 0) +#define GM45_GC_RENDER_CLOCK_400_MHZ (0xb << 0) +#define GM45_GC_RENDER_CLOCK_533_MHZ (0xc << 0) +#define I965_GC_RENDER_CLOCK_MASK (0xf << 0) +#define I965_GC_RENDER_CLOCK_267_MHZ (2 << 0) +#define I965_GC_RENDER_CLOCK_333_MHZ (3 << 0) +#define I965_GC_RENDER_CLOCK_444_MHZ (4 << 0) +#define I965_GC_RENDER_CLOCK_533_MHZ (5 << 0) +#define I945_GC_RENDER_CLOCK_MASK (7 << 0) +#define I945_GC_RENDER_CLOCK_166_MHZ (0 << 0) +#define I945_GC_RENDER_CLOCK_200_MHZ (1 << 0) +#define I945_GC_RENDER_CLOCK_250_MHZ (3 << 0) +#define I945_GC_RENDER_CLOCK_400_MHZ (5 << 0) +#define I915_GC_RENDER_CLOCK_MASK (7 << 0) +#define I915_GC_RENDER_CLOCK_166_MHZ (0 << 0) +#define I915_GC_RENDER_CLOCK_200_MHZ (1 << 0) +#define I915_GC_RENDER_CLOCK_333_MHZ (4 << 0) + +#define ASLE 0xe4 +#define ASLS 0xfc + +#define SWSCI 0xe8 +#define SWSCI_SCISEL (1 << 15) +#define SWSCI_GSSCIE (1 << 0) + +/* legacy/combination backlight modes, also called LBB */ +#define LBPC 0xf4 + +#endif /* __INTEL_PCI_CONFIG_H__ */ diff --git a/drivers/gpu/drm/i915/intel_pcode.c b/drivers/gpu/drm/i915/intel_pcode.c index e8c886e4e78d..391a37492ce5 100644 --- a/drivers/gpu/drm/i915/intel_pcode.c +++ b/drivers/gpu/drm/i915/intel_pcode.c @@ -4,6 +4,7 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "intel_pcode.h" static int gen6_check_mailbox_status(u32 mbox) @@ -51,11 +52,10 @@ static int gen7_check_mailbox_status(u32 mbox) } } -static int __sandybridge_pcode_rw(struct drm_i915_private *i915, - u32 mbox, u32 *val, u32 *val1, - int fast_timeout_us, - int slow_timeout_ms, - bool is_read) +static int __snb_pcode_rw(struct drm_i915_private *i915, u32 mbox, + u32 *val, u32 *val1, + int fast_timeout_us, int slow_timeout_ms, + bool is_read) { struct intel_uncore *uncore = &i915->uncore; @@ -94,15 +94,12 @@ static int __sandybridge_pcode_rw(struct drm_i915_private *i915, return gen6_check_mailbox_status(mbox); } -int sandybridge_pcode_read(struct drm_i915_private *i915, u32 mbox, - u32 *val, u32 *val1) +int snb_pcode_read(struct drm_i915_private *i915, u32 mbox, u32 *val, u32 *val1) { int err; mutex_lock(&i915->sb_lock); - err = __sandybridge_pcode_rw(i915, mbox, val, val1, - 500, 20, - true); + err = __snb_pcode_rw(i915, mbox, val, val1, 500, 20, true); mutex_unlock(&i915->sb_lock); if (err) { @@ -114,17 +111,14 @@ int sandybridge_pcode_read(struct drm_i915_private *i915, u32 mbox, return err; } -int sandybridge_pcode_write_timeout(struct drm_i915_private *i915, - u32 mbox, u32 val, - int fast_timeout_us, - int slow_timeout_ms) +int snb_pcode_write_timeout(struct drm_i915_private *i915, u32 mbox, u32 val, + int fast_timeout_us, int slow_timeout_ms) { int err; mutex_lock(&i915->sb_lock); - err = __sandybridge_pcode_rw(i915, mbox, &val, NULL, - fast_timeout_us, slow_timeout_ms, - false); + err = __snb_pcode_rw(i915, mbox, &val, NULL, + fast_timeout_us, slow_timeout_ms, false); mutex_unlock(&i915->sb_lock); if (err) { @@ -140,9 +134,7 @@ static bool skl_pcode_try_request(struct drm_i915_private *i915, u32 mbox, u32 request, u32 reply_mask, u32 reply, u32 *status) { - *status = __sandybridge_pcode_rw(i915, mbox, &request, NULL, - 500, 0, - true); + *status = __snb_pcode_rw(i915, mbox, &request, NULL, 500, 0, true); return *status || ((request & reply_mask) == reply); } diff --git a/drivers/gpu/drm/i915/intel_pcode.h b/drivers/gpu/drm/i915/intel_pcode.h index 50806649d4b6..0962a17fac48 100644 --- a/drivers/gpu/drm/i915/intel_pcode.h +++ b/drivers/gpu/drm/i915/intel_pcode.h @@ -10,13 +10,11 @@ struct drm_i915_private; -int sandybridge_pcode_read(struct drm_i915_private *i915, u32 mbox, - u32 *val, u32 *val1); -int sandybridge_pcode_write_timeout(struct drm_i915_private *i915, u32 mbox, - u32 val, int fast_timeout_us, - int slow_timeout_ms); -#define sandybridge_pcode_write(i915, mbox, val) \ - sandybridge_pcode_write_timeout(i915, mbox, val, 500, 0) +int snb_pcode_read(struct drm_i915_private *i915, u32 mbox, u32 *val, u32 *val1); +int snb_pcode_write_timeout(struct drm_i915_private *i915, u32 mbox, u32 val, + int fast_timeout_us, int slow_timeout_ms); +#define snb_pcode_write(i915, mbox, val) \ + snb_pcode_write_timeout(i915, mbox, val, 500, 0) int skl_pcode_request(struct drm_i915_private *i915, u32 mbox, u32 request, u32 reply_mask, u32 reply, int timeout_base_ms); diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index fae4f7818d28..71f7fba2c9e2 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -43,16 +43,23 @@ #include "display/intel_sprite.h" #include "display/skl_universal_plane.h" +#include "gt/intel_engine_regs.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_llc.h" #include "i915_drv.h" #include "i915_fixed.h" #include "i915_irq.h" +#include "intel_mchbar_regs.h" #include "intel_pcode.h" #include "intel_pm.h" #include "vlv_sideband.h" #include "../../../platform/x86/intel_ips.h" +struct drm_i915_clock_gating_funcs { + void (*init_clock_gating)(struct drm_i915_private *i915); +}; + /* Stores plane specific WM parameters */ struct skl_wm_params { bool x_tiled, y_tiled; @@ -78,8 +85,6 @@ struct intel_wm_config { static void gen9_init_clock_gating(struct drm_i915_private *dev_priv) { - enum pipe pipe; - if (HAS_LLC(dev_priv)) { /* * WaCompressedResourceDisplayNewHashMode:skl,kbl @@ -93,16 +98,6 @@ static void gen9_init_clock_gating(struct drm_i915_private *dev_priv) SKL_DE_COMPRESSED_HASH_MODE); } - for_each_pipe(dev_priv, pipe) { - /* - * "Plane N strech max must be programmed to 11b (x1) - * when Async flips are enabled on that plane." - */ - if (!IS_GEMINILAKE(dev_priv) && intel_vtd_active(dev_priv)) - intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe), - SKL_PLANE1_STRETCH_MAX_MASK, SKL_PLANE1_STRETCH_MAX_X1); - } - /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */ intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP); @@ -160,8 +155,9 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv) * WaFbcHighMemBwCorruptionAvoidance:bxt * Display WA #0883: bxt */ - intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | - DPFC_DISABLE_DUMMY0); + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), + intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) | + DPFC_DISABLE_DUMMY0); } static void glk_init_clock_gating(struct drm_i915_private *dev_priv) @@ -876,7 +872,7 @@ static bool intel_crtc_active(struct intel_crtc *crtc) * crtc->state->active once we have proper CRTC states wired up * for atomic. */ - return crtc->active && crtc->base.primary->state->fb && + return crtc && crtc->active && crtc->base.primary->state->fb && crtc->config->hw.adjusted_mode.crtc_clock; } @@ -915,15 +911,13 @@ static void pnv_update_wm(struct drm_i915_private *dev_priv) crtc = single_enabled_crtc(dev_priv); if (crtc) { - const struct drm_display_mode *pipe_mode = - &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; + int pixel_rate = crtc->config->pixel_rate; int cpp = fb->format->cpp[0]; - int clock = pipe_mode->crtc_clock; /* Display SR */ - wm = intel_calculate_wm(clock, &pnv_display_wm, + wm = intel_calculate_wm(pixel_rate, &pnv_display_wm, pnv_display_wm.fifo_size, cpp, latency->display_sr); reg = intel_uncore_read(&dev_priv->uncore, DSPFW1); @@ -933,7 +927,7 @@ static void pnv_update_wm(struct drm_i915_private *dev_priv) drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg); /* cursor SR */ - wm = intel_calculate_wm(clock, &pnv_cursor_wm, + wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm, pnv_display_wm.fifo_size, 4, latency->cursor_sr); reg = intel_uncore_read(&dev_priv->uncore, DSPFW3); @@ -942,7 +936,7 @@ static void pnv_update_wm(struct drm_i915_private *dev_priv) intel_uncore_write(&dev_priv->uncore, DSPFW3, reg); /* Display HPLL off SR */ - wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm, + wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm, pnv_display_hplloff_wm.fifo_size, cpp, latency->display_hpll_disable); reg = intel_uncore_read(&dev_priv->uncore, DSPFW3); @@ -951,7 +945,7 @@ static void pnv_update_wm(struct drm_i915_private *dev_priv) intel_uncore_write(&dev_priv->uncore, DSPFW3, reg); /* cursor HPLL off SR */ - wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm, + wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm, pnv_display_hplloff_wm.fifo_size, 4, latency->cursor_hpll_disable); reg = intel_uncore_read(&dev_priv->uncore, DSPFW3); @@ -1154,7 +1148,7 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; unsigned int latency = dev_priv->wm.pri_latency[level] * 10; - unsigned int clock, htotal, cpp, width, wm; + unsigned int pixel_rate, htotal, cpp, width, wm; if (latency == 0) return USHRT_MAX; @@ -1175,21 +1169,20 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, level != G4X_WM_LEVEL_NORMAL) cpp = max(cpp, 4u); - clock = pipe_mode->crtc_clock; + pixel_rate = crtc_state->pixel_rate; htotal = pipe_mode->crtc_htotal; - - width = drm_rect_width(&plane_state->uapi.dst); + width = drm_rect_width(&plane_state->uapi.src) >> 16; if (plane->id == PLANE_CURSOR) { - wm = intel_wm_method2(clock, htotal, width, cpp, latency); + wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency); } else if (plane->id == PLANE_PRIMARY && level == G4X_WM_LEVEL_NORMAL) { - wm = intel_wm_method1(clock, cpp, latency); + wm = intel_wm_method1(pixel_rate, cpp, latency); } else { unsigned int small, large; - small = intel_wm_method1(clock, cpp, latency); - large = intel_wm_method2(clock, htotal, width, cpp, latency); + small = intel_wm_method1(pixel_rate, cpp, latency); + large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency); wm = min(small, large); } @@ -1674,7 +1667,7 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, struct drm_i915_private *dev_priv = to_i915(plane->base.dev); const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; - unsigned int clock, htotal, cpp, width, wm; + unsigned int pixel_rate, htotal, cpp, width, wm; if (dev_priv->wm.pri_latency[level] == 0) return USHRT_MAX; @@ -1683,9 +1676,9 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, return 0; cpp = plane_state->hw.fb->format->cpp[0]; - clock = pipe_mode->crtc_clock; + pixel_rate = crtc_state->pixel_rate; htotal = pipe_mode->crtc_htotal; - width = crtc_state->pipe_src_w; + width = drm_rect_width(&plane_state->uapi.src) >> 16; if (plane->id == PLANE_CURSOR) { /* @@ -1696,7 +1689,7 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, */ wm = 63; } else { - wm = vlv_wm_method2(clock, htotal, width, cpp, + wm = vlv_wm_method2(pixel_rate, htotal, width, cpp, dev_priv->wm.pri_latency[level] * 10); } @@ -2277,14 +2270,14 @@ static void i965_update_wm(struct drm_i915_private *dev_priv) &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; - int clock = pipe_mode->crtc_clock; + int pixel_rate = crtc->config->pixel_rate; int htotal = pipe_mode->crtc_htotal; - int hdisplay = crtc->config->pipe_src_w; + int width = drm_rect_width(&crtc->base.primary->state->src) >> 16; int cpp = fb->format->cpp[0]; int entries; - entries = intel_wm_method2(clock, htotal, - hdisplay, cpp, sr_latency_ns / 100); + entries = intel_wm_method2(pixel_rate, htotal, + width, cpp, sr_latency_ns / 100); entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE); srwm = I965_FIFO_SIZE - entries; if (srwm < 0) @@ -2294,7 +2287,7 @@ static void i965_update_wm(struct drm_i915_private *dev_priv) "self-refresh entries: %d, wm: %d\n", entries, srwm); - entries = intel_wm_method2(clock, htotal, + entries = intel_wm_method2(pixel_rate, htotal, crtc->base.cursor->state->crtc_w, 4, sr_latency_ns / 100); entries = DIV_ROUND_UP(entries, @@ -2358,7 +2351,7 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) int cwm, srwm = 1; int fifo_size; int planea_wm, planeb_wm; - struct intel_crtc *crtc, *enabled = NULL; + struct intel_crtc *crtc; if (IS_I945GM(dev_priv)) wm_info = &i945_wm_info; @@ -2373,8 +2366,6 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A); crtc = intel_crtc_for_plane(dev_priv, PLANE_A); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *pipe_mode = - &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; int cpp; @@ -2384,10 +2375,9 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) else cpp = fb->format->cpp[0]; - planea_wm = intel_calculate_wm(pipe_mode->crtc_clock, + planea_wm = intel_calculate_wm(crtc->config->pixel_rate, wm_info, fifo_size, cpp, pessimal_latency_ns); - enabled = crtc; } else { planea_wm = fifo_size - wm_info->guard_size; if (planea_wm > (long)wm_info->max_wm) @@ -2403,8 +2393,6 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B); crtc = intel_crtc_for_plane(dev_priv, PLANE_B); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *pipe_mode = - &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; int cpp; @@ -2414,13 +2402,9 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) else cpp = fb->format->cpp[0]; - planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock, + planeb_wm = intel_calculate_wm(crtc->config->pixel_rate, wm_info, fifo_size, cpp, pessimal_latency_ns); - if (enabled == NULL) - enabled = crtc; - else - enabled = NULL; } else { planeb_wm = fifo_size - wm_info->guard_size; if (planeb_wm > (long)wm_info->max_wm) @@ -2430,14 +2414,15 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) drm_dbg_kms(&dev_priv->drm, "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); - if (IS_I915GM(dev_priv) && enabled) { + crtc = single_enabled_crtc(dev_priv); + if (IS_I915GM(dev_priv) && crtc) { struct drm_i915_gem_object *obj; - obj = intel_fb_obj(enabled->base.primary->state->fb); + obj = intel_fb_obj(crtc->base.primary->state->fb); /* self-refresh seems busted with untiled */ if (!i915_gem_object_is_tiled(obj)) - enabled = NULL; + crtc = NULL; } /* @@ -2449,16 +2434,16 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) intel_set_memory_cxsr(dev_priv, false); /* Calc sr entries for one plane configs */ - if (HAS_FW_BLC(dev_priv) && enabled) { + if (HAS_FW_BLC(dev_priv) && crtc) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 6000; const struct drm_display_mode *pipe_mode = - &enabled->config->hw.pipe_mode; + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = - enabled->base.primary->state->fb; - int clock = pipe_mode->crtc_clock; + crtc->base.primary->state->fb; + int pixel_rate = crtc->config->pixel_rate; int htotal = pipe_mode->crtc_htotal; - int hdisplay = enabled->config->pipe_src_w; + int width = drm_rect_width(&crtc->base.primary->state->src) >> 16; int cpp; int entries; @@ -2467,7 +2452,7 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) else cpp = fb->format->cpp[0]; - entries = intel_wm_method2(clock, htotal, hdisplay, cpp, + entries = intel_wm_method2(pixel_rate, htotal, width, cpp, sr_latency_ns / 100); entries = DIV_ROUND_UP(entries, wm_info->cacheline_size); drm_dbg_kms(&dev_priv->drm, @@ -2497,14 +2482,13 @@ static void i9xx_update_wm(struct drm_i915_private *dev_priv) intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo); intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi); - if (enabled) + if (crtc) intel_set_memory_cxsr(dev_priv, true); } static void i845_update_wm(struct drm_i915_private *dev_priv) { struct intel_crtc *crtc; - const struct drm_display_mode *pipe_mode; u32 fwater_lo; int planea_wm; @@ -2512,8 +2496,7 @@ static void i845_update_wm(struct drm_i915_private *dev_priv) if (crtc == NULL) return; - pipe_mode = &crtc->config->hw.pipe_mode; - planea_wm = intel_calculate_wm(pipe_mode->crtc_clock, + planea_wm = intel_calculate_wm(crtc->config->pixel_rate, &i845_wm_info, i845_get_fifo_size(dev_priv, PLANE_A), 4, pessimal_latency_ns); @@ -2604,7 +2587,7 @@ static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state, method2 = ilk_wm_method2(crtc_state->pixel_rate, crtc_state->hw.pipe_mode.crtc_htotal, - drm_rect_width(&plane_state->uapi.dst), + drm_rect_width(&plane_state->uapi.src) >> 16, cpp, mem_value); return min(method1, method2); @@ -2632,7 +2615,7 @@ static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state, method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); method2 = ilk_wm_method2(crtc_state->pixel_rate, crtc_state->hw.pipe_mode.crtc_htotal, - drm_rect_width(&plane_state->uapi.dst), + drm_rect_width(&plane_state->uapi.src) >> 16, cpp, mem_value); return min(method1, method2); } @@ -2657,7 +2640,7 @@ static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state, return ilk_wm_method2(crtc_state->pixel_rate, crtc_state->hw.pipe_mode.crtc_htotal, - drm_rect_width(&plane_state->uapi.dst), + drm_rect_width(&plane_state->uapi.src) >> 16, cpp, mem_value); } @@ -2673,7 +2656,7 @@ static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, cpp = plane_state->hw.fb->format->cpp[0]; - return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst), + return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16, cpp); } @@ -2888,9 +2871,8 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, /* read the first set of memory latencies[0:3] */ val = 0; /* data0 to be programmed to 0 for first set */ - ret = sandybridge_pcode_read(dev_priv, - GEN9_PCODE_READ_MEM_LATENCY, - &val, NULL); + ret = snb_pcode_read(dev_priv, GEN9_PCODE_READ_MEM_LATENCY, + &val, NULL); if (ret) { drm_err(&dev_priv->drm, @@ -2908,9 +2890,8 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, /* read the second set of memory latencies[4:7] */ val = 1; /* data0 to be programmed to 1 for second set */ - ret = sandybridge_pcode_read(dev_priv, - GEN9_PCODE_READ_MEM_LATENCY, - &val, NULL); + ret = snb_pcode_read(dev_priv, GEN9_PCODE_READ_MEM_LATENCY, + &val, NULL); if (ret) { drm_err(&dev_priv->drm, "SKL Mailbox read error = %d\n", ret); @@ -2966,27 +2947,27 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv, } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD); - wm[0] = (sskpd >> 56) & 0xFF; + wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd); if (wm[0] == 0) - wm[0] = sskpd & 0xF; - wm[1] = (sskpd >> 4) & 0xFF; - wm[2] = (sskpd >> 12) & 0xFF; - wm[3] = (sskpd >> 20) & 0x1FF; - wm[4] = (sskpd >> 32) & 0x1FF; + wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd); + wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd); + wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd); + wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd); + wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd); } else if (DISPLAY_VER(dev_priv) >= 6) { u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD); - wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK; - wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK; - wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK; - wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK; + wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd); + wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd); + wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd); + wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd); } else if (DISPLAY_VER(dev_priv) >= 5) { u32 mltr = intel_uncore_read(uncore, MLTR_ILK); /* ILK primary LP0 latency is 700 ns */ wm[0] = 7; - wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK; - wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK; + wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr); + wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr); } else { MISSING_CASE(INTEL_DEVID(dev_priv)); } @@ -3199,12 +3180,8 @@ static int ilk_compute_pipe_wm(struct intel_atomic_state *state, } pipe_wm->pipe_enabled = crtc_state->hw.active; - if (sprstate) { - pipe_wm->sprites_enabled = sprstate->uapi.visible; - pipe_wm->sprites_scaled = sprstate->uapi.visible && - (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 || - drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16); - } + pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0); + pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0); usable_level = max_level; @@ -3433,29 +3410,28 @@ static void ilk_compute_wm_results(struct drm_i915_private *dev_priv, * disabled. Doing otherwise could cause underruns. */ results->wm_lp[wm_lp - 1] = - (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) | - (r->pri_val << WM1_LP_SR_SHIFT) | - r->cur_val; + WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) | + WM_LP_PRIMARY(r->pri_val) | + WM_LP_CURSOR(r->cur_val); if (r->enable) - results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN; + results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE; if (DISPLAY_VER(dev_priv) >= 8) - results->wm_lp[wm_lp - 1] |= - r->fbc_val << WM1_LP_FBC_SHIFT_BDW; + results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val); else - results->wm_lp[wm_lp - 1] |= - r->fbc_val << WM1_LP_FBC_SHIFT; + results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val); + + results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val); /* - * Always set WM1S_LP_EN when spr_val != 0, even if the + * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the * level is disabled. Doing otherwise could cause underruns. */ if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) { drm_WARN_ON(&dev_priv->drm, wm_lp != 1); - results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val; - } else - results->wm_lp_spr[wm_lp - 1] = r->spr_val; + results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE; + } } /* LP0 register values */ @@ -3468,9 +3444,9 @@ static void ilk_compute_wm_results(struct drm_i915_private *dev_priv, continue; results->wm_pipe[pipe] = - (r->pri_val << WM0_PIPE_PLANE_SHIFT) | - (r->spr_val << WM0_PIPE_SPRITE_SHIFT) | - r->cur_val; + WM0_PIPE_PRIMARY(r->pri_val) | + WM0_PIPE_SPRITE(r->spr_val) | + WM0_PIPE_CURSOR(r->cur_val); } } @@ -3562,24 +3538,24 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, struct ilk_wm_values *previous = &dev_priv->wm.hw; bool changed = false; - if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) { - previous->wm_lp[2] &= ~WM1_LP_SR_EN; + if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) { + previous->wm_lp[2] &= ~WM_LP_ENABLE; intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]); changed = true; } - if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) { - previous->wm_lp[1] &= ~WM1_LP_SR_EN; + if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) { + previous->wm_lp[1] &= ~WM_LP_ENABLE; intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]); changed = true; } - if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) { - previous->wm_lp[0] &= ~WM1_LP_SR_EN; + if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) { + previous->wm_lp[0] &= ~WM_LP_ENABLE; intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]); changed = true; } /* - * Don't touch WM1S_LP_EN here. + * Don't touch WM_LP_SPRITE_ENABLE here. * Doing so could cause underruns. */ @@ -3700,9 +3676,9 @@ skl_setup_sagv_block_time(struct drm_i915_private *dev_priv) u32 val = 0; int ret; - ret = sandybridge_pcode_read(dev_priv, - GEN12_PCODE_READ_SAGV_BLOCK_TIME_US, - &val, NULL); + ret = snb_pcode_read(dev_priv, + GEN12_PCODE_READ_SAGV_BLOCK_TIME_US, + &val, NULL); if (!ret) { dev_priv->sagv_block_time_us = val; return; @@ -3749,8 +3725,8 @@ intel_enable_sagv(struct drm_i915_private *dev_priv) return 0; drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n"); - ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL, - GEN9_SAGV_ENABLE); + ret = snb_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL, + GEN9_SAGV_ENABLE); /* We don't need to wait for SAGV when enabling */ @@ -3805,50 +3781,89 @@ intel_disable_sagv(struct drm_i915_private *dev_priv) return 0; } -void intel_sagv_pre_plane_update(struct intel_atomic_state *state) +static void skl_sagv_pre_plane_update(struct intel_atomic_state *state) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_bw_state *new_bw_state; - const struct intel_bw_state *old_bw_state; - u32 new_mask = 0; + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); - /* - * Just return if we can't control SAGV or don't have it. - * This is different from situation when we have SAGV but just can't - * afford it due to DBuf limitation - in case if SAGV is completely - * disabled in a BIOS, we are not even allowed to send a PCode request, - * as it will throw an error. So have to check it here. - */ - if (!intel_has_sagv(dev_priv)) + if (!new_bw_state) + return; + + if (!intel_can_enable_sagv(i915, new_bw_state)) + intel_disable_sagv(i915); +} + +static void skl_sagv_post_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + + if (!new_bw_state) return; - new_bw_state = intel_atomic_get_new_bw_state(state); + if (intel_can_enable_sagv(i915, new_bw_state)) + intel_enable_sagv(i915); +} + +static void icl_sagv_pre_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_bw_state *old_bw_state = + intel_atomic_get_old_bw_state(state); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + u16 old_mask, new_mask; + if (!new_bw_state) return; - if (DISPLAY_VER(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) { - intel_disable_sagv(dev_priv); + old_mask = old_bw_state->qgv_points_mask; + new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; + + if (old_mask == new_mask) return; - } - old_bw_state = intel_atomic_get_old_bw_state(state); + WARN_ON(!new_bw_state->base.changed); + + drm_dbg_kms(&dev_priv->drm, "Restricting QGV points: 0x%x -> 0x%x\n", + old_mask, new_mask); + /* - * Nothing to mask + * Restrict required qgv points before updating the configuration. + * According to BSpec we can't mask and unmask qgv points at the same + * time. Also masking should be done before updating the configuration + * and unmasking afterwards. */ - if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask) + icl_pcode_restrict_qgv_points(dev_priv, new_mask); +} + +static void icl_sagv_post_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_bw_state *old_bw_state = + intel_atomic_get_old_bw_state(state); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + u16 old_mask, new_mask; + + if (!new_bw_state) return; - new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; + old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; + new_mask = new_bw_state->qgv_points_mask; - /* - * If new mask is zero - means there is nothing to mask, - * we can only unmask, which should be done in unmask. - */ - if (!new_mask) + if (old_mask == new_mask) return; + WARN_ON(!new_bw_state->base.changed); + + drm_dbg_kms(&dev_priv->drm, "Relaxing QGV points: 0x%x -> 0x%x\n", + old_mask, new_mask); + /* - * Restrict required qgv points before updating the configuration. + * Allow required qgv points after updating the configuration. * According to BSpec we can't mask and unmask qgv points at the same * time. Also masking should be done before updating the configuration * and unmasking afterwards. @@ -3856,12 +3871,9 @@ void intel_sagv_pre_plane_update(struct intel_atomic_state *state) icl_pcode_restrict_qgv_points(dev_priv, new_mask); } -void intel_sagv_post_plane_update(struct intel_atomic_state *state) +void intel_sagv_pre_plane_update(struct intel_atomic_state *state) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_bw_state *new_bw_state; - const struct intel_bw_state *old_bw_state; - u32 new_mask = 0; + struct drm_i915_private *i915 = to_i915(state->base.dev); /* * Just return if we can't control SAGV or don't have it. @@ -3870,34 +3882,33 @@ void intel_sagv_post_plane_update(struct intel_atomic_state *state) * disabled in a BIOS, we are not even allowed to send a PCode request, * as it will throw an error. So have to check it here. */ - if (!intel_has_sagv(dev_priv)) + if (!intel_has_sagv(i915)) return; - new_bw_state = intel_atomic_get_new_bw_state(state); - if (!new_bw_state) - return; + if (DISPLAY_VER(i915) >= 11) + icl_sagv_pre_plane_update(state); + else + skl_sagv_pre_plane_update(state); +} - if (DISPLAY_VER(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) { - intel_enable_sagv(dev_priv); - return; - } +void intel_sagv_post_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); - old_bw_state = intel_atomic_get_old_bw_state(state); /* - * Nothing to unmask + * Just return if we can't control SAGV or don't have it. + * This is different from situation when we have SAGV but just can't + * afford it due to DBuf limitation - in case if SAGV is completely + * disabled in a BIOS, we are not even allowed to send a PCode request, + * as it will throw an error. So have to check it here. */ - if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask) + if (!intel_has_sagv(i915)) return; - new_mask = new_bw_state->qgv_points_mask; - - /* - * Allow required qgv points after updating the configuration. - * According to BSpec we can't mask and unmask qgv points at the same - * time. Also masking should be done before updating the configuration - * and unmasking afterwards. - */ - icl_pcode_restrict_qgv_points(dev_priv, new_mask); + if (DISPLAY_VER(i915) >= 11) + icl_sagv_post_plane_update(state); + else + skl_sagv_post_plane_update(state); } static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) @@ -4058,6 +4069,15 @@ static int intel_compute_sagv_mask(struct intel_atomic_state *state) return 0; } +static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry, + u16 start, u16 end) +{ + entry->start = start; + entry->end = end; + + return end; +} + static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv) { return INTEL_INFO(dev_priv)->dbuf.size / @@ -4196,8 +4216,7 @@ skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) int ret; if (new_dbuf_state->weight[pipe] == 0) { - new_dbuf_state->ddb[pipe].start = 0; - new_dbuf_state->ddb[pipe].end = 0; + skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0); goto out; } @@ -4213,8 +4232,10 @@ skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) start = ddb_range_size * weight_start / weight_total; end = ddb_range_size * weight_end / weight_total; - new_dbuf_state->ddb[pipe].start = ddb_slices.start - mbus_offset + start; - new_dbuf_state->ddb[pipe].end = ddb_slices.start - mbus_offset + end; + skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], + ddb_slices.start - mbus_offset + start, + ddb_slices.start - mbus_offset + end); + out: if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] && skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe], @@ -4252,7 +4273,9 @@ static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state, u64 modifier, unsigned int rotation, u32 plane_pixel_rate, struct skl_wm_params *wp, int color_plane); + static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane, int level, unsigned int latency, const struct skl_wm_params *wp, @@ -4263,6 +4286,7 @@ static unsigned int skl_cursor_allocation(const struct intel_crtc_state *crtc_state, int num_active) { + struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor); struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); int level, max_level = ilk_wm_max_level(dev_priv); struct skl_wm_level wm = {}; @@ -4279,7 +4303,7 @@ skl_cursor_allocation(const struct intel_crtc_state *crtc_state, for (level = 0; level <= max_level; level++) { unsigned int latency = dev_priv->wm.skl_latency[level]; - skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm); + skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm); if (wm.min_ddb_alloc == U16_MAX) break; @@ -4289,14 +4313,13 @@ skl_cursor_allocation(const struct intel_crtc_state *crtc_state, return max(num_active == 1 ? 32 : 8, min_ddb_alloc); } -static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv, - struct skl_ddb_entry *entry, u32 reg) +static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg) { - entry->start = reg & DDB_ENTRY_MASK; - entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK; - + skl_ddb_entry_init(entry, + REG_FIELD_GET(PLANE_BUF_START_MASK, reg), + REG_FIELD_GET(PLANE_BUF_END_MASK, reg)); if (entry->end) - entry->end += 1; + entry->end++; } static void @@ -4312,7 +4335,7 @@ skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv, /* Cursor doesn't support NV12/planar, so no extra calculation needed */ if (plane_id == PLANE_CURSOR) { val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe)); - skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); + skl_ddb_entry_init_from_hw(ddb_y, val); return; } @@ -4320,13 +4343,13 @@ skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv, /* No DDB allocated for disabled planes */ if (val & PLANE_CTL_ENABLE) - fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK, + fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK_SKL, val & PLANE_CTL_ORDER_RGBX, val & PLANE_CTL_ALPHA_MASK); if (DISPLAY_VER(dev_priv) >= 11) { val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id)); - skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); + skl_ddb_entry_init_from_hw(ddb_y, val); } else { val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id)); val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id)); @@ -4335,8 +4358,8 @@ skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv, drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc))) swap(val, val2); - skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val); - skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2); + skl_ddb_entry_init_from_hw(ddb_y, val); + skl_ddb_entry_init_from_hw(ddb_uv, val2); } } @@ -4364,55 +4387,6 @@ void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc, intel_display_power_put(dev_priv, power_domain, wakeref); } -/* - * Determines the downscale amount of a plane for the purposes of watermark calculations. - * The bspec defines downscale amount as: - * - * """ - * Horizontal down scale amount = maximum[1, Horizontal source size / - * Horizontal destination size] - * Vertical down scale amount = maximum[1, Vertical source size / - * Vertical destination size] - * Total down scale amount = Horizontal down scale amount * - * Vertical down scale amount - * """ - * - * Return value is provided in 16.16 fixed point form to retain fractional part. - * Caller should take care of dividing & rounding off the value. - */ -static uint_fixed_16_16_t -skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) -{ - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - u32 src_w, src_h, dst_w, dst_h; - uint_fixed_16_16_t fp_w_ratio, fp_h_ratio; - uint_fixed_16_16_t downscale_h, downscale_w; - - if (drm_WARN_ON(&dev_priv->drm, - !intel_wm_plane_visible(crtc_state, plane_state))) - return u32_to_fixed16(0); - - /* - * Src coordinates are already rotated by 270 degrees for - * the 90/270 degree plane rotation cases (to match the - * GTT mapping), hence no need to account for rotation here. - * - * n.b., src is 16.16 fixed point, dst is whole integer. - */ - src_w = drm_rect_width(&plane_state->uapi.src) >> 16; - src_h = drm_rect_height(&plane_state->uapi.src) >> 16; - dst_w = drm_rect_width(&plane_state->uapi.dst); - dst_h = drm_rect_height(&plane_state->uapi.dst); - - fp_w_ratio = div_fixed16(src_w, dst_w); - fp_h_ratio = div_fixed16(src_h, dst_h); - downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1)); - downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1)); - - return mul_fixed16(downscale_w, downscale_h); -} - struct dbuf_slice_conf_entry { u8 active_pipes; u8 dbuf_mask[I915_MAX_PIPES]; @@ -4939,6 +4913,28 @@ static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0; } +static bool +use_min_ddb(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + return DISPLAY_VER(i915) >= 13 && + crtc_state->uapi.async_flip && + plane->async_flip; +} + +static bool +use_minimal_wm0_only(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + return DISPLAY_VER(i915) >= 13 && + crtc_state->uapi.async_flip && + plane->async_flip; +} + static u64 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, @@ -4946,10 +4942,7 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); const struct drm_framebuffer *fb = plane_state->hw.fb; - u32 data_rate; - u32 width = 0, height = 0; - uint_fixed_16_16_t down_scale_amount; - u64 rate; + int width, height; if (!plane_state->uapi.visible) return 0; @@ -4957,6 +4950,14 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, if (plane->id == PLANE_CURSOR) return 0; + /* + * We calculate extra ddb based on ratio plane rate/total data rate + * in case, in some cases we should not allocate extra ddb for the plane, + * so do not count its data rate, if this is the case. + */ + if (use_min_ddb(crtc_state, plane)) + return 0; + if (color_plane == 1 && !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier)) return 0; @@ -4975,14 +4976,7 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, height /= 2; } - data_rate = width * height; - - down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state); - - rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount); - - rate *= fb->format->cpp[color_plane]; - return rate; + return width * height * fb->format->cpp[color_plane]; } static u64 @@ -5139,9 +5133,34 @@ static bool icl_need_wm1_wa(struct drm_i915_private *i915, (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR); } +struct skl_plane_ddb_iter { + u64 data_rate; + u16 total[I915_MAX_PLANES]; + u16 uv_total[I915_MAX_PLANES]; + u16 start, size; +}; + +static u16 +skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter, + const struct skl_wm_level *wm, + u64 data_rate) +{ + u16 extra = 0; + + if (data_rate) { + extra = min_t(u16, iter->size, + DIV64_U64_ROUND_UP(iter->size * data_rate, + iter->data_rate)); + iter->size -= extra; + iter->data_rate -= data_rate; + } + + return wm->min_ddb_alloc + extra; +} + static int -skl_allocate_plane_ddb(struct intel_atomic_state *state, - struct intel_crtc *crtc) +skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state, + struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_crtc_state *crtc_state = @@ -5150,10 +5169,7 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, intel_atomic_get_new_dbuf_state(state); const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe]; int num_active = hweight8(dbuf_state->active_pipes); - u16 alloc_size, start = 0; - u16 total[I915_MAX_PLANES] = {}; - u16 uv_total[I915_MAX_PLANES] = {}; - u64 total_data_rate; + struct skl_plane_ddb_iter iter = {}; enum plane_id plane_id; u32 blocks; int level; @@ -5166,25 +5182,19 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, return 0; if (DISPLAY_VER(dev_priv) >= 11) - total_data_rate = - icl_get_total_relative_data_rate(state, crtc); + iter.data_rate = icl_get_total_relative_data_rate(state, crtc); else - total_data_rate = - skl_get_total_relative_data_rate(state, crtc); + iter.data_rate = skl_get_total_relative_data_rate(state, crtc); - alloc_size = skl_ddb_entry_size(alloc); - if (alloc_size == 0) + iter.size = skl_ddb_entry_size(alloc); + if (iter.size == 0) return 0; /* Allocate fixed number of blocks for cursor. */ - total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active); - alloc_size -= total[PLANE_CURSOR]; - crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start = - alloc->end - total[PLANE_CURSOR]; - crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end; - - if (total_data_rate == 0) - return 0; + iter.total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active); + iter.size -= iter.total[PLANE_CURSOR]; + skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR], + alloc->end - iter.total[PLANE_CURSOR], alloc->end); /* * Find the highest watermark level for which we can satisfy the block @@ -5197,7 +5207,7 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, &crtc_state->wm.skl.optimal.planes[plane_id]; if (plane_id == PLANE_CURSOR) { - if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) { + if (wm->wm[level].min_ddb_alloc > iter.total[PLANE_CURSOR]) { drm_WARN_ON(&dev_priv->drm, wm->wm[level].min_ddb_alloc != U16_MAX); blocks = U32_MAX; @@ -5210,8 +5220,8 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, blocks += wm->uv_wm[level].min_ddb_alloc; } - if (blocks <= alloc_size) { - alloc_size -= blocks; + if (blocks <= iter.size) { + iter.size -= blocks; break; } } @@ -5220,10 +5230,14 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, drm_dbg_kms(&dev_priv->drm, "Requested display configuration exceeds system DDB limitations"); drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n", - blocks, alloc_size); + blocks, iter.size); return -EINVAL; } + /* avoid the WARN later when we don't allocate any extra DDB */ + if (iter.data_rate == 0) + iter.size = 0; + /* * Grant each plane the blocks it requires at the highest achievable * watermark level, plus an extra share of the leftover blocks @@ -5232,42 +5246,22 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, for_each_plane_id_on_crtc(crtc, plane_id) { const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; - u64 rate; - u16 extra; if (plane_id == PLANE_CURSOR) continue; - /* - * We've accounted for all active planes; remaining planes are - * all disabled. - */ - if (total_data_rate == 0) - break; - - rate = crtc_state->plane_data_rate[plane_id]; - extra = min_t(u16, alloc_size, - DIV64_U64_ROUND_UP(alloc_size * rate, - total_data_rate)); - total[plane_id] = wm->wm[level].min_ddb_alloc + extra; - alloc_size -= extra; - total_data_rate -= rate; - - if (total_data_rate == 0) - break; + iter.total[plane_id] = + skl_allocate_plane_ddb(&iter, &wm->wm[level], + crtc_state->plane_data_rate[plane_id]); - rate = crtc_state->uv_plane_data_rate[plane_id]; - extra = min_t(u16, alloc_size, - DIV64_U64_ROUND_UP(alloc_size * rate, - total_data_rate)); - uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra; - alloc_size -= extra; - total_data_rate -= rate; + iter.uv_total[plane_id] = + skl_allocate_plane_ddb(&iter, &wm->uv_wm[level], + crtc_state->uv_plane_data_rate[plane_id]); } - drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0); + drm_WARN_ON(&dev_priv->drm, iter.size != 0 || iter.data_rate != 0); /* Set the actual DDB start/end points for each plane */ - start = alloc->start; + iter.start = alloc->start; for_each_plane_id_on_crtc(crtc, plane_id) { struct skl_ddb_entry *plane_alloc = &crtc_state->wm.skl.plane_ddb_y[plane_id]; @@ -5279,20 +5273,16 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, /* Gen11+ uses a separate plane for UV watermarks */ drm_WARN_ON(&dev_priv->drm, - DISPLAY_VER(dev_priv) >= 11 && uv_total[plane_id]); + DISPLAY_VER(dev_priv) >= 11 && iter.uv_total[plane_id]); /* Leave disabled planes at (0,0) */ - if (total[plane_id]) { - plane_alloc->start = start; - start += total[plane_id]; - plane_alloc->end = start; - } + if (iter.total[plane_id]) + iter.start = skl_ddb_entry_init(plane_alloc, iter.start, + iter.start + iter.total[plane_id]); - if (uv_total[plane_id]) { - uv_plane_alloc->start = start; - start += uv_total[plane_id]; - uv_plane_alloc->end = start; - } + if (iter.uv_total[plane_id]) + iter.start = skl_ddb_entry_init(uv_plane_alloc, iter.start, + iter.start + iter.uv_total[plane_id]); } /* @@ -5307,7 +5297,8 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, &crtc_state->wm.skl.optimal.planes[plane_id]; skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level], - total[plane_id], uv_total[plane_id]); + iter.total[plane_id], + iter.uv_total[plane_id]); if (icl_need_wm1_wa(dev_priv, plane_id) && level == 1 && wm->wm[0].enable) { @@ -5326,9 +5317,9 @@ skl_allocate_plane_ddb(struct intel_atomic_state *state, struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; - skl_check_wm_level(&wm->trans_wm, total[plane_id]); - skl_check_wm_level(&wm->sagv.wm0, total[plane_id]); - skl_check_wm_level(&wm->sagv.trans_wm, total[plane_id]); + skl_check_wm_level(&wm->trans_wm, iter.total[plane_id]); + skl_check_wm_level(&wm->sagv.wm0, iter.total[plane_id]); + skl_check_wm_level(&wm->sagv.trans_wm, iter.total[plane_id]); } return 0; @@ -5531,6 +5522,7 @@ static int skl_wm_max_lines(struct drm_i915_private *dev_priv) } static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane, int level, unsigned int latency, const struct skl_wm_params *wp, @@ -5542,7 +5534,8 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, uint_fixed_16_16_t selected_result; u32 blocks, lines, min_ddb_alloc = 0; - if (latency == 0) { + if (latency == 0 || + (use_minimal_wm0_only(crtc_state, plane) && level > 0)) { /* reject it */ result->min_ddb_alloc = U16_MAX; return; @@ -5658,6 +5651,7 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, static void skl_compute_wm_levels(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane, const struct skl_wm_params *wm_params, struct skl_wm_level *levels) { @@ -5669,7 +5663,7 @@ skl_compute_wm_levels(const struct intel_crtc_state *crtc_state, struct skl_wm_level *result = &levels[level]; unsigned int latency = dev_priv->wm.skl_latency[level]; - skl_compute_plane_wm(crtc_state, level, latency, + skl_compute_plane_wm(crtc_state, plane, level, latency, wm_params, result_prev, result); result_prev = result; @@ -5677,6 +5671,7 @@ skl_compute_wm_levels(const struct intel_crtc_state *crtc_state, } static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane, const struct skl_wm_params *wm_params, struct skl_plane_wm *plane_wm) { @@ -5685,7 +5680,7 @@ static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state, struct skl_wm_level *levels = plane_wm->wm; unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us; - skl_compute_plane_wm(crtc_state, 0, latency, + skl_compute_plane_wm(crtc_state, plane, 0, latency, wm_params, &levels[0], sagv_wm); } @@ -5755,11 +5750,11 @@ static void skl_compute_transition_wm(struct drm_i915_private *dev_priv, static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, - enum plane_id plane_id, int color_plane) + struct intel_plane *plane, int color_plane) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id]; struct skl_wm_params wm_params; int ret; @@ -5768,13 +5763,13 @@ static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, if (ret) return ret; - skl_compute_wm_levels(crtc_state, &wm_params, wm->wm); + skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm); skl_compute_transition_wm(dev_priv, &wm->trans_wm, &wm->wm[0], &wm_params); if (DISPLAY_VER(dev_priv) >= 12) { - tgl_compute_sagv_wm(crtc_state, &wm_params, wm); + tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm); skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm, &wm->sagv.wm0, &wm_params); @@ -5785,9 +5780,9 @@ static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, - enum plane_id plane_id) + struct intel_plane *plane) { - struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id]; struct skl_wm_params wm_params; int ret; @@ -5799,7 +5794,7 @@ static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, if (ret) return ret; - skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm); + skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm); return 0; } @@ -5819,13 +5814,13 @@ static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, return 0; ret = skl_build_plane_wm_single(crtc_state, plane_state, - plane_id, 0); + plane, 0); if (ret) return ret; if (fb->format->is_yuv && fb->format->num_planes > 1) { ret = skl_build_plane_wm_uv(crtc_state, plane_state, - plane_id); + plane); if (ret) return ret; } @@ -5850,7 +5845,6 @@ static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, if (plane_state->planar_linked_plane) { const struct drm_framebuffer *fb = plane_state->hw.fb; - enum plane_id y_plane_id = plane_state->planar_linked_plane->id; drm_WARN_ON(&dev_priv->drm, !intel_wm_plane_visible(crtc_state, plane_state)); @@ -5858,17 +5852,17 @@ static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, fb->format->num_planes == 1); ret = skl_build_plane_wm_single(crtc_state, plane_state, - y_plane_id, 0); + plane_state->planar_linked_plane, 0); if (ret) return ret; ret = skl_build_plane_wm_single(crtc_state, plane_state, - plane_id, 1); + plane, 1); if (ret) return ret; } else if (intel_wm_plane_visible(crtc_state, plane_state)) { ret = skl_build_plane_wm_single(crtc_state, plane_state, - plane_id, 0); + plane, 0); if (ret) return ret; } @@ -5914,7 +5908,8 @@ static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, { if (entry->end) intel_de_write_fw(dev_priv, reg, - (entry->end - 1) << 16 | entry->start); + PLANE_BUF_END(entry->end - 1) | + PLANE_BUF_START(entry->start)); else intel_de_write_fw(dev_priv, reg, 0); } @@ -6214,7 +6209,7 @@ skl_compute_ddb(struct intel_atomic_state *state) for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - ret = skl_allocate_plane_ddb(state, crtc); + ret = skl_crtc_allocate_plane_ddb(state, crtc); if (ret) return ret; @@ -6791,9 +6786,9 @@ static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) * multiple pipes are active. */ active->wm[0].enable = true; - active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT; - active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT; - active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK; + active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp); + active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp); + active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp); } else { int level, max_level = ilk_wm_max_level(dev_priv); @@ -7217,12 +7212,12 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv) */ static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) { - intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN); - intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN); - intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN); + intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM_LP_ENABLE); + intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM_LP_ENABLE); + intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM_LP_ENABLE); /* - * Don't touch WM1S_LP_EN here. + * Don't touch WM_LP_SPRITE_ENABLE here. * Doing so could cause underruns. */ } @@ -7317,7 +7312,7 @@ static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) for_each_pipe(dev_priv, pipe) { intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe), intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) | - DISPPLANE_TRICKLE_FEED_DISABLE); + DISP_TRICKLE_FEED_DISABLE); intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe))); intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe)); @@ -7425,7 +7420,7 @@ static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) u32 tmp; tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD); - if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) + if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12) drm_dbg_kms(&dev_priv->drm, "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n", tmp); @@ -7548,8 +7543,8 @@ static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv, static void icl_init_clock_gating(struct drm_i915_private *dev_priv) { /* Wa_1409120013:icl,ehl */ - intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, - DPFC_CHICKEN_COMP_DUMMY_PIXEL); + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), + DPFC_CHICKEN_COMP_DUMMY_PIXEL); /*Wa_14010594013:icl, ehl */ intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, @@ -7561,7 +7556,7 @@ static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv) /* Wa_1409120013:tgl,rkl,adl-s,dg1,dg2 */ if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) || IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv) || IS_DG2(dev_priv)) - intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), DPFC_CHICKEN_COMP_DUMMY_PIXEL); /* Wa_1409825376:tgl (pre-prod)*/ @@ -7646,8 +7641,9 @@ static void cfl_init_clock_gating(struct drm_i915_private *dev_priv) * WaFbcNukeOnHostModify:cfl * Display WA #0873: cfl */ - intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | - DPFC_NUKE_ON_ANY_MODIFICATION); + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), + intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) | + DPFC_NUKE_ON_ANY_MODIFICATION); } static void kbl_init_clock_gating(struct drm_i915_private *dev_priv) @@ -7679,8 +7675,9 @@ static void kbl_init_clock_gating(struct drm_i915_private *dev_priv) * WaFbcNukeOnHostModify:kbl * Display WA #0873: kbl */ - intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | - DPFC_NUKE_ON_ANY_MODIFICATION); + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), + intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) | + DPFC_NUKE_ON_ANY_MODIFICATION); } static void skl_init_clock_gating(struct drm_i915_private *dev_priv) @@ -7706,15 +7703,17 @@ static void skl_init_clock_gating(struct drm_i915_private *dev_priv) * WaFbcNukeOnHostModify:skl * Display WA #0873: skl */ - intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | - DPFC_NUKE_ON_ANY_MODIFICATION); + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), + intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) | + DPFC_NUKE_ON_ANY_MODIFICATION); /* * WaFbcHighMemBwCorruptionAvoidance:skl * Display WA #0883: skl */ - intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) | - DPFC_DISABLE_DUMMY0); + intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), + intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) | + DPFC_DISABLE_DUMMY0); } static void bdw_init_clock_gating(struct drm_i915_private *dev_priv) @@ -7746,7 +7745,7 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv) intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) & ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); - intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL, + intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE), _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); /* WaDisableSDEUnitClockGating:bdw */ @@ -7887,7 +7886,7 @@ static void chv_init_clock_gating(struct drm_i915_private *dev_priv) ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); /* WaDisableSemaphoreAndSyncFlipWait:chv */ - intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL, + intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE), _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); /* WaDisableCSUnitClockGating:chv */ @@ -7960,10 +7959,12 @@ static void gen3_init_clock_gating(struct drm_i915_private *dev_priv) intel_uncore_write(&dev_priv->uncore, D_STATE, dstate); if (IS_PINEVIEW(dev_priv)) - intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); + intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE), + _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); /* IIR "flip pending" means done if this bit is set */ - intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); + intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE), + _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); /* interrupts should cause a wake up from C3 */ intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN)); diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h index d2243653a893..51705151b842 100644 --- a/drivers/gpu/drm/i915/intel_pm.h +++ b/drivers/gpu/drm/i915/intel_pm.h @@ -12,7 +12,6 @@ #include "display/intel_global_state.h" #include "i915_drv.h" -#include "i915_reg.h" struct drm_device; struct drm_i915_private; diff --git a/drivers/gpu/drm/i915/intel_region_ttm.c b/drivers/gpu/drm/i915/intel_region_ttm.c index f2b888c16958..737ef3f4ab54 100644 --- a/drivers/gpu/drm/i915/intel_region_ttm.c +++ b/drivers/gpu/drm/i915/intel_region_ttm.c @@ -87,6 +87,7 @@ int intel_region_ttm_init(struct intel_memory_region *mem) ret = i915_ttm_buddy_man_init(bdev, mem_type, false, resource_size(&mem->region), + mem->io_size, mem->min_page_size, PAGE_SIZE); if (ret) return ret; @@ -199,12 +200,25 @@ intel_region_ttm_resource_alloc(struct intel_memory_region *mem, struct ttm_resource *res; int ret; + if (flags & I915_BO_ALLOC_CONTIGUOUS) + place.flags |= TTM_PL_FLAG_CONTIGUOUS; + if (mem->io_size && mem->io_size < mem->total) { + if (flags & I915_BO_ALLOC_GPU_ONLY) { + place.flags |= TTM_PL_FLAG_TOPDOWN; + } else { + place.fpfn = 0; + place.lpfn = mem->io_size >> PAGE_SHIFT; + } + } + mock_bo.base.size = size; - place.flags = flags; + mock_bo.bdev = &mem->i915->bdev; ret = man->func->alloc(man, &mock_bo, &place, &res); if (ret == -ENOSPC) ret = -ENXIO; + if (!ret) + res->bo = NULL; /* Rather blow up, then some uaf */ return ret ? ERR_PTR(ret) : res; } @@ -219,6 +233,11 @@ void intel_region_ttm_resource_free(struct intel_memory_region *mem, struct ttm_resource *res) { struct ttm_resource_manager *man = mem->region_private; + struct ttm_buffer_object mock_bo = {}; + + mock_bo.base.size = res->num_pages << PAGE_SHIFT; + mock_bo.bdev = &mem->i915->bdev; + res->bo = &mock_bo; man->func->free(man, res); } diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 64c2708efc9e..6ed5786bcd29 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -77,7 +77,7 @@ track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) depot_stack_handle_t stack, *stacks; unsigned long flags; - if (!rpm->available) + if (rpm->no_wakeref_tracking) return -1; stack = __save_depot_stack(); diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.h b/drivers/gpu/drm/i915/intel_runtime_pm.h index 47a85fab4130..d9160e3ff4af 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.h +++ b/drivers/gpu/drm/i915/intel_runtime_pm.h @@ -51,6 +51,7 @@ struct intel_runtime_pm { bool available; bool suspended; bool irqs_enabled; + bool no_wakeref_tracking; #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) /* diff --git a/drivers/gpu/drm/i915/intel_sbi.c b/drivers/gpu/drm/i915/intel_sbi.c index 5ba8490a31e6..5c6e517c73f4 100644 --- a/drivers/gpu/drm/i915/intel_sbi.c +++ b/drivers/gpu/drm/i915/intel_sbi.c @@ -7,6 +7,7 @@ #include "i915_drv.h" #include "intel_sbi.h" +#include "i915_reg.h" /* SBI access */ static int intel_sbi_rw(struct drm_i915_private *i915, u16 reg, diff --git a/drivers/gpu/drm/i915/intel_step.c b/drivers/gpu/drm/i915/intel_step.c index a4b16b9e2e55..4fd69ecd1481 100644 --- a/drivers/gpu/drm/i915/intel_step.c +++ b/drivers/gpu/drm/i915/intel_step.c @@ -122,6 +122,15 @@ static const struct intel_step_info dg2_g11_revid_step_tbl[] = { [0x5] = { COMMON_GT_MEDIA_STEP(B1), .display_step = STEP_C0 }, }; +static const struct intel_step_info dg2_g12_revid_step_tbl[] = { + [0x0] = { COMMON_GT_MEDIA_STEP(A0), .display_step = STEP_C0 }, +}; + +static const struct intel_step_info adls_rpls_revids[] = { + [0x4] = { COMMON_GT_MEDIA_STEP(D0), .display_step = STEP_D0 }, + [0xC] = { COMMON_GT_MEDIA_STEP(D0), .display_step = STEP_C0 }, +}; + void intel_step_init(struct drm_i915_private *i915) { const struct intel_step_info *revids = NULL; @@ -135,12 +144,18 @@ void intel_step_init(struct drm_i915_private *i915) } else if (IS_DG2_G11(i915)) { revids = dg2_g11_revid_step_tbl; size = ARRAY_SIZE(dg2_g11_revid_step_tbl); + } else if (IS_DG2_G12(i915)) { + revids = dg2_g12_revid_step_tbl; + size = ARRAY_SIZE(dg2_g12_revid_step_tbl); } else if (IS_XEHPSDV(i915)) { revids = xehpsdv_revids; size = ARRAY_SIZE(xehpsdv_revids); } else if (IS_ALDERLAKE_P(i915)) { revids = adlp_revids; size = ARRAY_SIZE(adlp_revids); + } else if (IS_ADLS_RPLS(i915)) { + revids = adls_rpls_revids; + size = ARRAY_SIZE(adls_rpls_revids); } else if (IS_ALDERLAKE_S(i915)) { revids = adls_revids; size = ARRAY_SIZE(adls_revids); @@ -150,7 +165,7 @@ void intel_step_init(struct drm_i915_private *i915) } else if (IS_ROCKETLAKE(i915)) { revids = rkl_revids; size = ARRAY_SIZE(rkl_revids); - } else if (IS_TGL_U(i915) || IS_TGL_Y(i915)) { + } else if (IS_TGL_UY(i915)) { revids = tgl_uy_revids; size = ARRAY_SIZE(tgl_uy_revids); } else if (IS_TIGERLAKE(i915)) { diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 778da3179b3c..dd8fdd5863de 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -23,7 +23,8 @@ #include <linux/pm_runtime.h> -#include "gt/intel_lrc_reg.h" /* for shadow reg list */ +#include "gt/intel_engine_regs.h" +#include "gt/intel_gt_regs.h" #include "i915_drv.h" #include "i915_iosf_mbi.h" @@ -1495,7 +1496,7 @@ ilk_dummy_write(struct intel_uncore *uncore) /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up * the chip from rc6 before touching it for real. MI_MODE is masked, * hence harmless to write 0 into. */ - __raw_uncore_write32(uncore, MI_MODE, 0); + __raw_uncore_write32(uncore, RING_MI_MODE(RENDER_RING_BASE), 0); } static void @@ -2273,76 +2274,6 @@ void intel_uncore_fini_mmio(struct intel_uncore *uncore) } } -static const struct reg_whitelist { - i915_reg_t offset_ldw; - i915_reg_t offset_udw; - u8 min_graphics_ver; - u8 max_graphics_ver; - u8 size; -} reg_read_whitelist[] = { { - .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE), - .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE), - .min_graphics_ver = 4, - .max_graphics_ver = 12, - .size = 8 -} }; - -int i915_reg_read_ioctl(struct drm_device *dev, - void *data, struct drm_file *file) -{ - struct drm_i915_private *i915 = to_i915(dev); - struct intel_uncore *uncore = &i915->uncore; - struct drm_i915_reg_read *reg = data; - struct reg_whitelist const *entry; - intel_wakeref_t wakeref; - unsigned int flags; - int remain; - int ret = 0; - - entry = reg_read_whitelist; - remain = ARRAY_SIZE(reg_read_whitelist); - while (remain) { - u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw); - - GEM_BUG_ON(!is_power_of_2(entry->size)); - GEM_BUG_ON(entry->size > 8); - GEM_BUG_ON(entry_offset & (entry->size - 1)); - - if (IS_GRAPHICS_VER(i915, entry->min_graphics_ver, entry->max_graphics_ver) && - entry_offset == (reg->offset & -entry->size)) - break; - entry++; - remain--; - } - - if (!remain) - return -EINVAL; - - flags = reg->offset & (entry->size - 1); - - with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - if (entry->size == 8 && flags == I915_REG_READ_8B_WA) - reg->val = intel_uncore_read64_2x32(uncore, - entry->offset_ldw, - entry->offset_udw); - else if (entry->size == 8 && flags == 0) - reg->val = intel_uncore_read64(uncore, - entry->offset_ldw); - else if (entry->size == 4 && flags == 0) - reg->val = intel_uncore_read(uncore, entry->offset_ldw); - else if (entry->size == 2 && flags == 0) - reg->val = intel_uncore_read16(uncore, - entry->offset_ldw); - else if (entry->size == 1 && flags == 0) - reg->val = intel_uncore_read8(uncore, - entry->offset_ldw); - else - ret = -EINVAL; - } - - return ret; -} - /** * __intel_wait_for_register_fw - wait until register matches expected state * @uncore: the struct intel_uncore diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h index 2a15b2b2e2fc..6ff56d673e2b 100644 --- a/drivers/gpu/drm/i915/intel_uncore.h +++ b/drivers/gpu/drm/i915/intel_uncore.h @@ -30,7 +30,7 @@ #include <linux/hrtimer.h> #include <linux/io-64-nonatomic-lo-hi.h> -#include "i915_reg.h" +#include "i915_reg_defs.h" struct drm_i915_private; struct intel_runtime_pm; diff --git a/drivers/gpu/drm/i915/intel_wopcm.c b/drivers/gpu/drm/i915/intel_wopcm.c index f06d21005106..322fb9eeb880 100644 --- a/drivers/gpu/drm/i915/intel_wopcm.c +++ b/drivers/gpu/drm/i915/intel_wopcm.c @@ -43,6 +43,7 @@ /* Default WOPCM size is 2MB from Gen11, 1MB on previous platforms */ #define GEN11_WOPCM_SIZE SZ_2M #define GEN9_WOPCM_SIZE SZ_1M +#define MAX_WOPCM_SIZE SZ_8M /* 16KB WOPCM (RSVD WOPCM) is reserved from HuC firmware top. */ #define WOPCM_RESERVED_SIZE SZ_16K @@ -207,6 +208,14 @@ static bool __wopcm_regs_locked(struct intel_uncore *uncore, return true; } +static bool __wopcm_regs_writable(struct intel_uncore *uncore) +{ + if (!HAS_GUC_DEPRIVILEGE(uncore->i915)) + return true; + + return intel_uncore_read(uncore, GUC_SHIM_CONTROL2) & GUC_IS_PRIVILEGED; +} + /** * intel_wopcm_init() - Initialize the WOPCM structure. * @wopcm: pointer to intel_wopcm. @@ -224,18 +233,19 @@ void intel_wopcm_init(struct intel_wopcm *wopcm) u32 guc_fw_size = intel_uc_fw_get_upload_size(>->uc.guc.fw); u32 huc_fw_size = intel_uc_fw_get_upload_size(>->uc.huc.fw); u32 ctx_rsvd = context_reserved_size(i915); + u32 wopcm_size = wopcm->size; u32 guc_wopcm_base; u32 guc_wopcm_size; if (!guc_fw_size) return; - GEM_BUG_ON(!wopcm->size); + GEM_BUG_ON(!wopcm_size); GEM_BUG_ON(wopcm->guc.base); GEM_BUG_ON(wopcm->guc.size); - GEM_BUG_ON(guc_fw_size >= wopcm->size); - GEM_BUG_ON(huc_fw_size >= wopcm->size); - GEM_BUG_ON(ctx_rsvd + WOPCM_RESERVED_SIZE >= wopcm->size); + GEM_BUG_ON(guc_fw_size >= wopcm_size); + GEM_BUG_ON(huc_fw_size >= wopcm_size); + GEM_BUG_ON(ctx_rsvd + WOPCM_RESERVED_SIZE >= wopcm_size); if (i915_inject_probe_failure(i915)) return; @@ -243,6 +253,24 @@ void intel_wopcm_init(struct intel_wopcm *wopcm) if (__wopcm_regs_locked(gt->uncore, &guc_wopcm_base, &guc_wopcm_size)) { drm_dbg(&i915->drm, "GuC WOPCM is already locked [%uK, %uK)\n", guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); + /* + * Note that to keep things simple (i.e. avoid different + * defines per platform) our WOPCM math doesn't always use the + * actual WOPCM size, but a value that is less or equal to it. + * This is perfectly fine when i915 programs the registers, but + * on platforms with GuC deprivilege the registers are not + * writable from i915 and are instead pre-programmed by the + * bios/IFWI, so there might be a mismatch of sizes. + * Instead of handling the size difference, we trust that the + * programmed values make sense and disable the relevant check + * by using the maximum possible WOPCM size in the verification + * math. In the extremely unlikely case that the registers + * were pre-programmed with an invalid value, we will still + * gracefully fail later during the GuC/HuC dma. + */ + if (!__wopcm_regs_writable(gt->uncore)) + wopcm_size = MAX_WOPCM_SIZE; + goto check; } @@ -257,17 +285,17 @@ void intel_wopcm_init(struct intel_wopcm *wopcm) * Need to clamp guc_wopcm_base now to make sure the following math is * correct. Formal check of whole WOPCM layout will be done below. */ - guc_wopcm_base = min(guc_wopcm_base, wopcm->size - ctx_rsvd); + guc_wopcm_base = min(guc_wopcm_base, wopcm_size - ctx_rsvd); /* Aligned remainings of usable WOPCM space can be assigned to GuC. */ - guc_wopcm_size = wopcm->size - ctx_rsvd - guc_wopcm_base; + guc_wopcm_size = wopcm_size - ctx_rsvd - guc_wopcm_base; guc_wopcm_size &= GUC_WOPCM_SIZE_MASK; drm_dbg(&i915->drm, "Calculated GuC WOPCM [%uK, %uK)\n", guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); check: - if (__check_layout(i915, wopcm->size, guc_wopcm_base, guc_wopcm_size, + if (__check_layout(i915, wopcm_size, guc_wopcm_base, guc_wopcm_size, guc_fw_size, huc_fw_size)) { wopcm->guc.base = guc_wopcm_base; wopcm->guc.size = guc_wopcm_size; diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c index 8d5553772ded..04745f914407 100644 --- a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c @@ -7,6 +7,7 @@ #include "intel_pxp_irq.h" #include "intel_pxp_session.h" #include "gt/intel_gt_irq.h" +#include "gt/intel_gt_regs.h" #include "gt/intel_gt_types.h" #include "i915_irq.h" #include "i915_reg.h" diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h index 16990a3f2f85..586be769104f 100644 --- a/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_pm.h @@ -6,7 +6,7 @@ #ifndef __INTEL_PXP_PM_H__ #define __INTEL_PXP_PM_H__ -#include "intel_pxp_types.h" +struct intel_pxp; #ifdef CONFIG_DRM_I915_PXP void intel_pxp_suspend_prepare(struct intel_pxp *pxp); diff --git a/drivers/gpu/drm/i915/selftests/i915_buddy.c b/drivers/gpu/drm/i915/selftests/i915_buddy.c deleted file mode 100644 index d61ec9c951bf..000000000000 --- a/drivers/gpu/drm/i915/selftests/i915_buddy.c +++ /dev/null @@ -1,787 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2019 Intel Corporation - */ - -#include <linux/prime_numbers.h> - -#include "../i915_selftest.h" -#include "i915_random.h" - -static void __igt_dump_block(struct i915_buddy_mm *mm, - struct i915_buddy_block *block, - bool buddy) -{ - pr_err("block info: header=%llx, state=%u, order=%d, offset=%llx size=%llx root=%s buddy=%s\n", - block->header, - i915_buddy_block_state(block), - i915_buddy_block_order(block), - i915_buddy_block_offset(block), - i915_buddy_block_size(mm, block), - yesno(!block->parent), - yesno(buddy)); -} - -static void igt_dump_block(struct i915_buddy_mm *mm, - struct i915_buddy_block *block) -{ - struct i915_buddy_block *buddy; - - __igt_dump_block(mm, block, false); - - buddy = get_buddy(block); - if (buddy) - __igt_dump_block(mm, buddy, true); -} - -static int igt_check_block(struct i915_buddy_mm *mm, - struct i915_buddy_block *block) -{ - struct i915_buddy_block *buddy; - unsigned int block_state; - u64 block_size; - u64 offset; - int err = 0; - - block_state = i915_buddy_block_state(block); - - if (block_state != I915_BUDDY_ALLOCATED && - block_state != I915_BUDDY_FREE && - block_state != I915_BUDDY_SPLIT) { - pr_err("block state mismatch\n"); - err = -EINVAL; - } - - block_size = i915_buddy_block_size(mm, block); - offset = i915_buddy_block_offset(block); - - if (block_size < mm->chunk_size) { - pr_err("block size smaller than min size\n"); - err = -EINVAL; - } - - if (!is_power_of_2(block_size)) { - pr_err("block size not power of two\n"); - err = -EINVAL; - } - - if (!IS_ALIGNED(block_size, mm->chunk_size)) { - pr_err("block size not aligned to min size\n"); - err = -EINVAL; - } - - if (!IS_ALIGNED(offset, mm->chunk_size)) { - pr_err("block offset not aligned to min size\n"); - err = -EINVAL; - } - - if (!IS_ALIGNED(offset, block_size)) { - pr_err("block offset not aligned to block size\n"); - err = -EINVAL; - } - - buddy = get_buddy(block); - - if (!buddy && block->parent) { - pr_err("buddy has gone fishing\n"); - err = -EINVAL; - } - - if (buddy) { - if (i915_buddy_block_offset(buddy) != (offset ^ block_size)) { - pr_err("buddy has wrong offset\n"); - err = -EINVAL; - } - - if (i915_buddy_block_size(mm, buddy) != block_size) { - pr_err("buddy size mismatch\n"); - err = -EINVAL; - } - - if (i915_buddy_block_state(buddy) == block_state && - block_state == I915_BUDDY_FREE) { - pr_err("block and its buddy are free\n"); - err = -EINVAL; - } - } - - return err; -} - -static int igt_check_blocks(struct i915_buddy_mm *mm, - struct list_head *blocks, - u64 expected_size, - bool is_contiguous) -{ - struct i915_buddy_block *block; - struct i915_buddy_block *prev; - u64 total; - int err = 0; - - block = NULL; - prev = NULL; - total = 0; - - list_for_each_entry(block, blocks, link) { - err = igt_check_block(mm, block); - - if (!i915_buddy_block_is_allocated(block)) { - pr_err("block not allocated\n"), - err = -EINVAL; - } - - if (is_contiguous && prev) { - u64 prev_block_size; - u64 prev_offset; - u64 offset; - - prev_offset = i915_buddy_block_offset(prev); - prev_block_size = i915_buddy_block_size(mm, prev); - offset = i915_buddy_block_offset(block); - - if (offset != (prev_offset + prev_block_size)) { - pr_err("block offset mismatch\n"); - err = -EINVAL; - } - } - - if (err) - break; - - total += i915_buddy_block_size(mm, block); - prev = block; - } - - if (!err) { - if (total != expected_size) { - pr_err("size mismatch, expected=%llx, found=%llx\n", - expected_size, total); - err = -EINVAL; - } - return err; - } - - if (prev) { - pr_err("prev block, dump:\n"); - igt_dump_block(mm, prev); - } - - pr_err("bad block, dump:\n"); - igt_dump_block(mm, block); - - return err; -} - -static int igt_check_mm(struct i915_buddy_mm *mm) -{ - struct i915_buddy_block *root; - struct i915_buddy_block *prev; - unsigned int i; - u64 total; - int err = 0; - - if (!mm->n_roots) { - pr_err("n_roots is zero\n"); - return -EINVAL; - } - - if (mm->n_roots != hweight64(mm->size)) { - pr_err("n_roots mismatch, n_roots=%u, expected=%lu\n", - mm->n_roots, hweight64(mm->size)); - return -EINVAL; - } - - root = NULL; - prev = NULL; - total = 0; - - for (i = 0; i < mm->n_roots; ++i) { - struct i915_buddy_block *block; - unsigned int order; - - root = mm->roots[i]; - if (!root) { - pr_err("root(%u) is NULL\n", i); - err = -EINVAL; - break; - } - - err = igt_check_block(mm, root); - - if (!i915_buddy_block_is_free(root)) { - pr_err("root not free\n"); - err = -EINVAL; - } - - order = i915_buddy_block_order(root); - - if (!i) { - if (order != mm->max_order) { - pr_err("max order root missing\n"); - err = -EINVAL; - } - } - - if (prev) { - u64 prev_block_size; - u64 prev_offset; - u64 offset; - - prev_offset = i915_buddy_block_offset(prev); - prev_block_size = i915_buddy_block_size(mm, prev); - offset = i915_buddy_block_offset(root); - - if (offset != (prev_offset + prev_block_size)) { - pr_err("root offset mismatch\n"); - err = -EINVAL; - } - } - - block = list_first_entry_or_null(&mm->free_list[order], - struct i915_buddy_block, - link); - if (block != root) { - pr_err("root mismatch at order=%u\n", order); - err = -EINVAL; - } - - if (err) - break; - - prev = root; - total += i915_buddy_block_size(mm, root); - } - - if (!err) { - if (total != mm->size) { - pr_err("expected mm size=%llx, found=%llx\n", mm->size, - total); - err = -EINVAL; - } - return err; - } - - if (prev) { - pr_err("prev root(%u), dump:\n", i - 1); - igt_dump_block(mm, prev); - } - - if (root) { - pr_err("bad root(%u), dump:\n", i); - igt_dump_block(mm, root); - } - - return err; -} - -static void igt_mm_config(u64 *size, u64 *chunk_size) -{ - I915_RND_STATE(prng); - u32 s, ms; - - /* Nothing fancy, just try to get an interesting bit pattern */ - - prandom_seed_state(&prng, i915_selftest.random_seed); - - /* Let size be a random number of pages up to 8 GB (2M pages) */ - s = 1 + i915_prandom_u32_max_state((BIT(33 - 12)) - 1, &prng); - /* Let the chunk size be a random power of 2 less than size */ - ms = BIT(i915_prandom_u32_max_state(ilog2(s), &prng)); - /* Round size down to the chunk size */ - s &= -ms; - - /* Convert from pages to bytes */ - *chunk_size = (u64)ms << 12; - *size = (u64)s << 12; -} - -static int igt_buddy_alloc_smoke(void *arg) -{ - struct i915_buddy_mm mm; - IGT_TIMEOUT(end_time); - I915_RND_STATE(prng); - u64 chunk_size; - u64 mm_size; - int *order; - int err, i; - - igt_mm_config(&mm_size, &chunk_size); - - pr_info("buddy_init with size=%llx, chunk_size=%llx\n", mm_size, chunk_size); - - err = i915_buddy_init(&mm, mm_size, chunk_size); - if (err) { - pr_err("buddy_init failed(%d)\n", err); - return err; - } - - order = i915_random_order(mm.max_order + 1, &prng); - if (!order) - goto out_fini; - - for (i = 0; i <= mm.max_order; ++i) { - struct i915_buddy_block *block; - int max_order = order[i]; - bool timeout = false; - LIST_HEAD(blocks); - int order; - u64 total; - - err = igt_check_mm(&mm); - if (err) { - pr_err("pre-mm check failed, abort\n"); - break; - } - - pr_info("filling from max_order=%u\n", max_order); - - order = max_order; - total = 0; - - do { -retry: - block = i915_buddy_alloc(&mm, order); - if (IS_ERR(block)) { - err = PTR_ERR(block); - if (err == -ENOMEM) { - pr_info("buddy_alloc hit -ENOMEM with order=%d\n", - order); - } else { - if (order--) { - err = 0; - goto retry; - } - - pr_err("buddy_alloc with order=%d failed(%d)\n", - order, err); - } - - break; - } - - list_add_tail(&block->link, &blocks); - - if (i915_buddy_block_order(block) != order) { - pr_err("buddy_alloc order mismatch\n"); - err = -EINVAL; - break; - } - - total += i915_buddy_block_size(&mm, block); - - if (__igt_timeout(end_time, NULL)) { - timeout = true; - break; - } - } while (total < mm.size); - - if (!err) - err = igt_check_blocks(&mm, &blocks, total, false); - - i915_buddy_free_list(&mm, &blocks); - - if (!err) { - err = igt_check_mm(&mm); - if (err) - pr_err("post-mm check failed\n"); - } - - if (err || timeout) - break; - - cond_resched(); - } - - if (err == -ENOMEM) - err = 0; - - kfree(order); -out_fini: - i915_buddy_fini(&mm); - - return err; -} - -static int igt_buddy_alloc_pessimistic(void *arg) -{ - const unsigned int max_order = 16; - struct i915_buddy_block *block, *bn; - struct i915_buddy_mm mm; - unsigned int order; - LIST_HEAD(blocks); - int err; - - /* - * Create a pot-sized mm, then allocate one of each possible - * order within. This should leave the mm with exactly one - * page left. - */ - - err = i915_buddy_init(&mm, PAGE_SIZE << max_order, PAGE_SIZE); - if (err) { - pr_err("buddy_init failed(%d)\n", err); - return err; - } - GEM_BUG_ON(mm.max_order != max_order); - - for (order = 0; order < max_order; order++) { - block = i915_buddy_alloc(&mm, order); - if (IS_ERR(block)) { - pr_info("buddy_alloc hit -ENOMEM with order=%d\n", - order); - err = PTR_ERR(block); - goto err; - } - - list_add_tail(&block->link, &blocks); - } - - /* And now the last remaining block available */ - block = i915_buddy_alloc(&mm, 0); - if (IS_ERR(block)) { - pr_info("buddy_alloc hit -ENOMEM on final alloc\n"); - err = PTR_ERR(block); - goto err; - } - list_add_tail(&block->link, &blocks); - - /* Should be completely full! */ - for (order = max_order; order--; ) { - block = i915_buddy_alloc(&mm, order); - if (!IS_ERR(block)) { - pr_info("buddy_alloc unexpectedly succeeded at order %d, it should be full!", - order); - list_add_tail(&block->link, &blocks); - err = -EINVAL; - goto err; - } - } - - block = list_last_entry(&blocks, typeof(*block), link); - list_del(&block->link); - i915_buddy_free(&mm, block); - - /* As we free in increasing size, we make available larger blocks */ - order = 1; - list_for_each_entry_safe(block, bn, &blocks, link) { - list_del(&block->link); - i915_buddy_free(&mm, block); - - block = i915_buddy_alloc(&mm, order); - if (IS_ERR(block)) { - pr_info("buddy_alloc (realloc) hit -ENOMEM with order=%d\n", - order); - err = PTR_ERR(block); - goto err; - } - i915_buddy_free(&mm, block); - order++; - } - - /* To confirm, now the whole mm should be available */ - block = i915_buddy_alloc(&mm, max_order); - if (IS_ERR(block)) { - pr_info("buddy_alloc (realloc) hit -ENOMEM with order=%d\n", - max_order); - err = PTR_ERR(block); - goto err; - } - i915_buddy_free(&mm, block); - -err: - i915_buddy_free_list(&mm, &blocks); - i915_buddy_fini(&mm); - return err; -} - -static int igt_buddy_alloc_optimistic(void *arg) -{ - const int max_order = 16; - struct i915_buddy_block *block; - struct i915_buddy_mm mm; - LIST_HEAD(blocks); - int order; - int err; - - /* - * Create a mm with one block of each order available, and - * try to allocate them all. - */ - - err = i915_buddy_init(&mm, - PAGE_SIZE * ((1 << (max_order + 1)) - 1), - PAGE_SIZE); - if (err) { - pr_err("buddy_init failed(%d)\n", err); - return err; - } - GEM_BUG_ON(mm.max_order != max_order); - - for (order = 0; order <= max_order; order++) { - block = i915_buddy_alloc(&mm, order); - if (IS_ERR(block)) { - pr_info("buddy_alloc hit -ENOMEM with order=%d\n", - order); - err = PTR_ERR(block); - goto err; - } - - list_add_tail(&block->link, &blocks); - } - - /* Should be completely full! */ - block = i915_buddy_alloc(&mm, 0); - if (!IS_ERR(block)) { - pr_info("buddy_alloc unexpectedly succeeded, it should be full!"); - list_add_tail(&block->link, &blocks); - err = -EINVAL; - goto err; - } - -err: - i915_buddy_free_list(&mm, &blocks); - i915_buddy_fini(&mm); - return err; -} - -static int igt_buddy_alloc_pathological(void *arg) -{ - const int max_order = 16; - struct i915_buddy_block *block; - struct i915_buddy_mm mm; - LIST_HEAD(blocks); - LIST_HEAD(holes); - int order, top; - int err; - - /* - * Create a pot-sized mm, then allocate one of each possible - * order within. This should leave the mm with exactly one - * page left. Free the largest block, then whittle down again. - * Eventually we will have a fully 50% fragmented mm. - */ - - err = i915_buddy_init(&mm, PAGE_SIZE << max_order, PAGE_SIZE); - if (err) { - pr_err("buddy_init failed(%d)\n", err); - return err; - } - GEM_BUG_ON(mm.max_order != max_order); - - for (top = max_order; top; top--) { - /* Make room by freeing the largest allocated block */ - block = list_first_entry_or_null(&blocks, typeof(*block), link); - if (block) { - list_del(&block->link); - i915_buddy_free(&mm, block); - } - - for (order = top; order--; ) { - block = i915_buddy_alloc(&mm, order); - if (IS_ERR(block)) { - pr_info("buddy_alloc hit -ENOMEM with order=%d, top=%d\n", - order, top); - err = PTR_ERR(block); - goto err; - } - list_add_tail(&block->link, &blocks); - } - - /* There should be one final page for this sub-allocation */ - block = i915_buddy_alloc(&mm, 0); - if (IS_ERR(block)) { - pr_info("buddy_alloc hit -ENOMEM for hole\n"); - err = PTR_ERR(block); - goto err; - } - list_add_tail(&block->link, &holes); - - block = i915_buddy_alloc(&mm, top); - if (!IS_ERR(block)) { - pr_info("buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!", - top, max_order); - list_add_tail(&block->link, &blocks); - err = -EINVAL; - goto err; - } - } - - i915_buddy_free_list(&mm, &holes); - - /* Nothing larger than blocks of chunk_size now available */ - for (order = 1; order <= max_order; order++) { - block = i915_buddy_alloc(&mm, order); - if (!IS_ERR(block)) { - pr_info("buddy_alloc unexpectedly succeeded at order %d, it should be full!", - order); - list_add_tail(&block->link, &blocks); - err = -EINVAL; - goto err; - } - } - -err: - list_splice_tail(&holes, &blocks); - i915_buddy_free_list(&mm, &blocks); - i915_buddy_fini(&mm); - return err; -} - -static int igt_buddy_alloc_range(void *arg) -{ - struct i915_buddy_mm mm; - unsigned long page_num; - LIST_HEAD(blocks); - u64 chunk_size; - u64 offset; - u64 size; - u64 rem; - int err; - - igt_mm_config(&size, &chunk_size); - - pr_info("buddy_init with size=%llx, chunk_size=%llx\n", size, chunk_size); - - err = i915_buddy_init(&mm, size, chunk_size); - if (err) { - pr_err("buddy_init failed(%d)\n", err); - return err; - } - - err = igt_check_mm(&mm); - if (err) { - pr_err("pre-mm check failed, abort, abort, abort!\n"); - goto err_fini; - } - - rem = mm.size; - offset = 0; - - for_each_prime_number_from(page_num, 1, ULONG_MAX - 1) { - struct i915_buddy_block *block; - LIST_HEAD(tmp); - - size = min(page_num * mm.chunk_size, rem); - - err = i915_buddy_alloc_range(&mm, &tmp, offset, size); - if (err) { - if (err == -ENOMEM) { - pr_info("alloc_range hit -ENOMEM with size=%llx\n", - size); - } else { - pr_err("alloc_range with offset=%llx, size=%llx failed(%d)\n", - offset, size, err); - } - - break; - } - - block = list_first_entry_or_null(&tmp, - struct i915_buddy_block, - link); - if (!block) { - pr_err("alloc_range has no blocks\n"); - err = -EINVAL; - break; - } - - if (i915_buddy_block_offset(block) != offset) { - pr_err("alloc_range start offset mismatch, found=%llx, expected=%llx\n", - i915_buddy_block_offset(block), offset); - err = -EINVAL; - } - - if (!err) - err = igt_check_blocks(&mm, &tmp, size, true); - - list_splice_tail(&tmp, &blocks); - - if (err) - break; - - offset += size; - - rem -= size; - if (!rem) - break; - - cond_resched(); - } - - if (err == -ENOMEM) - err = 0; - - i915_buddy_free_list(&mm, &blocks); - - if (!err) { - err = igt_check_mm(&mm); - if (err) - pr_err("post-mm check failed\n"); - } - -err_fini: - i915_buddy_fini(&mm); - - return err; -} - -static int igt_buddy_alloc_limit(void *arg) -{ - struct i915_buddy_block *block; - struct i915_buddy_mm mm; - const u64 size = U64_MAX; - int err; - - err = i915_buddy_init(&mm, size, PAGE_SIZE); - if (err) - return err; - - if (mm.max_order != I915_BUDDY_MAX_ORDER) { - pr_err("mm.max_order(%d) != %d\n", - mm.max_order, I915_BUDDY_MAX_ORDER); - err = -EINVAL; - goto out_fini; - } - - block = i915_buddy_alloc(&mm, mm.max_order); - if (IS_ERR(block)) { - err = PTR_ERR(block); - goto out_fini; - } - - if (i915_buddy_block_order(block) != mm.max_order) { - pr_err("block order(%d) != %d\n", - i915_buddy_block_order(block), mm.max_order); - err = -EINVAL; - goto out_free; - } - - if (i915_buddy_block_size(&mm, block) != - BIT_ULL(mm.max_order) * PAGE_SIZE) { - pr_err("block size(%llu) != %llu\n", - i915_buddy_block_size(&mm, block), - BIT_ULL(mm.max_order) * PAGE_SIZE); - err = -EINVAL; - goto out_free; - } - -out_free: - i915_buddy_free(&mm, block); -out_fini: - i915_buddy_fini(&mm); - return err; -} - -int i915_buddy_mock_selftests(void) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_buddy_alloc_pessimistic), - SUBTEST(igt_buddy_alloc_optimistic), - SUBTEST(igt_buddy_alloc_pathological), - SUBTEST(igt_buddy_alloc_smoke), - SUBTEST(igt_buddy_alloc_range), - SUBTEST(igt_buddy_alloc_limit), - }; - - return i915_subtests(tests, NULL); -} diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index b5576888cd78..e5dd82e7e480 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -6,9 +6,10 @@ #include <linux/random.h> +#include "gem/i915_gem_internal.h" +#include "gem/i915_gem_pm.h" #include "gem/selftests/igt_gem_utils.h" #include "gem/selftests/mock_context.h" -#include "gem/i915_gem_pm.h" #include "gt/intel_gt.h" #include "gt/intel_gt_pm.h" @@ -41,7 +42,7 @@ static int switch_to_context(struct i915_gem_context *ctx) static void trash_stolen(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; const u64 slot = ggtt->error_capture.start; const resource_size_t size = resource_size(&i915->dsm); unsigned long page; @@ -99,7 +100,7 @@ static void igt_pm_suspend(struct drm_i915_private *i915) intel_wakeref_t wakeref; with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_suspend(&i915->ggtt); + i915_ggtt_suspend(to_gt(i915)->ggtt); i915_gem_suspend_late(i915); } } @@ -109,7 +110,7 @@ static void igt_pm_hibernate(struct drm_i915_private *i915) intel_wakeref_t wakeref; with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_suspend(&i915->ggtt); + i915_ggtt_suspend(to_gt(i915)->ggtt); i915_gem_freeze(i915); i915_gem_freeze_late(i915); @@ -125,7 +126,7 @@ static void igt_pm_resume(struct drm_i915_private *i915) * that runtime-pm just works. */ with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_resume(&i915->ggtt); + i915_ggtt_resume(to_gt(i915)->ggtt); i915_gem_resume(i915); } } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c index 75b709c26dd3..8c6517d29b8e 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c @@ -22,6 +22,7 @@ * */ +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gem/selftests/igt_gem_utils.h" #include "gem/selftests/mock_context.h" @@ -117,7 +118,7 @@ static int igt_evict_something(void *arg) /* Everything is pinned, nothing should happen */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_something(&ggtt->vm, + err = i915_gem_evict_something(&ggtt->vm, NULL, I915_GTT_PAGE_SIZE, 0, 0, 0, U64_MAX, 0); @@ -132,7 +133,7 @@ static int igt_evict_something(void *arg) /* Everything is unpinned, we should be able to evict something */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_something(&ggtt->vm, + err = i915_gem_evict_something(&ggtt->vm, NULL, I915_GTT_PAGE_SIZE, 0, 0, 0, U64_MAX, 0); @@ -204,7 +205,7 @@ static int igt_evict_for_vma(void *arg) /* Everything is pinned, nothing should happen */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_for_node(&ggtt->vm, &target, 0); + err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); mutex_unlock(&ggtt->vm.mutex); if (err != -ENOSPC) { pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n", @@ -216,7 +217,7 @@ static int igt_evict_for_vma(void *arg) /* Everything is unpinned, we should be able to evict the node */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_for_node(&ggtt->vm, &target, 0); + err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); mutex_unlock(&ggtt->vm.mutex); if (err) { pr_err("i915_gem_evict_for_node returned err=%d\n", @@ -297,7 +298,7 @@ static int igt_evict_for_cache_color(void *arg) /* Remove just the second vma */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_for_node(&ggtt->vm, &target, 0); + err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); mutex_unlock(&ggtt->vm.mutex); if (err) { pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err); @@ -310,7 +311,7 @@ static int igt_evict_for_cache_color(void *arg) target.color = I915_CACHE_L3_LLC; mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_for_node(&ggtt->vm, &target, 0); + err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); mutex_unlock(&ggtt->vm.mutex); if (!err) { pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err); @@ -331,6 +332,7 @@ static int igt_evict_vm(void *arg) { struct intel_gt *gt = arg; struct i915_ggtt *ggtt = gt->ggtt; + struct i915_gem_ww_ctx ww; LIST_HEAD(objects); int err; @@ -342,7 +344,7 @@ static int igt_evict_vm(void *arg) /* Everything is pinned, nothing should happen */ mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_vm(&ggtt->vm); + err = i915_gem_evict_vm(&ggtt->vm, NULL); mutex_unlock(&ggtt->vm.mutex); if (err) { pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n", @@ -352,9 +354,12 @@ static int igt_evict_vm(void *arg) unpin_ggtt(ggtt); - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_evict_vm(&ggtt->vm); - mutex_unlock(&ggtt->vm.mutex); + for_i915_gem_ww(&ww, err, false) { + mutex_lock(&ggtt->vm.mutex); + err = i915_gem_evict_vm(&ggtt->vm, &ww); + mutex_unlock(&ggtt->vm.mutex); + } + if (err) { pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n", err); @@ -402,7 +407,7 @@ static int igt_evict_contexts(void *arg) /* Reserve a block so that we know we have enough to fit a few rq */ memset(&hole, 0, sizeof(hole)); mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &hole, + err = i915_gem_gtt_insert(&ggtt->vm, NULL, &hole, PRETEND_GGTT_SIZE, 0, I915_COLOR_UNEVICTABLE, 0, ggtt->vm.total, PIN_NOEVICT); @@ -422,7 +427,7 @@ static int igt_evict_contexts(void *arg) goto out_locked; } - if (i915_gem_gtt_insert(&ggtt->vm, &r->node, + if (i915_gem_gtt_insert(&ggtt->vm, NULL, &r->node, 1ul << 20, 0, I915_COLOR_UNEVICTABLE, 0, ggtt->vm.total, PIN_NOEVICT)) { diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 575705c3bce9..ab751192eb3b 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -26,12 +26,16 @@ #include <linux/prime_numbers.h> #include "gem/i915_gem_context.h" +#include "gem/i915_gem_internal.h" +#include "gem/i915_gem_region.h" #include "gem/selftests/mock_context.h" #include "gt/intel_context.h" #include "gt/intel_gpu_commands.h" +#include "gt/intel_gtt.h" #include "i915_random.h" #include "i915_selftest.h" +#include "i915_vma_resource.h" #include "mock_drm.h" #include "mock_gem_device.h" @@ -237,12 +241,14 @@ static int lowlevel_hole(struct i915_address_space *vm, u64 hole_start, u64 hole_end, unsigned long end_time) { + const unsigned int min_alignment = + i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); I915_RND_STATE(seed_prng); - struct i915_vma *mock_vma; + struct i915_vma_resource *mock_vma_res; unsigned int size; - mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL); - if (!mock_vma) + mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL); + if (!mock_vma_res) return -ENOMEM; /* Keep creating larger objects until one cannot fit into the hole */ @@ -250,9 +256,10 @@ static int lowlevel_hole(struct i915_address_space *vm, I915_RND_SUBSTATE(prng, seed_prng); struct drm_i915_gem_object *obj; unsigned int *order, count, n; - u64 hole_size; + u64 hole_size, aligned_size; - hole_size = (hole_end - hole_start) >> size; + aligned_size = max_t(u32, ilog2(min_alignment), size); + hole_size = (hole_end - hole_start) >> aligned_size; if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) hole_size = KMALLOC_MAX_SIZE / sizeof(u32); count = hole_size >> 1; @@ -268,13 +275,13 @@ static int lowlevel_hole(struct i915_address_space *vm, break; } while (count >>= 1); if (!count) { - kfree(mock_vma); + kfree(mock_vma_res); return -ENOMEM; } GEM_BUG_ON(!order); - GEM_BUG_ON(count * BIT_ULL(size) > vm->total); - GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end); + GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total); + GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end); /* Ignore allocation failures (i.e. don't report them as * a test failure) as we are purposefully allocating very @@ -297,10 +304,10 @@ static int lowlevel_hole(struct i915_address_space *vm, } for (n = 0; n < count; n++) { - u64 addr = hole_start + order[n] * BIT_ULL(size); + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); intel_wakeref_t wakeref; - GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); + GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total); if (igt_timeout(end_time, "%s timed out before %d/%d\n", @@ -342,19 +349,19 @@ alloc_vm_end: break; } - mock_vma->pages = obj->mm.pages; - mock_vma->node.size = BIT_ULL(size); - mock_vma->node.start = addr; + mock_vma_res->bi.pages = obj->mm.pages; + mock_vma_res->node_size = BIT_ULL(aligned_size); + mock_vma_res->start = addr; with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) - vm->insert_entries(vm, mock_vma, + vm->insert_entries(vm, mock_vma_res, I915_CACHE_NONE, 0); } count = n; i915_random_reorder(order, count, &prng); for (n = 0; n < count; n++) { - u64 addr = hole_start + order[n] * BIT_ULL(size); + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); intel_wakeref_t wakeref; GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); @@ -370,7 +377,7 @@ alloc_vm_end: cleanup_freed_objects(vm->i915); } - kfree(mock_vma); + kfree(mock_vma_res); return 0; } @@ -385,7 +392,7 @@ static void close_object_list(struct list_head *objects, vma = i915_vma_instance(obj, vm, NULL); if (!IS_ERR(vma)) - ignored = i915_vma_unbind(vma); + ignored = i915_vma_unbind_unlocked(vma); list_del(&obj->st_link); i915_gem_object_put(obj); @@ -398,8 +405,10 @@ static int fill_hole(struct i915_address_space *vm, { const u64 hole_size = hole_end - hole_start; struct drm_i915_gem_object *obj; + const unsigned int min_alignment = + i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); const unsigned long max_pages = - min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT); + min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment)); const unsigned long max_step = max(int_sqrt(max_pages), 2UL); unsigned long npages, prime, flags; struct i915_vma *vma; @@ -440,14 +449,17 @@ static int fill_hole(struct i915_address_space *vm, offset = p->offset; list_for_each_entry(obj, &objects, st_link) { + u64 aligned_size = round_up(obj->base.size, + min_alignment); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) continue; if (p->step < 0) { - if (offset < hole_start + obj->base.size) + if (offset < hole_start + aligned_size) break; - offset -= obj->base.size; + offset -= aligned_size; } err = i915_vma_pin(vma, 0, 0, offset | flags); @@ -469,22 +481,25 @@ static int fill_hole(struct i915_address_space *vm, i915_vma_unpin(vma); if (p->step > 0) { - if (offset + obj->base.size > hole_end) + if (offset + aligned_size > hole_end) break; - offset += obj->base.size; + offset += aligned_size; } } offset = p->offset; list_for_each_entry(obj, &objects, st_link) { + u64 aligned_size = round_up(obj->base.size, + min_alignment); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) continue; if (p->step < 0) { - if (offset < hole_start + obj->base.size) + if (offset < hole_start + aligned_size) break; - offset -= obj->base.size; + offset -= aligned_size; } if (!drm_mm_node_allocated(&vma->node) || @@ -496,7 +511,7 @@ static int fill_hole(struct i915_address_space *vm, goto err; } - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", __func__, p->name, vma->node.start, vma->node.size, @@ -505,22 +520,25 @@ static int fill_hole(struct i915_address_space *vm, } if (p->step > 0) { - if (offset + obj->base.size > hole_end) + if (offset + aligned_size > hole_end) break; - offset += obj->base.size; + offset += aligned_size; } } offset = p->offset; list_for_each_entry_reverse(obj, &objects, st_link) { + u64 aligned_size = round_up(obj->base.size, + min_alignment); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) continue; if (p->step < 0) { - if (offset < hole_start + obj->base.size) + if (offset < hole_start + aligned_size) break; - offset -= obj->base.size; + offset -= aligned_size; } err = i915_vma_pin(vma, 0, 0, offset | flags); @@ -542,22 +560,25 @@ static int fill_hole(struct i915_address_space *vm, i915_vma_unpin(vma); if (p->step > 0) { - if (offset + obj->base.size > hole_end) + if (offset + aligned_size > hole_end) break; - offset += obj->base.size; + offset += aligned_size; } } offset = p->offset; list_for_each_entry_reverse(obj, &objects, st_link) { + u64 aligned_size = round_up(obj->base.size, + min_alignment); + vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) continue; if (p->step < 0) { - if (offset < hole_start + obj->base.size) + if (offset < hole_start + aligned_size) break; - offset -= obj->base.size; + offset -= aligned_size; } if (!drm_mm_node_allocated(&vma->node) || @@ -569,7 +590,7 @@ static int fill_hole(struct i915_address_space *vm, goto err; } - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", __func__, p->name, vma->node.start, vma->node.size, @@ -578,9 +599,9 @@ static int fill_hole(struct i915_address_space *vm, } if (p->step > 0) { - if (offset + obj->base.size > hole_end) + if (offset + aligned_size > hole_end) break; - offset += obj->base.size; + offset += aligned_size; } } } @@ -610,6 +631,7 @@ static int walk_hole(struct i915_address_space *vm, const u64 hole_size = hole_end - hole_start; const unsigned long max_pages = min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); + unsigned long min_alignment; unsigned long flags; u64 size; @@ -619,6 +641,8 @@ static int walk_hole(struct i915_address_space *vm, if (i915_is_ggtt(vm)) flags |= PIN_GLOBAL; + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); + for_each_prime_number_from(size, 1, max_pages) { struct drm_i915_gem_object *obj; struct i915_vma *vma; @@ -637,7 +661,7 @@ static int walk_hole(struct i915_address_space *vm, for (addr = hole_start; addr + obj->base.size < hole_end; - addr += obj->base.size) { + addr += round_up(obj->base.size, min_alignment)) { err = i915_vma_pin(vma, 0, 0, addr | flags); if (err) { pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", @@ -655,7 +679,7 @@ static int walk_hole(struct i915_address_space *vm, goto err_put; } - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("%s unbind failed at %llx + %llx with err=%d\n", __func__, addr, vma->size, err); @@ -689,6 +713,7 @@ static int pot_hole(struct i915_address_space *vm, { struct drm_i915_gem_object *obj; struct i915_vma *vma; + unsigned int min_alignment; unsigned long flags; unsigned int pot; int err = 0; @@ -697,6 +722,8 @@ static int pot_hole(struct i915_address_space *vm, if (i915_is_ggtt(vm)) flags |= PIN_GLOBAL; + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); + obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE); if (IS_ERR(obj)) return PTR_ERR(obj); @@ -709,13 +736,13 @@ static int pot_hole(struct i915_address_space *vm, /* Insert a pair of pages across every pot boundary within the hole */ for (pot = fls64(hole_end - 1) - 1; - pot > ilog2(2 * I915_GTT_PAGE_SIZE); + pot > ilog2(2 * min_alignment); pot--) { u64 step = BIT_ULL(pot); u64 addr; - for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; - addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE; + for (addr = round_up(hole_start + min_alignment, step) - min_alignment; + addr <= round_down(hole_end - (2 * min_alignment), step) - min_alignment; addr += step) { err = i915_vma_pin(vma, 0, 0, addr | flags); if (err) { @@ -732,13 +759,13 @@ static int pot_hole(struct i915_address_space *vm, pr_err("%s incorrect at %llx + %llx\n", __func__, addr, vma->size); i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); err = -EINVAL; goto err_obj; } i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); GEM_BUG_ON(err); } @@ -760,6 +787,7 @@ static int drunk_hole(struct i915_address_space *vm, unsigned long end_time) { I915_RND_STATE(prng); + unsigned int min_alignment; unsigned int size; unsigned long flags; @@ -767,15 +795,18 @@ static int drunk_hole(struct i915_address_space *vm, if (i915_is_ggtt(vm)) flags |= PIN_GLOBAL; + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); + /* Keep creating larger objects until one cannot fit into the hole */ for (size = 12; (hole_end - hole_start) >> size; size++) { struct drm_i915_gem_object *obj; unsigned int *order, count, n; struct i915_vma *vma; - u64 hole_size; + u64 hole_size, aligned_size; int err = -ENODEV; - hole_size = (hole_end - hole_start) >> size; + aligned_size = max_t(u32, ilog2(min_alignment), size); + hole_size = (hole_end - hole_start) >> aligned_size; if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) hole_size = KMALLOC_MAX_SIZE / sizeof(u32); count = hole_size >> 1; @@ -815,7 +846,7 @@ static int drunk_hole(struct i915_address_space *vm, GEM_BUG_ON(vma->size != BIT_ULL(size)); for (n = 0; n < count; n++) { - u64 addr = hole_start + order[n] * BIT_ULL(size); + u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); err = i915_vma_pin(vma, 0, 0, addr | flags); if (err) { @@ -832,13 +863,13 @@ static int drunk_hole(struct i915_address_space *vm, pr_err("%s incorrect at %llx + %llx\n", __func__, addr, BIT_ULL(size)); i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); err = -EINVAL; goto err_obj; } i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); GEM_BUG_ON(err); if (igt_timeout(end_time, @@ -867,11 +898,14 @@ static int __shrink_hole(struct i915_address_space *vm, { struct drm_i915_gem_object *obj; unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; + unsigned int min_alignment; unsigned int order = 12; LIST_HEAD(objects); int err = 0; u64 addr; + min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); + /* Keep creating larger objects until one cannot fit into the hole */ for (addr = hole_start; addr < hole_end; ) { struct i915_vma *vma; @@ -906,13 +940,13 @@ static int __shrink_hole(struct i915_address_space *vm, pr_err("%s incorrect at %llx + %llx\n", __func__, addr, size); i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); err = -EINVAL; break; } i915_vma_unpin(vma); - addr += size; + addr += round_up(size, min_alignment); /* * Since we are injecting allocation faults at random intervals, @@ -1036,6 +1070,118 @@ err_purge: return err; } +static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr, + u64 addr, u64 size, unsigned long flags) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + int err = 0; + u64 expected_vma_size, expected_node_size; + bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM || + mr->type == INTEL_MEMORY_STOLEN_LOCAL; + + obj = i915_gem_object_create_region(mr, size, 0, 0); + if (IS_ERR(obj)) { + /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */ + if (PTR_ERR(obj) == -ENODEV && is_stolen) + return 0; + return PTR_ERR(obj); + } + + vma = i915_vma_instance(obj, vm, NULL); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto err_put; + } + + err = i915_vma_pin(vma, 0, 0, addr | flags); + if (err) + goto err_put; + i915_vma_unpin(vma); + + if (!drm_mm_node_allocated(&vma->node)) { + err = -EINVAL; + goto err_put; + } + + if (i915_vma_misplaced(vma, 0, 0, addr | flags)) { + err = -EINVAL; + goto err_put; + } + + expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1)); + expected_node_size = expected_vma_size; + + if (NEEDS_COMPACT_PT(vm->i915) && i915_gem_object_is_lmem(obj)) { + /* compact-pt should expand lmem node to 2MB */ + expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K); + expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_2M); + } + + if (vma->size != expected_vma_size || vma->node.size != expected_node_size) { + err = i915_vma_unbind_unlocked(vma); + err = -EBADSLT; + goto err_put; + } + + err = i915_vma_unbind_unlocked(vma); + if (err) + goto err_put; + + GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); + +err_put: + i915_gem_object_put(obj); + cleanup_freed_objects(vm->i915); + return err; +} + +static int misaligned_pin(struct i915_address_space *vm, + u64 hole_start, u64 hole_end, + unsigned long end_time) +{ + struct intel_memory_region *mr; + enum intel_region_id id; + unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; + int err = 0; + u64 hole_size = hole_end - hole_start; + + if (i915_is_ggtt(vm)) + flags |= PIN_GLOBAL; + + for_each_memory_region(mr, vm->i915, id) { + u64 min_alignment = i915_vm_min_alignment(vm, (enum intel_memory_type)id); + u64 size = min_alignment; + u64 addr = round_down(hole_start + (hole_size / 2), min_alignment); + + /* avoid -ENOSPC on very small hole setups */ + if (hole_size < 3 * min_alignment) + continue; + + /* we can't test < 4k alignment due to flags being encoded in lower bits */ + if (min_alignment != I915_GTT_PAGE_SIZE_4K) { + err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags); + /* misaligned should error with -EINVAL*/ + if (!err) + err = -EBADSLT; + if (err != -EINVAL) + return err; + } + + /* test for vma->size expansion to min page size */ + err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags); + if (err) + return err; + + /* test for intermediate size not expanding vma->size for large alignments */ + err = misaligned_case(vm, mr, addr, size / 2, flags); + if (err) + return err; + } + + return 0; +} + static int exercise_ppgtt(struct drm_i915_private *dev_priv, int (*func)(struct i915_address_space *vm, u64 hole_start, u64 hole_end, @@ -1105,6 +1251,11 @@ static int igt_ppgtt_shrink_boom(void *arg) return exercise_ppgtt(arg, shrink_boom); } +static int igt_ppgtt_misaligned_pin(void *arg) +{ + return exercise_ppgtt(arg, misaligned_pin); +} + static int sort_holes(void *priv, const struct list_head *A, const struct list_head *B) { @@ -1122,7 +1273,7 @@ static int exercise_ggtt(struct drm_i915_private *i915, u64 hole_start, u64 hole_end, unsigned long end_time)) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; u64 hole_start, hole_end, last = 0; struct drm_mm_node *node; IGT_TIMEOUT(end_time); @@ -1177,12 +1328,17 @@ static int igt_ggtt_lowlevel(void *arg) return exercise_ggtt(arg, lowlevel_hole); } +static int igt_ggtt_misaligned_pin(void *arg) +{ + return exercise_ggtt(arg, misaligned_pin); +} + static int igt_ggtt_page(void *arg) { const unsigned int count = PAGE_SIZE/sizeof(u32); I915_RND_STATE(prng); struct drm_i915_private *i915 = arg; - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct drm_i915_gem_object *obj; intel_wakeref_t wakeref; struct drm_mm_node tmp; @@ -1279,6 +1435,7 @@ static void track_vma_bind(struct i915_vma *vma) atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE); __i915_gem_object_pin_pages(obj); vma->pages = obj->mm.pages; + vma->resource->bi.pages = vma->pages; mutex_lock(&vma->vm->mutex); list_add_tail(&vma->vm_link, &vma->vm->bound_list); @@ -1336,6 +1493,33 @@ static int igt_mock_drunk(void *arg) return exercise_mock(ggtt->vm.i915, drunk_hole); } +static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset) +{ + struct i915_address_space *vm = vma->vm; + struct i915_vma_resource *vma_res; + struct drm_i915_gem_object *obj = vma->obj; + int err; + + vma_res = i915_vma_resource_alloc(); + if (IS_ERR(vma_res)) + return PTR_ERR(vma_res); + + mutex_lock(&vm->mutex); + err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size, + offset, + obj->cache_level, + 0); + if (!err) { + i915_vma_resource_init_from_vma(vma_res, vma); + vma->resource = vma_res; + } else { + kfree(vma_res); + } + mutex_unlock(&vm->mutex); + + return err; +} + static int igt_gtt_reserve(void *arg) { struct i915_ggtt *ggtt = arg; @@ -1370,20 +1554,13 @@ static int igt_gtt_reserve(void *arg) } list_add(&obj->st_link, &objects); - vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, - obj->base.size, - total, - obj->cache_level, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = reserve_gtt_with_resource(vma, total); if (err) { pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1429,13 +1606,7 @@ static int igt_gtt_reserve(void *arg) goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, - obj->base.size, - total, - obj->cache_level, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = reserve_gtt_with_resource(vma, total); if (err) { pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1465,7 +1636,7 @@ static int igt_gtt_reserve(void *arg) goto out; } - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("i915_vma_unbind failed with err=%d!\n", err); goto out; @@ -1476,13 +1647,7 @@ static int igt_gtt_reserve(void *arg) 2 * I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT); - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, - obj->base.size, - offset, - obj->cache_level, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = reserve_gtt_with_resource(vma, offset); if (err) { pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1509,6 +1674,31 @@ out: return err; } +static int insert_gtt_with_resource(struct i915_vma *vma) +{ + struct i915_address_space *vm = vma->vm; + struct i915_vma_resource *vma_res; + struct drm_i915_gem_object *obj = vma->obj; + int err; + + vma_res = i915_vma_resource_alloc(); + if (IS_ERR(vma_res)) + return PTR_ERR(vma_res); + + mutex_lock(&vm->mutex); + err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0, + obj->cache_level, 0, vm->total, 0); + if (!err) { + i915_vma_resource_init_from_vma(vma_res, vma); + vma->resource = vma_res; + } else { + kfree(vma_res); + } + mutex_unlock(&vm->mutex); + + return err; +} + static int igt_gtt_insert(void *arg) { struct i915_ggtt *ggtt = arg; @@ -1552,7 +1742,7 @@ static int igt_gtt_insert(void *arg) /* Check a couple of obviously invalid requests */ for (ii = invalid_insert; ii->size; ii++) { mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &tmp, + err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp, ii->size, ii->alignment, I915_COLOR_UNEVICTABLE, ii->start, ii->end, @@ -1593,12 +1783,7 @@ static int igt_gtt_insert(void *arg) goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, - obj->base.size, 0, obj->cache_level, - 0, ggtt->vm.total, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = insert_gtt_with_resource(vma); if (err == -ENOSPC) { /* maxed out the GGTT space */ i915_gem_object_put(obj); @@ -1647,18 +1832,13 @@ static int igt_gtt_insert(void *arg) GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); offset = vma->node.start; - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("i915_vma_unbind failed with err=%d!\n", err); goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, - obj->base.size, 0, obj->cache_level, - 0, ggtt->vm.total, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = insert_gtt_with_resource(vma); if (err) { pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1702,12 +1882,7 @@ static int igt_gtt_insert(void *arg) goto out; } - mutex_lock(&ggtt->vm.mutex); - err = i915_gem_gtt_insert(&ggtt->vm, &vma->node, - obj->base.size, 0, obj->cache_level, - 0, ggtt->vm.total, - 0); - mutex_unlock(&ggtt->vm.mutex); + err = insert_gtt_with_resource(vma); if (err) { pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", total, ggtt->vm.total, err); @@ -1737,26 +1912,28 @@ int i915_gem_gtt_mock_selftests(void) SUBTEST(igt_gtt_insert), }; struct drm_i915_private *i915; - struct i915_ggtt *ggtt; + struct intel_gt *gt; int err; i915 = mock_gem_device(); if (!i915) return -ENOMEM; - ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL); - if (!ggtt) { - err = -ENOMEM; + /* allocate the ggtt */ + err = intel_gt_assign_ggtt(to_gt(i915)); + if (err) goto out_put; - } - mock_init_ggtt(i915, ggtt); - err = i915_subtests(tests, ggtt); + gt = to_gt(i915); + + mock_init_ggtt(gt); + + err = i915_subtests(tests, gt->ggtt); mock_device_flush(i915); i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(ggtt); - kfree(ggtt); + mock_fini_ggtt(gt->ggtt); + out_put: mock_destroy_device(i915); return err; @@ -1939,6 +2116,7 @@ static int igt_cs_tlb(void *arg) struct i915_vm_pt_stash stash = {}; struct i915_request *rq; struct i915_gem_ww_ctx ww; + struct i915_vma_resource *vma_res; u64 offset; offset = igt_random_offset(&prng, @@ -1959,6 +2137,13 @@ static int igt_cs_tlb(void *arg) if (err) goto end; + vma_res = i915_vma_resource_alloc(); + if (IS_ERR(vma_res)) { + i915_vma_put_pages(vma); + err = PTR_ERR(vma_res); + goto end; + } + i915_gem_ww_ctx_init(&ww, false); retry: err = i915_vm_lock_objects(vm, &ww); @@ -1980,33 +2165,41 @@ end_ww: goto retry; } i915_gem_ww_ctx_fini(&ww); - if (err) + if (err) { + kfree(vma_res); goto end; + } + i915_vma_resource_init_from_vma(vma_res, vma); /* Prime the TLB with the dummy pages */ for (i = 0; i < count; i++) { - vma->node.start = offset + i * PAGE_SIZE; - vm->insert_entries(vm, vma, I915_CACHE_NONE, 0); + vma_res->start = offset + i * PAGE_SIZE; + vm->insert_entries(vm, vma_res, I915_CACHE_NONE, + 0); - rq = submit_batch(ce, vma->node.start); + rq = submit_batch(ce, vma_res->start); if (IS_ERR(rq)) { err = PTR_ERR(rq); + i915_vma_resource_fini(vma_res); + kfree(vma_res); goto end; } i915_request_put(rq); } - + i915_vma_resource_fini(vma_res); i915_vma_put_pages(vma); err = context_sync(ce); if (err) { pr_err("%s: dummy setup timed out\n", ce->engine->name); + kfree(vma_res); goto end; } vma = i915_vma_instance(act, vm, NULL); if (IS_ERR(vma)) { + kfree(vma_res); err = PTR_ERR(vma); goto end; } @@ -2014,19 +2207,22 @@ end_ww: i915_gem_object_lock(act, NULL); err = i915_vma_get_pages(vma); i915_gem_object_unlock(act); - if (err) + if (err) { + kfree(vma_res); goto end; + } + i915_vma_resource_init_from_vma(vma_res, vma); /* Replace the TLB with target batches */ for (i = 0; i < count; i++) { struct i915_request *rq; u32 *cs = batch + i * 64 / sizeof(*cs); u64 addr; - vma->node.start = offset + i * PAGE_SIZE; - vm->insert_entries(vm, vma, I915_CACHE_NONE, 0); + vma_res->start = offset + i * PAGE_SIZE; + vm->insert_entries(vm, vma_res, I915_CACHE_NONE, 0); - addr = vma->node.start + i * 64; + addr = vma_res->start + i * 64; cs[4] = MI_NOOP; cs[6] = lower_32_bits(addr); cs[7] = upper_32_bits(addr); @@ -2035,6 +2231,8 @@ end_ww: rq = submit_batch(ce, addr); if (IS_ERR(rq)) { err = PTR_ERR(rq); + i915_vma_resource_fini(vma_res); + kfree(vma_res); goto end; } @@ -2051,6 +2249,8 @@ end_ww: } end_spin(batch, count - 1); + i915_vma_resource_fini(vma_res); + kfree(vma_res); i915_vma_put_pages(vma); err = context_sync(ce); @@ -2105,16 +2305,18 @@ int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_ppgtt_fill), SUBTEST(igt_ppgtt_shrink), SUBTEST(igt_ppgtt_shrink_boom), + SUBTEST(igt_ppgtt_misaligned_pin), SUBTEST(igt_ggtt_lowlevel), SUBTEST(igt_ggtt_drunk), SUBTEST(igt_ggtt_walk), SUBTEST(igt_ggtt_pot), SUBTEST(igt_ggtt_fill), SUBTEST(igt_ggtt_page), + SUBTEST(igt_ggtt_misaligned_pin), SUBTEST(igt_cs_tlb), }; - GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total)); + GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total)); return i915_subtests(tests, i915); } diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h index 793fb28a770d..0c22e0fc9059 100644 --- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h @@ -33,4 +33,3 @@ selftest(evict, i915_gem_evict_mock_selftests) selftest(gtt, i915_gem_gtt_mock_selftests) selftest(hugepages, i915_gem_huge_page_mock_selftests) selftest(memory_region, intel_memory_region_mock_selftests) -selftest(buddy, i915_buddy_mock_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index 92a859b34190..c56a0c2cd2f7 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -26,6 +26,7 @@ #include <linux/pm_qos.h> #include <linux/sort.h> +#include "gem/i915_gem_internal.h" #include "gem/i915_gem_pm.h" #include "gem/selftests/mock_context.h" @@ -782,6 +783,115 @@ out_spin: return err; } +/* + * Test to prove a non-preemptable request can be cancelled and a subsequent + * request on the same context can successfully complete after cancellation. + * + * Testing methodology is to create a non-preemptible request and submit it, + * wait for spinner to start, create a NOP request and submit it, cancel the + * spinner, wait for spinner to complete and verify it failed with an error, + * finally wait for NOP request to complete verify it succeeded without an + * error. Preemption timeout also reduced / restored so test runs in a timely + * maner. + */ +static int __cancel_reset(struct drm_i915_private *i915, + struct intel_engine_cs *engine) +{ + struct intel_context *ce; + struct igt_spinner spin; + struct i915_request *rq, *nop; + unsigned long preempt_timeout_ms; + int err = 0; + + if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT || + !intel_has_reset_engine(engine->gt)) + return 0; + + preempt_timeout_ms = engine->props.preempt_timeout_ms; + engine->props.preempt_timeout_ms = 100; + + if (igt_spinner_init(&spin, engine->gt)) + goto out_restore; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + goto out_spin; + } + + rq = igt_spinner_create_request(&spin, ce, MI_NOOP); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_ce; + } + + pr_debug("%s: Cancelling active non-preemptable request\n", + engine->name); + i915_request_get(rq); + i915_request_add(rq); + if (!igt_wait_for_spinner(&spin, rq)) { + struct drm_printer p = drm_info_printer(engine->i915->drm.dev); + + pr_err("Failed to start spinner on %s\n", engine->name); + intel_engine_dump(engine, &p, "%s\n", engine->name); + err = -ETIME; + goto out_rq; + } + + nop = intel_context_create_request(ce); + if (IS_ERR(nop)) + goto out_rq; + i915_request_get(nop); + i915_request_add(nop); + + i915_request_cancel(rq, -EINTR); + + if (i915_request_wait(rq, 0, HZ) < 0) { + struct drm_printer p = drm_info_printer(engine->i915->drm.dev); + + pr_err("%s: Failed to cancel hung request\n", engine->name); + intel_engine_dump(engine, &p, "%s\n", engine->name); + err = -ETIME; + goto out_nop; + } + + if (rq->fence.error != -EINTR) { + pr_err("%s: fence not cancelled (%u)\n", + engine->name, rq->fence.error); + err = -EINVAL; + goto out_nop; + } + + if (i915_request_wait(nop, 0, HZ) < 0) { + struct drm_printer p = drm_info_printer(engine->i915->drm.dev); + + pr_err("%s: Failed to complete nop request\n", engine->name); + intel_engine_dump(engine, &p, "%s\n", engine->name); + err = -ETIME; + goto out_nop; + } + + if (nop->fence.error != 0) { + pr_err("%s: Nop request errored (%u)\n", + engine->name, nop->fence.error); + err = -EINVAL; + } + +out_nop: + i915_request_put(nop); +out_rq: + i915_request_put(rq); +out_ce: + intel_context_put(ce); +out_spin: + igt_spinner_fini(&spin); +out_restore: + engine->props.preempt_timeout_ms = preempt_timeout_ms; + if (err) + pr_err("%s: %s error %d\n", __func__, engine->name, err); + return err; +} + static int live_cancel_request(void *arg) { struct drm_i915_private *i915 = arg; @@ -814,6 +924,14 @@ static int live_cancel_request(void *arg) return err; if (err2) return err2; + + /* Expects reset so call outside of igt_live_test_* */ + err = __cancel_reset(i915, engine); + if (err) + return err; + + if (igt_flush_test(i915)) + return -EIO; } return 0; @@ -843,7 +961,7 @@ static struct i915_vma *empty_batch(struct drm_i915_private *i915) intel_gt_chipset_flush(to_gt(i915)); - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &to_gt(i915)->ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto err; diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c index 2d6d7bd13c3c..c4e932368b37 100644 --- a/drivers/gpu/drm/i915/selftests/i915_selftest.c +++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c @@ -24,6 +24,7 @@ #include <linux/random.h> #include "gt/intel_gt_pm.h" +#include "i915_driver.h" #include "i915_drv.h" #include "i915_selftest.h" diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index 5c5809dfe9b2..6921ba128015 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -25,6 +25,7 @@ #include <linux/prime_numbers.h> #include "gem/i915_gem_context.h" +#include "gem/i915_gem_internal.h" #include "gem/selftests/mock_context.h" #include "i915_scatterlist.h" @@ -340,7 +341,7 @@ static int igt_vma_pin1(void *arg) if (!err) { i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("Failed to unbind single page from GGTT, err=%d\n", err); goto out; @@ -691,7 +692,7 @@ static int igt_vma_rotate_remap(void *arg) } i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("Unbinding returned %i\n", err); goto out_object; @@ -852,7 +853,7 @@ static int igt_vma_partial(void *arg) i915_vma_unpin(vma); nvma++; - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("Unbinding returned %i\n", err); goto out_object; @@ -891,7 +892,7 @@ static int igt_vma_partial(void *arg) i915_vma_unpin(vma); - err = i915_vma_unbind(vma); + err = i915_vma_unbind_unlocked(vma); if (err) { pr_err("Unbinding returned %i\n", err); goto out_object; @@ -922,26 +923,28 @@ int i915_vma_mock_selftests(void) SUBTEST(igt_vma_partial), }; struct drm_i915_private *i915; - struct i915_ggtt *ggtt; + struct intel_gt *gt; int err; i915 = mock_gem_device(); if (!i915) return -ENOMEM; - ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL); - if (!ggtt) { - err = -ENOMEM; + /* allocate the ggtt */ + err = intel_gt_assign_ggtt(to_gt(i915)); + if (err) goto out_put; - } - mock_init_ggtt(i915, ggtt); - err = i915_subtests(tests, ggtt); + gt = to_gt(i915); + + mock_init_ggtt(gt); + + err = i915_subtests(tests, gt->ggtt); mock_device_flush(i915); i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(ggtt); - kfree(ggtt); + mock_fini_ggtt(gt->ggtt); + out_put: mock_destroy_device(i915); return err; @@ -982,7 +985,7 @@ static int igt_vma_remapped_gtt(void *arg) intel_wakeref_t wakeref; int err = 0; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return 0; obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE); diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c index b84594601d30..b484e12df417 100644 --- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c +++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c @@ -19,7 +19,7 @@ int igt_flush_test(struct drm_i915_private *i915) cond_resched(); - if (intel_gt_wait_for_idle(gt, HZ) == -ETIME) { + if (intel_gt_wait_for_idle(gt, HZ * 3) == -ETIME) { pr_err("%pS timed out, cancelling all further testing.\n", __builtin_return_address(0)); diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.c b/drivers/gpu/drm/i915/selftests/igt_spinner.c index 24d87d0fc747..0c22594ae274 100644 --- a/drivers/gpu/drm/i915/selftests/igt_spinner.c +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.c @@ -6,6 +6,7 @@ #include "gt/intel_gpu_commands.h" #include "gt/intel_gt.h" +#include "gem/i915_gem_internal.h" #include "gem/selftests/igt_gem_utils.h" #include "igt_spinner.h" diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index 8255561ff853..ba32893e0873 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -6,6 +6,8 @@ #include <linux/prime_numbers.h> #include <linux/sort.h> +#include <drm/drm_buddy.h> + #include "../i915_selftest.h" #include "mock_drm.h" @@ -15,12 +17,12 @@ #include "gem/i915_gem_context.h" #include "gem/i915_gem_lmem.h" #include "gem/i915_gem_region.h" +#include "gem/i915_gem_ttm.h" #include "gem/selftests/igt_gem_utils.h" #include "gem/selftests/mock_context.h" #include "gt/intel_engine_pm.h" #include "gt/intel_engine_user.h" #include "gt/intel_gt.h" -#include "i915_buddy.h" #include "gt/intel_migrate.h" #include "i915_memcpy.h" #include "i915_ttm_buddy_manager.h" @@ -169,7 +171,7 @@ static int igt_mock_reserve(void *arg) if (!order) return 0; - mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0); if (IS_ERR(mem)) { pr_err("failed to create memory region\n"); err = PTR_ERR(mem); @@ -369,7 +371,7 @@ static int igt_mock_splintered_region(void *arg) struct drm_i915_private *i915 = mem->i915; struct i915_ttm_buddy_resource *res; struct drm_i915_gem_object *obj; - struct i915_buddy_mm *mm; + struct drm_buddy *mm; unsigned int expected_order; LIST_HEAD(objects); u64 size; @@ -382,7 +384,7 @@ static int igt_mock_splintered_region(void *arg) */ size = (SZ_4G - 1) & PAGE_MASK; - mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); + mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0, 0); if (IS_ERR(mem)) return PTR_ERR(mem); @@ -454,8 +456,8 @@ static int igt_mock_max_segment(void *arg) struct drm_i915_private *i915 = mem->i915; struct i915_ttm_buddy_resource *res; struct drm_i915_gem_object *obj; - struct i915_buddy_block *block; - struct i915_buddy_mm *mm; + struct drm_buddy_block *block; + struct drm_buddy *mm; struct list_head *blocks; struct scatterlist *sg; LIST_HEAD(objects); @@ -470,7 +472,7 @@ static int igt_mock_max_segment(void *arg) */ size = SZ_8G; - mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0); + mem = mock_region_create(i915, 0, size, PAGE_SIZE, 0, 0); if (IS_ERR(mem)) return PTR_ERR(mem); @@ -485,8 +487,8 @@ static int igt_mock_max_segment(void *arg) mm = res->mm; size = 0; list_for_each_entry(block, blocks, link) { - if (i915_buddy_block_size(mm, block) > size) - size = i915_buddy_block_size(mm, block); + if (drm_buddy_block_size(mm, block) > size) + size = drm_buddy_block_size(mm, block); } if (size < max_segment) { pr_err("%s: Failed to create a huge contiguous block [> %u], largest block %lld\n", @@ -511,6 +513,147 @@ out_put: return err; } +static u64 igt_object_mappable_total(struct drm_i915_gem_object *obj) +{ + struct intel_memory_region *mr = obj->mm.region; + struct i915_ttm_buddy_resource *bman_res = + to_ttm_buddy_resource(obj->mm.res); + struct drm_buddy *mm = bman_res->mm; + struct drm_buddy_block *block; + u64 total; + + total = 0; + list_for_each_entry(block, &bman_res->blocks, link) { + u64 start = drm_buddy_block_offset(block); + u64 end = start + drm_buddy_block_size(mm, block); + + if (start < mr->io_size) + total += min_t(u64, end, mr->io_size) - start; + } + + return total; +} + +static int igt_mock_io_size(void *arg) +{ + struct intel_memory_region *mr = arg; + struct drm_i915_private *i915 = mr->i915; + struct drm_i915_gem_object *obj; + u64 mappable_theft_total; + u64 io_size; + u64 total; + u64 ps; + u64 rem; + u64 size; + I915_RND_STATE(prng); + LIST_HEAD(objects); + int err = 0; + + ps = SZ_4K; + if (i915_prandom_u64_state(&prng) & 1) + ps = SZ_64K; /* For something like DG2 */ + + div64_u64_rem(i915_prandom_u64_state(&prng), SZ_8G, &total); + total = round_down(total, ps); + total = max_t(u64, total, SZ_1G); + + div64_u64_rem(i915_prandom_u64_state(&prng), total - ps, &io_size); + io_size = round_down(io_size, ps); + io_size = max_t(u64, io_size, SZ_256M); /* 256M seems to be the common lower limit */ + + pr_info("%s with ps=%llx, io_size=%llx, total=%llx\n", + __func__, ps, io_size, total); + + mr = mock_region_create(i915, 0, total, ps, 0, io_size); + if (IS_ERR(mr)) { + err = PTR_ERR(mr); + goto out_err; + } + + mappable_theft_total = 0; + rem = total - io_size; + do { + div64_u64_rem(i915_prandom_u64_state(&prng), rem, &size); + size = round_down(size, ps); + size = max(size, ps); + + obj = igt_object_create(mr, &objects, size, + I915_BO_ALLOC_GPU_ONLY); + if (IS_ERR(obj)) { + pr_err("%s TOPDOWN failed with rem=%llx, size=%llx\n", + __func__, rem, size); + err = PTR_ERR(obj); + goto out_close; + } + + mappable_theft_total += igt_object_mappable_total(obj); + rem -= size; + } while (rem); + + pr_info("%s mappable theft=(%lluMiB/%lluMiB), total=%lluMiB\n", + __func__, + (u64)mappable_theft_total >> 20, + (u64)io_size >> 20, + (u64)total >> 20); + + /* + * Even if we allocate all of the non-mappable portion, we should still + * be able to dip into the mappable portion. + */ + obj = igt_object_create(mr, &objects, io_size, + I915_BO_ALLOC_GPU_ONLY); + if (IS_ERR(obj)) { + pr_err("%s allocation unexpectedly failed\n", __func__); + err = PTR_ERR(obj); + goto out_close; + } + + close_objects(mr, &objects); + + rem = io_size; + do { + div64_u64_rem(i915_prandom_u64_state(&prng), rem, &size); + size = round_down(size, ps); + size = max(size, ps); + + obj = igt_object_create(mr, &objects, size, 0); + if (IS_ERR(obj)) { + pr_err("%s MAPPABLE failed with rem=%llx, size=%llx\n", + __func__, rem, size); + err = PTR_ERR(obj); + goto out_close; + } + + if (igt_object_mappable_total(obj) != size) { + pr_err("%s allocation is not mappable(size=%llx)\n", + __func__, size); + err = -EINVAL; + goto out_close; + } + rem -= size; + } while (rem); + + /* + * We assume CPU access is required by default, which should result in a + * failure here, even though the non-mappable portion is free. + */ + obj = igt_object_create(mr, &objects, ps, 0); + if (!IS_ERR(obj)) { + pr_err("%s allocation unexpectedly succeeded\n", __func__); + err = -EINVAL; + goto out_close; + } + +out_close: + close_objects(mr, &objects); + intel_memory_region_destroy(mr); +out_err: + if (err == -ENOMEM) + err = 0; + + return err; +} + static int igt_gpu_write_dw(struct intel_context *ce, struct i915_vma *vma, u32 dword, @@ -679,8 +822,14 @@ static int igt_lmem_create_with_ps(void *arg) i915_gem_object_lock(obj, NULL); err = i915_gem_object_pin_pages(obj); - if (err) + if (err) { + if (err == -ENXIO || err == -E2BIG || err == -ENOMEM) { + pr_info("%s not enough lmem for ps(%u) err=%d\n", + __func__, ps, err); + err = 0; + } goto out_put; + } daddr = i915_gem_object_get_dma_address(obj, 0); if (!IS_ALIGNED(daddr, ps)) { @@ -1178,6 +1327,7 @@ int intel_memory_region_mock_selftests(void) SUBTEST(igt_mock_contiguous), SUBTEST(igt_mock_splintered_region), SUBTEST(igt_mock_max_segment), + SUBTEST(igt_mock_io_size), }; struct intel_memory_region *mem; struct drm_i915_private *i915; @@ -1187,7 +1337,7 @@ int intel_memory_region_mock_selftests(void) if (!i915) return -ENOMEM; - mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0); + mem = mock_region_create(i915, 0, SZ_2G, I915_GTT_PAGE_SIZE_4K, 0, 0); if (IS_ERR(mem)) { pr_err("failed to create memory region\n"); err = PTR_ERR(mem); diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index 8aa7b1d33865..573d9b2e1a4a 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -69,7 +69,7 @@ static void mock_device_release(struct drm_device *dev) i915_gem_drain_workqueue(i915); i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(&i915->ggtt); + mock_fini_ggtt(to_gt(i915)->ggtt); destroy_workqueue(i915->wq); intel_region_ttm_device_fini(i915); @@ -161,6 +161,8 @@ struct drm_i915_private *mock_gem_device(void) i915_params_copy(&i915->params, &i915_modparams); intel_runtime_pm_init_early(&i915->runtime_pm); + /* wakeref tracking has significant overhead */ + i915->runtime_pm.no_wakeref_tracking = true; /* Using the global GTT may ask questions about KMS users, so prepare */ drm_mode_config_init(&i915->drm); @@ -194,8 +196,13 @@ struct drm_i915_private *mock_gem_device(void) mock_init_contexts(i915); - mock_init_ggtt(i915, &i915->ggtt); - to_gt(i915)->vm = i915_vm_get(&i915->ggtt.vm); + /* allocate the ggtt */ + ret = intel_gt_assign_ggtt(to_gt(i915)); + if (ret) + goto err_unlock; + + mock_init_ggtt(to_gt(i915)); + to_gt(i915)->vm = i915_vm_get(&to_gt(i915)->ggtt->vm); mkwrite_device_info(i915)->platform_engine_mask = BIT(0); to_gt(i915)->info.engine_mask = BIT(0); diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c index 1802baf80a17..568840e7ca66 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.c +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c @@ -33,23 +33,23 @@ static void mock_insert_page(struct i915_address_space *vm, } static void mock_insert_entries(struct i915_address_space *vm, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level level, u32 flags) { } static void mock_bind_ppgtt(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags) { GEM_BUG_ON(flags & I915_VMA_GLOBAL_BIND); - set_bit(I915_VMA_LOCAL_BIND_BIT, __i915_vma_flags(vma)); + vma_res->bound_flags |= flags; } static void mock_unbind_ppgtt(struct i915_address_space *vm, - struct i915_vma *vma) + struct i915_vma_resource *vma_res) { } @@ -93,23 +93,23 @@ struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name) static void mock_bind_ggtt(struct i915_address_space *vm, struct i915_vm_pt_stash *stash, - struct i915_vma *vma, + struct i915_vma_resource *vma_res, enum i915_cache_level cache_level, u32 flags) { } static void mock_unbind_ggtt(struct i915_address_space *vm, - struct i915_vma *vma) + struct i915_vma_resource *vma_res) { } -void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt) +void mock_init_ggtt(struct intel_gt *gt) { - memset(ggtt, 0, sizeof(*ggtt)); + struct i915_ggtt *ggtt = gt->ggtt; - ggtt->vm.gt = to_gt(i915); - ggtt->vm.i915 = i915; + ggtt->vm.gt = gt; + ggtt->vm.i915 = gt->i915; ggtt->vm.is_ggtt = true; ggtt->gmadr = (struct resource) DEFINE_RES_MEM(0, 2048 * PAGE_SIZE); @@ -128,7 +128,6 @@ void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt) ggtt->vm.vma_ops.unbind_vma = mock_unbind_ggtt; i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); - to_gt(i915)->ggtt = ggtt; } void mock_fini_ggtt(struct i915_ggtt *ggtt) diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.h b/drivers/gpu/drm/i915/selftests/mock_gtt.h index e3f224f43beb..d6eb90bd7f3f 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.h +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.h @@ -27,8 +27,9 @@ struct drm_i915_private; struct i915_ggtt; +struct intel_gt; -void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt); +void mock_init_ggtt(struct intel_gt *gt); void mock_fini_ggtt(struct i915_ggtt *ggtt); struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name); diff --git a/drivers/gpu/drm/i915/selftests/mock_region.c b/drivers/gpu/drm/i915/selftests/mock_region.c index 19bff8afcaaa..f64325491f35 100644 --- a/drivers/gpu/drm/i915/selftests/mock_region.c +++ b/drivers/gpu/drm/i915/selftests/mock_region.c @@ -22,17 +22,12 @@ static void mock_region_put_pages(struct drm_i915_gem_object *obj, static int mock_region_get_pages(struct drm_i915_gem_object *obj) { - unsigned int flags; struct sg_table *pages; int err; - flags = 0; - if (obj->flags & I915_BO_ALLOC_CONTIGUOUS) - flags |= TTM_PL_FLAG_CONTIGUOUS; - obj->mm.res = intel_region_ttm_resource_alloc(obj->mm.region, obj->base.size, - flags); + obj->flags); if (IS_ERR(obj->mm.res)) return PTR_ERR(obj->mm.res); @@ -107,7 +102,8 @@ mock_region_create(struct drm_i915_private *i915, resource_size_t start, resource_size_t size, resource_size_t min_page_size, - resource_size_t io_start) + resource_size_t io_start, + resource_size_t io_size) { int instance = ida_alloc_max(&i915->selftest.mock_region_instances, TTM_NUM_MEM_TYPES - TTM_PL_PRIV - 1, @@ -117,6 +113,7 @@ mock_region_create(struct drm_i915_private *i915, return ERR_PTR(instance); return intel_memory_region_create(i915, start, size, min_page_size, - io_start, INTEL_MEMORY_MOCK, instance, + io_start, io_size, + INTEL_MEMORY_MOCK, instance, &mock_region_ops); } diff --git a/drivers/gpu/drm/i915/selftests/mock_region.h b/drivers/gpu/drm/i915/selftests/mock_region.h index 329bf74dfaca..e36c3a433551 100644 --- a/drivers/gpu/drm/i915/selftests/mock_region.h +++ b/drivers/gpu/drm/i915/selftests/mock_region.h @@ -16,6 +16,7 @@ mock_region_create(struct drm_i915_private *i915, resource_size_t start, resource_size_t size, resource_size_t min_page_size, - resource_size_t io_start); + resource_size_t io_start, + resource_size_t io_size); #endif /* !__MOCK_REGION_H */ diff --git a/drivers/gpu/drm/i915/vlv_sideband.c b/drivers/gpu/drm/i915/vlv_sideband.c index ed2ac5752ac4..c26001300ebd 100644 --- a/drivers/gpu/drm/i915/vlv_sideband.c +++ b/drivers/gpu/drm/i915/vlv_sideband.c @@ -5,6 +5,7 @@ #include "i915_drv.h" #include "i915_iosf_mbi.h" +#include "i915_reg.h" #include "vlv_sideband.h" /* diff --git a/drivers/gpu/drm/i915/vlv_sideband.h b/drivers/gpu/drm/i915/vlv_sideband.h index d7732f612e7f..9ce283d96b80 100644 --- a/drivers/gpu/drm/i915/vlv_sideband.h +++ b/drivers/gpu/drm/i915/vlv_sideband.h @@ -9,6 +9,8 @@ #include <linux/bitops.h> #include <linux/types.h> +#include "vlv_sideband_reg.h" + enum pipe; struct drm_i915_private; diff --git a/drivers/gpu/drm/i915/vlv_sideband_reg.h b/drivers/gpu/drm/i915/vlv_sideband_reg.h new file mode 100644 index 000000000000..b7fbff3d0409 --- /dev/null +++ b/drivers/gpu/drm/i915/vlv_sideband_reg.h @@ -0,0 +1,180 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _VLV_SIDEBAND_REG_H_ +#define _VLV_SIDEBAND_REG_H_ + +/* See configdb bunit SB addr map */ +#define BUNIT_REG_BISOC 0x11 + +/* PUNIT_REG_*SSPM0 */ +#define _SSPM0_SSC(val) ((val) << 0) +#define SSPM0_SSC_MASK _SSPM0_SSC(0x3) +#define SSPM0_SSC_PWR_ON _SSPM0_SSC(0x0) +#define SSPM0_SSC_CLK_GATE _SSPM0_SSC(0x1) +#define SSPM0_SSC_RESET _SSPM0_SSC(0x2) +#define SSPM0_SSC_PWR_GATE _SSPM0_SSC(0x3) +#define _SSPM0_SSS(val) ((val) << 24) +#define SSPM0_SSS_MASK _SSPM0_SSS(0x3) +#define SSPM0_SSS_PWR_ON _SSPM0_SSS(0x0) +#define SSPM0_SSS_CLK_GATE _SSPM0_SSS(0x1) +#define SSPM0_SSS_RESET _SSPM0_SSS(0x2) +#define SSPM0_SSS_PWR_GATE _SSPM0_SSS(0x3) + +/* PUNIT_REG_*SSPM1 */ +#define SSPM1_FREQSTAT_SHIFT 24 +#define SSPM1_FREQSTAT_MASK (0x1f << SSPM1_FREQSTAT_SHIFT) +#define SSPM1_FREQGUAR_SHIFT 8 +#define SSPM1_FREQGUAR_MASK (0x1f << SSPM1_FREQGUAR_SHIFT) +#define SSPM1_FREQ_SHIFT 0 +#define SSPM1_FREQ_MASK (0x1f << SSPM1_FREQ_SHIFT) + +#define PUNIT_REG_VEDSSPM0 0x32 +#define PUNIT_REG_VEDSSPM1 0x33 + +#define PUNIT_REG_DSPSSPM 0x36 +#define DSPFREQSTAT_SHIFT_CHV 24 +#define DSPFREQSTAT_MASK_CHV (0x1f << DSPFREQSTAT_SHIFT_CHV) +#define DSPFREQGUAR_SHIFT_CHV 8 +#define DSPFREQGUAR_MASK_CHV (0x1f << DSPFREQGUAR_SHIFT_CHV) +#define DSPFREQSTAT_SHIFT 30 +#define DSPFREQSTAT_MASK (0x3 << DSPFREQSTAT_SHIFT) +#define DSPFREQGUAR_SHIFT 14 +#define DSPFREQGUAR_MASK (0x3 << DSPFREQGUAR_SHIFT) +#define DSP_MAXFIFO_PM5_STATUS (1 << 22) /* chv */ +#define DSP_AUTO_CDCLK_GATE_DISABLE (1 << 7) /* chv */ +#define DSP_MAXFIFO_PM5_ENABLE (1 << 6) /* chv */ +#define _DP_SSC(val, pipe) ((val) << (2 * (pipe))) +#define DP_SSC_MASK(pipe) _DP_SSC(0x3, (pipe)) +#define DP_SSC_PWR_ON(pipe) _DP_SSC(0x0, (pipe)) +#define DP_SSC_CLK_GATE(pipe) _DP_SSC(0x1, (pipe)) +#define DP_SSC_RESET(pipe) _DP_SSC(0x2, (pipe)) +#define DP_SSC_PWR_GATE(pipe) _DP_SSC(0x3, (pipe)) +#define _DP_SSS(val, pipe) ((val) << (2 * (pipe) + 16)) +#define DP_SSS_MASK(pipe) _DP_SSS(0x3, (pipe)) +#define DP_SSS_PWR_ON(pipe) _DP_SSS(0x0, (pipe)) +#define DP_SSS_CLK_GATE(pipe) _DP_SSS(0x1, (pipe)) +#define DP_SSS_RESET(pipe) _DP_SSS(0x2, (pipe)) +#define DP_SSS_PWR_GATE(pipe) _DP_SSS(0x3, (pipe)) + +#define PUNIT_REG_ISPSSPM0 0x39 +#define PUNIT_REG_ISPSSPM1 0x3a + +#define PUNIT_REG_PWRGT_CTRL 0x60 +#define PUNIT_REG_PWRGT_STATUS 0x61 +#define PUNIT_PWRGT_MASK(pw_idx) (3 << ((pw_idx) * 2)) +#define PUNIT_PWRGT_PWR_ON(pw_idx) (0 << ((pw_idx) * 2)) +#define PUNIT_PWRGT_CLK_GATE(pw_idx) (1 << ((pw_idx) * 2)) +#define PUNIT_PWRGT_RESET(pw_idx) (2 << ((pw_idx) * 2)) +#define PUNIT_PWRGT_PWR_GATE(pw_idx) (3 << ((pw_idx) * 2)) + +#define PUNIT_PWGT_IDX_RENDER 0 +#define PUNIT_PWGT_IDX_MEDIA 1 +#define PUNIT_PWGT_IDX_DISP2D 3 +#define PUNIT_PWGT_IDX_DPIO_CMN_BC 5 +#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01 6 +#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23 7 +#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01 8 +#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23 9 +#define PUNIT_PWGT_IDX_DPIO_RX0 10 +#define PUNIT_PWGT_IDX_DPIO_RX1 11 +#define PUNIT_PWGT_IDX_DPIO_CMN_D 12 + +#define PUNIT_REG_GPU_LFM 0xd3 +#define PUNIT_REG_GPU_FREQ_REQ 0xd4 +#define PUNIT_REG_GPU_FREQ_STS 0xd8 +#define GPLLENABLE (1 << 4) +#define GENFREQSTATUS (1 << 0) +#define PUNIT_REG_MEDIA_TURBO_FREQ_REQ 0xdc +#define PUNIT_REG_CZ_TIMESTAMP 0xce + +#define PUNIT_FUSE_BUS2 0xf6 /* bits 47:40 */ +#define PUNIT_FUSE_BUS1 0xf5 /* bits 55:48 */ + +#define FB_GFX_FMAX_AT_VMAX_FUSE 0x136 +#define FB_GFX_FREQ_FUSE_MASK 0xff +#define FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT 24 +#define FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT 16 +#define FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT 8 + +#define FB_GFX_FMIN_AT_VMIN_FUSE 0x137 +#define FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT 8 + +#define PUNIT_REG_DDR_SETUP2 0x139 +#define FORCE_DDR_FREQ_REQ_ACK (1 << 8) +#define FORCE_DDR_LOW_FREQ (1 << 1) +#define FORCE_DDR_HIGH_FREQ (1 << 0) + +#define PUNIT_GPU_STATUS_REG 0xdb +#define PUNIT_GPU_STATUS_MAX_FREQ_SHIFT 16 +#define PUNIT_GPU_STATUS_MAX_FREQ_MASK 0xff +#define PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT 8 +#define PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK 0xff + +#define PUNIT_GPU_DUTYCYCLE_REG 0xdf +#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT 8 +#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK 0xff + +#define IOSF_NC_FB_GFX_FREQ_FUSE 0x1c +#define FB_GFX_MAX_FREQ_FUSE_SHIFT 3 +#define FB_GFX_MAX_FREQ_FUSE_MASK 0x000007f8 +#define FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT 11 +#define FB_GFX_FGUARANTEED_FREQ_FUSE_MASK 0x0007f800 +#define IOSF_NC_FB_GFX_FMAX_FUSE_HI 0x34 +#define FB_FMAX_VMIN_FREQ_HI_MASK 0x00000007 +#define IOSF_NC_FB_GFX_FMAX_FUSE_LO 0x30 +#define FB_FMAX_VMIN_FREQ_LO_SHIFT 27 +#define FB_FMAX_VMIN_FREQ_LO_MASK 0xf8000000 + +#define VLV_TURBO_SOC_OVERRIDE 0x04 +#define VLV_OVERRIDE_EN 1 +#define VLV_SOC_TDP_EN (1 << 1) +#define VLV_BIAS_CPU_125_SOC_875 (6 << 2) +#define CHV_BIAS_CPU_50_SOC_50 (3 << 2) + +/* vlv2 north clock has */ +#define CCK_FUSE_REG 0x8 +#define CCK_FUSE_HPLL_FREQ_MASK 0x3 +#define CCK_REG_DSI_PLL_FUSE 0x44 +#define CCK_REG_DSI_PLL_CONTROL 0x48 +#define DSI_PLL_VCO_EN (1 << 31) +#define DSI_PLL_LDO_GATE (1 << 30) +#define DSI_PLL_P1_POST_DIV_SHIFT 17 +#define DSI_PLL_P1_POST_DIV_MASK (0x1ff << 17) +#define DSI_PLL_P2_MUX_DSI0_DIV2 (1 << 13) +#define DSI_PLL_P3_MUX_DSI1_DIV2 (1 << 12) +#define DSI_PLL_MUX_MASK (3 << 9) +#define DSI_PLL_MUX_DSI0_DSIPLL (0 << 10) +#define DSI_PLL_MUX_DSI0_CCK (1 << 10) +#define DSI_PLL_MUX_DSI1_DSIPLL (0 << 9) +#define DSI_PLL_MUX_DSI1_CCK (1 << 9) +#define DSI_PLL_CLK_GATE_MASK (0xf << 5) +#define DSI_PLL_CLK_GATE_DSI0_DSIPLL (1 << 8) +#define DSI_PLL_CLK_GATE_DSI1_DSIPLL (1 << 7) +#define DSI_PLL_CLK_GATE_DSI0_CCK (1 << 6) +#define DSI_PLL_CLK_GATE_DSI1_CCK (1 << 5) +#define DSI_PLL_LOCK (1 << 0) +#define CCK_REG_DSI_PLL_DIVIDER 0x4c +#define DSI_PLL_LFSR (1 << 31) +#define DSI_PLL_FRACTION_EN (1 << 30) +#define DSI_PLL_FRAC_COUNTER_SHIFT 27 +#define DSI_PLL_FRAC_COUNTER_MASK (7 << 27) +#define DSI_PLL_USYNC_CNT_SHIFT 18 +#define DSI_PLL_USYNC_CNT_MASK (0x1ff << 18) +#define DSI_PLL_N1_DIV_SHIFT 16 +#define DSI_PLL_N1_DIV_MASK (3 << 16) +#define DSI_PLL_M1_DIV_SHIFT 0 +#define DSI_PLL_M1_DIV_MASK (0x1ff << 0) +#define CCK_CZ_CLOCK_CONTROL 0x62 +#define CCK_GPLL_CLOCK_CONTROL 0x67 +#define CCK_DISPLAY_CLOCK_CONTROL 0x6b +#define CCK_DISPLAY_REF_CLOCK_CONTROL 0x6c +#define CCK_TRUNK_FORCE_ON (1 << 17) +#define CCK_TRUNK_FORCE_OFF (1 << 16) +#define CCK_FREQUENCY_STATUS (0x1f << 8) +#define CCK_FREQUENCY_STATUS_SHIFT 8 +#define CCK_FREQUENCY_VALUES (0x1f << 0) + +#endif /* _VLV_SIDEBAND_REG_H_ */ diff --git a/drivers/gpu/drm/i915/vlv_suspend.c b/drivers/gpu/drm/i915/vlv_suspend.c index 23adb64d640a..1d9da32195c2 100644 --- a/drivers/gpu/drm/i915/vlv_suspend.c +++ b/drivers/gpu/drm/i915/vlv_suspend.c @@ -14,6 +14,8 @@ #include "intel_pm.h" #include "vlv_suspend.h" +#include "gt/intel_gt_regs.h" + struct vlv_s0ix_state { /* GAM */ u32 wr_watermark; |