diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem_gtt.h')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_gem_gtt.h | 206 |
1 files changed, 56 insertions, 150 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index 812717ccc69b..b97a47fc7a68 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -35,15 +35,19 @@ #define __I915_GEM_GTT_H__ #include <linux/io-mapping.h> +#include <linux/kref.h> #include <linux/mm.h> #include <linux/pagevec.h> +#include <linux/workqueue.h> + +#include <drm/drm_mm.h> #include "gt/intel_reset.h" #include "i915_gem_fence_reg.h" #include "i915_request.h" #include "i915_scatterlist.h" #include "i915_selftest.h" -#include "i915_timeline.h" +#include "gt/intel_timeline.h" #define I915_GTT_PAGE_SIZE_4K BIT_ULL(12) #define I915_GTT_PAGE_SIZE_64K BIT_ULL(16) @@ -64,12 +68,10 @@ struct drm_i915_file_private; struct drm_i915_gem_object; struct i915_vma; +struct intel_gt; typedef u32 gen6_pte_t; typedef u64 gen8_pte_t; -typedef u64 gen8_pde_t; -typedef u64 gen8_ppgtt_pdpe_t; -typedef u64 gen8_ppgtt_pml4e_t; #define ggtt_total_entries(ggtt) ((ggtt)->vm.total >> PAGE_SHIFT) @@ -113,30 +115,18 @@ typedef u64 gen8_ppgtt_pml4e_t; #define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0)) #define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr) -/* GEN8 32b style address is defined as a 3 level page table: +/* + * GEN8 32b style address is defined as a 3 level page table: * 31:30 | 29:21 | 20:12 | 11:0 * PDPE | PDE | PTE | offset * The difference as compared to normal x86 3 level page table is the PDPEs are * programmed via register. - */ -#define GEN8_3LVL_PDPES 4 -#define GEN8_PDE_SHIFT 21 -#define GEN8_PDE_MASK 0x1ff -#define GEN8_PTE_SHIFT 12 -#define GEN8_PTE_MASK 0x1ff -#define GEN8_PTES I915_PTES(sizeof(gen8_pte_t)) - -/* GEN8 48b style address is defined as a 4 level page table: + * + * GEN8 48b style address is defined as a 4 level page table: * 47:39 | 38:30 | 29:21 | 20:12 | 11:0 * PML4E | PDPE | PDE | PTE | offset */ -#define GEN8_PML4ES_PER_PML4 512 -#define GEN8_PML4E_SHIFT 39 -#define GEN8_PML4E_MASK (GEN8_PML4ES_PER_PML4 - 1) -#define GEN8_PDPE_SHIFT 30 -/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page - * tables */ -#define GEN8_PDPE_MASK 0x1ff +#define GEN8_3LVL_PDPES 4 #define PPAT_UNCACHED (_PAGE_PWT | _PAGE_PCD) #define PPAT_CACHED_PDE 0 /* WB LLC */ @@ -155,11 +145,6 @@ typedef u64 gen8_ppgtt_pml4e_t; #define GEN8_PPAT_ELLC_OVERRIDE (0<<2) #define GEN8_PPAT(i, x) ((u64)(x) << ((i) * 8)) -#define GEN8_PPAT_GET_CA(x) ((x) & 3) -#define GEN8_PPAT_GET_TC(x) ((x) & (3 << 2)) -#define GEN8_PPAT_GET_AGE(x) ((x) & (3 << 4)) -#define CHV_PPAT_GET_SNOOP(x) ((x) & (1 << 6)) - #define GEN8_PDE_IPS_64K BIT(11) #define GEN8_PDE_PS_2M BIT(7) @@ -243,8 +228,10 @@ struct i915_page_dma { }; }; -#define px_base(px) (&(px)->base) -#define px_dma(px) (px_base(px)->daddr) +struct i915_page_scratch { + struct i915_page_dma base; + u64 encode; +}; struct i915_page_table { struct i915_page_dma base; @@ -252,12 +239,32 @@ struct i915_page_table { }; struct i915_page_directory { - struct i915_page_dma base; - atomic_t used; + struct i915_page_table pt; spinlock_t lock; void *entry[512]; }; +#define __px_choose_expr(x, type, expr, other) \ + __builtin_choose_expr( \ + __builtin_types_compatible_p(typeof(x), type) || \ + __builtin_types_compatible_p(typeof(x), const type), \ + ({ type __x = (type)(x); expr; }), \ + other) + +#define px_base(px) \ + __px_choose_expr(px, struct i915_page_dma *, __x, \ + __px_choose_expr(px, struct i915_page_scratch *, &__x->base, \ + __px_choose_expr(px, struct i915_page_table *, &__x->base, \ + __px_choose_expr(px, struct i915_page_directory *, &__x->pt.base, \ + (void)0)))) +#define px_dma(px) (px_base(px)->daddr) + +#define px_pt(px) \ + __px_choose_expr(px, struct i915_page_table *, __x, \ + __px_choose_expr(px, struct i915_page_directory *, &__x->pt, \ + (void)0)) +#define px_used(px) (&px_pt(px)->used) + struct i915_vma_ops { /* Map an object into an address space with the given cache flags. */ int (*bind_vma)(struct i915_vma *vma, @@ -280,8 +287,10 @@ struct pagestash { struct i915_address_space { struct kref ref; + struct rcu_work rcu; struct drm_mm mm; + struct intel_gt *gt; struct drm_i915_private *i915; struct device *dma; /* Every address space belongs to a struct file - except for the global @@ -302,12 +311,9 @@ struct i915_address_space { #define VM_CLASS_GGTT 0 #define VM_CLASS_PPGTT 1 - u64 scratch_pte; - int scratch_order; - struct i915_page_dma scratch_page; - struct i915_page_table *scratch_pt; - struct i915_page_directory *scratch_pd; - struct i915_page_directory *scratch_pdp; /* GEN8+ & 48b PPGTT */ + struct i915_page_scratch scratch[4]; + unsigned int scratch_order; + unsigned int top; /** * List of vma currently bound. @@ -386,7 +392,10 @@ struct i915_ggtt { /** "Graphics Stolen Memory" holds the global PTEs */ void __iomem *gsm; - void (*invalidate)(struct drm_i915_private *dev_priv); + void (*invalidate)(struct i915_ggtt *ggtt); + + /** PPGTT used for aliasing the PPGTT with the GTT */ + struct i915_ppgtt *alias; bool do_idle_maps; @@ -425,8 +434,6 @@ struct gen6_ppgtt { unsigned int pin_count; bool scan_for_unused_pt; - - struct gen6_ppgtt_cleanup_work *work; }; #define __to_gen6_ppgtt(base) container_of(base, struct gen6_ppgtt, base) @@ -506,15 +513,6 @@ static inline u32 gen6_pde_index(u32 addr) return i915_pde_index(addr, GEN6_PDE_SHIFT); } -static inline unsigned int -i915_pdpes_per_pdp(const struct i915_address_space *vm) -{ - if (i915_vm_is_4lvl(vm)) - return GEN8_PML4ES_PER_PML4; - - return GEN8_3LVL_PDPES; -} - static inline struct i915_page_table * i915_pt_entry(const struct i915_page_directory * const pd, const unsigned short n) @@ -529,73 +527,12 @@ i915_pd_entry(const struct i915_page_directory * const pdp, return pdp->entry[n]; } -static inline struct i915_page_directory * -i915_pdp_entry(const struct i915_page_directory * const pml4, - const unsigned short n) -{ - return pml4->entry[n]; -} - -/* Equivalent to the gen6 version, For each pde iterates over every pde - * between from start until start + length. On gen8+ it simply iterates - * over every page directory entry in a page directory. - */ -#define gen8_for_each_pde(pt, pd, start, length, iter) \ - for (iter = gen8_pde_index(start); \ - length > 0 && iter < I915_PDES && \ - (pt = i915_pt_entry(pd, iter), true); \ - ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT); \ - temp = min(temp - start, length); \ - start += temp, length -= temp; }), ++iter) - -#define gen8_for_each_pdpe(pd, pdp, start, length, iter) \ - for (iter = gen8_pdpe_index(start); \ - length > 0 && iter < i915_pdpes_per_pdp(vm) && \ - (pd = i915_pd_entry(pdp, iter), true); \ - ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT); \ - temp = min(temp - start, length); \ - start += temp, length -= temp; }), ++iter) - -#define gen8_for_each_pml4e(pdp, pml4, start, length, iter) \ - for (iter = gen8_pml4e_index(start); \ - length > 0 && iter < GEN8_PML4ES_PER_PML4 && \ - (pdp = i915_pdp_entry(pml4, iter), true); \ - ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT); \ - temp = min(temp - start, length); \ - start += temp, length -= temp; }), ++iter) - -static inline u32 gen8_pte_index(u64 address) -{ - return i915_pte_index(address, GEN8_PDE_SHIFT); -} - -static inline u32 gen8_pde_index(u64 address) -{ - return i915_pde_index(address, GEN8_PDE_SHIFT); -} - -static inline u32 gen8_pdpe_index(u64 address) -{ - return (address >> GEN8_PDPE_SHIFT) & GEN8_PDPE_MASK; -} - -static inline u32 gen8_pml4e_index(u64 address) -{ - return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK; -} - -static inline u64 gen8_pte_count(u64 address, u64 length) -{ - return i915_pte_count(address, length, GEN8_PDE_SHIFT); -} - static inline dma_addr_t i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n) { - struct i915_page_directory *pd; + struct i915_page_dma *pt = ppgtt->pd->entry[n]; - pd = i915_pdp_entry(ppgtt->pd, n); - return px_dma(pd); + return px_dma(pt ?: px_base(&ppgtt->vm.scratch[ppgtt->vm.top])); } static inline struct i915_ggtt * @@ -614,46 +551,15 @@ i915_vm_to_ppgtt(struct i915_address_space *vm) return container_of(vm, struct i915_ppgtt, vm); } -#define INTEL_MAX_PPAT_ENTRIES 8 -#define INTEL_PPAT_PERFECT_MATCH (~0U) - -struct intel_ppat; - -struct intel_ppat_entry { - struct intel_ppat *ppat; - struct kref ref; - u8 value; -}; - -struct intel_ppat { - struct intel_ppat_entry entries[INTEL_MAX_PPAT_ENTRIES]; - DECLARE_BITMAP(used, INTEL_MAX_PPAT_ENTRIES); - DECLARE_BITMAP(dirty, INTEL_MAX_PPAT_ENTRIES); - unsigned int max_entries; - u8 clear_value; - /* - * Return a score to show how two PPAT values match, - * a INTEL_PPAT_PERFECT_MATCH indicates a perfect match - */ - unsigned int (*match)(u8 src, u8 dst); - void (*update_hw)(struct drm_i915_private *i915); - - struct drm_i915_private *i915; -}; - -const struct intel_ppat_entry * -intel_ppat_get(struct drm_i915_private *i915, u8 value); -void intel_ppat_put(const struct intel_ppat_entry *entry); - int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv); int i915_ggtt_init_hw(struct drm_i915_private *dev_priv); int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv); -void i915_ggtt_enable_guc(struct drm_i915_private *i915); -void i915_ggtt_disable_guc(struct drm_i915_private *i915); -int i915_gem_init_ggtt(struct drm_i915_private *dev_priv); -void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv); +void i915_ggtt_enable_guc(struct i915_ggtt *ggtt); +void i915_ggtt_disable_guc(struct i915_ggtt *ggtt); +int i915_init_ggtt(struct drm_i915_private *dev_priv); +void i915_ggtt_driver_release(struct drm_i915_private *dev_priv); -int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv); +int i915_ppgtt_init_hw(struct intel_gt *gt); struct i915_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv); @@ -694,9 +600,9 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, u64 start, u64 end, unsigned int flags); /* Flags used by pin/bind&friends. */ -#define PIN_NONBLOCK BIT_ULL(0) -#define PIN_NONFAULT BIT_ULL(1) -#define PIN_NOEVICT BIT_ULL(2) +#define PIN_NOEVICT BIT_ULL(0) +#define PIN_NOSEARCH BIT_ULL(1) +#define PIN_NONBLOCK BIT_ULL(2) #define PIN_MAPPABLE BIT_ULL(3) #define PIN_ZONE_4G BIT_ULL(4) #define PIN_HIGH BIT_ULL(5) |