diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_gem.c | 1916 |
1 files changed, 837 insertions, 1079 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 0d1e4b7b4b99..c1e5c66553df 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -35,31 +35,41 @@ #include <linux/slab.h> #include <linux/swap.h> #include <linux/pci.h> +#include <linux/dma-buf.h> static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj); static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); -static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, - bool write); -static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, - uint64_t offset, - uint64_t size); -static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj); static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, unsigned alignment, bool map_and_fenceable); -static void i915_gem_clear_fence_reg(struct drm_device *dev, - struct drm_i915_fence_reg *reg); static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file); -static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj); + +static void i915_gem_write_fence(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj); +static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, + struct drm_i915_fence_reg *fence, + bool enable); static int i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc); static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); +static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) +{ + if (obj->tiling_mode) + i915_gem_release_mmap(obj); + + /* As we do not have an associated fence register, we will force + * a tiling change if we ever need to acquire one. + */ + obj->fence_dirty = false; + obj->fence_reg = I915_FENCE_REG_NONE; +} + /* some bookkeeping */ static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) @@ -122,26 +132,7 @@ int i915_mutex_lock_interruptible(struct drm_device *dev) static inline bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) { - return obj->gtt_space && !obj->active && obj->pin_count == 0; -} - -void i915_gem_do_init(struct drm_device *dev, - unsigned long start, - unsigned long mappable_end, - unsigned long end) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - - drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); - - dev_priv->mm.gtt_start = start; - dev_priv->mm.gtt_mappable_end = mappable_end; - dev_priv->mm.gtt_end = end; - dev_priv->mm.gtt_total = end - start; - dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; - - /* Take over this portion of the GTT */ - intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); + return !obj->active; } int @@ -150,12 +141,20 @@ i915_gem_init_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_init *args = data; + if (drm_core_check_feature(dev, DRIVER_MODESET)) + return -ENODEV; + if (args->gtt_start >= args->gtt_end || (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) return -EINVAL; + /* GEM with user mode setting was never supported on ilk and later. */ + if (INTEL_INFO(dev)->gen >= 5) + return -ENODEV; + mutex_lock(&dev->struct_mutex); - i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end); + i915_gem_init_global_gtt(dev, args->gtt_start, + args->gtt_end, args->gtt_end); mutex_unlock(&dev->struct_mutex); return 0; @@ -170,13 +169,11 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; size_t pinned; - if (!(dev->driver->driver_features & DRIVER_GEM)) - return -ENODEV; - pinned = 0; mutex_lock(&dev->struct_mutex); - list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) - pinned += obj->gtt_space->size; + list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) + if (obj->pin_count) + pinned += obj->gtt_space->size; mutex_unlock(&dev->struct_mutex); args->aper_size = dev_priv->mm.gtt_total; @@ -247,6 +244,7 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_create *args = data; + return i915_gem_create(file, dev, args->size, &args->handle); } @@ -259,66 +257,6 @@ static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) obj->tiling_mode != I915_TILING_NONE; } -/** - * This is the fast shmem pread path, which attempts to copy_from_user directly - * from the backing pages of the object to the user's address space. On a - * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). - */ -static int -i915_gem_shmem_pread_fast(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pread *args, - struct drm_file *file) -{ - struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; - ssize_t remain; - loff_t offset; - char __user *user_data; - int page_offset, page_length; - - user_data = (char __user *) (uintptr_t) args->data_ptr; - remain = args->size; - - offset = args->offset; - - while (remain > 0) { - struct page *page; - char *vaddr; - int ret; - - /* Operation in this page - * - * page_offset = offset within page - * page_length = bytes to copy for this page - */ - page_offset = offset_in_page(offset); - page_length = remain; - if ((page_offset + remain) > PAGE_SIZE) - page_length = PAGE_SIZE - page_offset; - - page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); - if (IS_ERR(page)) - return PTR_ERR(page); - - vaddr = kmap_atomic(page); - ret = __copy_to_user_inatomic(user_data, - vaddr + page_offset, - page_length); - kunmap_atomic(vaddr); - - mark_page_accessed(page); - page_cache_release(page); - if (ret) - return -EFAULT; - - remain -= page_length; - user_data += page_length; - offset += page_length; - } - - return 0; -} - static inline int __copy_to_user_swizzled(char __user *cpu_vaddr, const char *gpu_vaddr, int gpu_offset, @@ -346,8 +284,8 @@ __copy_to_user_swizzled(char __user *cpu_vaddr, } static inline int -__copy_from_user_swizzled(char __user *gpu_vaddr, int gpu_offset, - const char *cpu_vaddr, +__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, + const char __user *cpu_vaddr, int length) { int ret, cpu_offset = 0; @@ -371,37 +309,121 @@ __copy_from_user_swizzled(char __user *gpu_vaddr, int gpu_offset, return 0; } -/** - * This is the fallback shmem pread path, which allocates temporary storage - * in kernel space to copy_to_user into outside of the struct_mutex, so we - * can copy out of the object's backing pages while holding the struct mutex - * and not take page faults. - */ +/* Per-page copy function for the shmem pread fastpath. + * Flushes invalid cachelines before reading the target if + * needs_clflush is set. */ static int -i915_gem_shmem_pread_slow(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pread *args, - struct drm_file *file) +shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length, + char __user *user_data, + bool page_do_bit17_swizzling, bool needs_clflush) +{ + char *vaddr; + int ret; + + if (unlikely(page_do_bit17_swizzling)) + return -EINVAL; + + vaddr = kmap_atomic(page); + if (needs_clflush) + drm_clflush_virt_range(vaddr + shmem_page_offset, + page_length); + ret = __copy_to_user_inatomic(user_data, + vaddr + shmem_page_offset, + page_length); + kunmap_atomic(vaddr); + + return ret; +} + +static void +shmem_clflush_swizzled_range(char *addr, unsigned long length, + bool swizzled) +{ + if (unlikely(swizzled)) { + unsigned long start = (unsigned long) addr; + unsigned long end = (unsigned long) addr + length; + + /* For swizzling simply ensure that we always flush both + * channels. Lame, but simple and it works. Swizzled + * pwrite/pread is far from a hotpath - current userspace + * doesn't use it at all. */ + start = round_down(start, 128); + end = round_up(end, 128); + + drm_clflush_virt_range((void *)start, end - start); + } else { + drm_clflush_virt_range(addr, length); + } + +} + +/* Only difference to the fast-path function is that this can handle bit17 + * and uses non-atomic copy and kmap functions. */ +static int +shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, + char __user *user_data, + bool page_do_bit17_swizzling, bool needs_clflush) +{ + char *vaddr; + int ret; + + vaddr = kmap(page); + if (needs_clflush) + shmem_clflush_swizzled_range(vaddr + shmem_page_offset, + page_length, + page_do_bit17_swizzling); + + if (page_do_bit17_swizzling) + ret = __copy_to_user_swizzled(user_data, + vaddr, shmem_page_offset, + page_length); + else + ret = __copy_to_user(user_data, + vaddr + shmem_page_offset, + page_length); + kunmap(page); + + return ret; +} + +static int +i915_gem_shmem_pread(struct drm_device *dev, + struct drm_i915_gem_object *obj, + struct drm_i915_gem_pread *args, + struct drm_file *file) { struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; char __user *user_data; ssize_t remain; loff_t offset; - int shmem_page_offset, page_length, ret; + int shmem_page_offset, page_length, ret = 0; int obj_do_bit17_swizzling, page_do_bit17_swizzling; + int hit_slowpath = 0; + int prefaulted = 0; + int needs_clflush = 0; + int release_page; user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - offset = args->offset; + if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { + /* If we're not in the cpu read domain, set ourself into the gtt + * read domain and manually flush cachelines (if required). This + * optimizes for the case when the gpu will dirty the data + * anyway again before the next pread happens. */ + if (obj->cache_level == I915_CACHE_NONE) + needs_clflush = 1; + ret = i915_gem_object_set_to_gtt_domain(obj, false); + if (ret) + return ret; + } - mutex_unlock(&dev->struct_mutex); + offset = args->offset; while (remain > 0) { struct page *page; - char *vaddr; /* Operation in this page * @@ -413,28 +435,51 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, if ((shmem_page_offset + page_length) > PAGE_SIZE) page_length = PAGE_SIZE - shmem_page_offset; - page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); - if (IS_ERR(page)) { - ret = PTR_ERR(page); - goto out; + if (obj->pages) { + page = obj->pages[offset >> PAGE_SHIFT]; + release_page = 0; + } else { + page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); + if (IS_ERR(page)) { + ret = PTR_ERR(page); + goto out; + } + release_page = 1; } page_do_bit17_swizzling = obj_do_bit17_swizzling && (page_to_phys(page) & (1 << 17)) != 0; - vaddr = kmap(page); - if (page_do_bit17_swizzling) - ret = __copy_to_user_swizzled(user_data, - vaddr, shmem_page_offset, - page_length); - else - ret = __copy_to_user(user_data, - vaddr + shmem_page_offset, - page_length); - kunmap(page); + ret = shmem_pread_fast(page, shmem_page_offset, page_length, + user_data, page_do_bit17_swizzling, + needs_clflush); + if (ret == 0) + goto next_page; - mark_page_accessed(page); + hit_slowpath = 1; + page_cache_get(page); + mutex_unlock(&dev->struct_mutex); + + if (!prefaulted) { + ret = fault_in_multipages_writeable(user_data, remain); + /* Userspace is tricking us, but we've already clobbered + * its pages with the prefault and promised to write the + * data up to the first fault. Hence ignore any errors + * and just continue. */ + (void)ret; + prefaulted = 1; + } + + ret = shmem_pread_slow(page, shmem_page_offset, page_length, + user_data, page_do_bit17_swizzling, + needs_clflush); + + mutex_lock(&dev->struct_mutex); page_cache_release(page); +next_page: + mark_page_accessed(page); + if (release_page) + page_cache_release(page); if (ret) { ret = -EFAULT; @@ -447,10 +492,11 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, } out: - mutex_lock(&dev->struct_mutex); - /* Fixup: Kill any reinstated backing storage pages */ - if (obj->madv == __I915_MADV_PURGED) - i915_gem_object_truncate(obj); + if (hit_slowpath) { + /* Fixup: Kill any reinstated backing storage pages */ + if (obj->madv == __I915_MADV_PURGED) + i915_gem_object_truncate(obj); + } return ret; } @@ -476,11 +522,6 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, args->size)) return -EFAULT; - ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr, - args->size); - if (ret) - return -EFAULT; - ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; @@ -498,19 +539,17 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, goto out; } - trace_i915_gem_object_pread(obj, args->offset, args->size); - - ret = i915_gem_object_set_cpu_read_domain_range(obj, - args->offset, - args->size); - if (ret) + /* prime objects have no backing filp to GEM pread/pwrite + * pages from. + */ + if (!obj->base.filp) { + ret = -EINVAL; goto out; + } - ret = -EFAULT; - if (!i915_gem_object_needs_bit17_swizzle(obj)) - ret = i915_gem_shmem_pread_fast(dev, obj, args, file); - if (ret == -EFAULT) - ret = i915_gem_shmem_pread_slow(dev, obj, args, file); + trace_i915_gem_object_pread(obj, args->offset, args->size); + + ret = i915_gem_shmem_pread(dev, obj, args, file); out: drm_gem_object_unreference(&obj->base); @@ -529,40 +568,19 @@ fast_user_write(struct io_mapping *mapping, char __user *user_data, int length) { - char *vaddr_atomic; + void __iomem *vaddr_atomic; + void *vaddr; unsigned long unwritten; vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); - unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset, + /* We can use the cpu mem copy function because this is X86. */ + vaddr = (void __force*)vaddr_atomic + page_offset; + unwritten = __copy_from_user_inatomic_nocache(vaddr, user_data, length); io_mapping_unmap_atomic(vaddr_atomic); return unwritten; } -/* Here's the write path which can sleep for - * page faults - */ - -static inline void -slow_kernel_write(struct io_mapping *mapping, - loff_t gtt_base, int gtt_offset, - struct page *user_page, int user_offset, - int length) -{ - char __iomem *dst_vaddr; - char *src_vaddr; - - dst_vaddr = io_mapping_map_wc(mapping, gtt_base); - src_vaddr = kmap(user_page); - - memcpy_toio(dst_vaddr + gtt_offset, - src_vaddr + user_offset, - length); - - kunmap(user_page); - io_mapping_unmap(dst_vaddr); -} - /** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached. @@ -577,7 +595,19 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, ssize_t remain; loff_t offset, page_base; char __user *user_data; - int page_offset, page_length; + int page_offset, page_length, ret; + + ret = i915_gem_object_pin(obj, 0, true); + if (ret) + goto out; + + ret = i915_gem_object_set_to_gtt_domain(obj, true); + if (ret) + goto out_unpin; + + ret = i915_gem_object_put_fence(obj); + if (ret) + goto out_unpin; user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; @@ -602,214 +632,133 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, * retry in the slow path. */ if (fast_user_write(dev_priv->mm.gtt_mapping, page_base, - page_offset, user_data, page_length)) - return -EFAULT; + page_offset, user_data, page_length)) { + ret = -EFAULT; + goto out_unpin; + } remain -= page_length; user_data += page_length; offset += page_length; } - return 0; +out_unpin: + i915_gem_object_unpin(obj); +out: + return ret; } -/** - * This is the fallback GTT pwrite path, which uses get_user_pages to pin - * the memory and maps it using kmap_atomic for copying. - * - * This code resulted in x11perf -rgb10text consuming about 10% more CPU - * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit). - */ +/* Per-page copy function for the shmem pwrite fastpath. + * Flushes invalid cachelines before writing to the target if + * needs_clflush_before is set and flushes out any written cachelines after + * writing if needs_clflush is set. */ static int -i915_gem_gtt_pwrite_slow(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) +shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, + char __user *user_data, + bool page_do_bit17_swizzling, + bool needs_clflush_before, + bool needs_clflush_after) { - drm_i915_private_t *dev_priv = dev->dev_private; - ssize_t remain; - loff_t gtt_page_base, offset; - loff_t first_data_page, last_data_page, num_pages; - loff_t pinned_pages, i; - struct page **user_pages; - struct mm_struct *mm = current->mm; - int gtt_page_offset, data_page_offset, data_page_index, page_length; + char *vaddr; int ret; - uint64_t data_ptr = args->data_ptr; - - remain = args->size; - - /* Pin the user pages containing the data. We can't fault while - * holding the struct mutex, and all of the pwrite implementations - * want to hold it while dereferencing the user data. - */ - first_data_page = data_ptr / PAGE_SIZE; - last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; - num_pages = last_data_page - first_data_page + 1; - - user_pages = drm_malloc_ab(num_pages, sizeof(struct page *)); - if (user_pages == NULL) - return -ENOMEM; - - mutex_unlock(&dev->struct_mutex); - down_read(&mm->mmap_sem); - pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, - num_pages, 0, 0, user_pages, NULL); - up_read(&mm->mmap_sem); - mutex_lock(&dev->struct_mutex); - if (pinned_pages < num_pages) { - ret = -EFAULT; - goto out_unpin_pages; - } - - ret = i915_gem_object_set_to_gtt_domain(obj, true); - if (ret) - goto out_unpin_pages; - - ret = i915_gem_object_put_fence(obj); - if (ret) - goto out_unpin_pages; - - offset = obj->gtt_offset + args->offset; - - while (remain > 0) { - /* Operation in this page - * - * gtt_page_base = page offset within aperture - * gtt_page_offset = offset within page in aperture - * data_page_index = page number in get_user_pages return - * data_page_offset = offset with data_page_index page. - * page_length = bytes to copy for this page - */ - gtt_page_base = offset & PAGE_MASK; - gtt_page_offset = offset_in_page(offset); - data_page_index = data_ptr / PAGE_SIZE - first_data_page; - data_page_offset = offset_in_page(data_ptr); - - page_length = remain; - if ((gtt_page_offset + page_length) > PAGE_SIZE) - page_length = PAGE_SIZE - gtt_page_offset; - if ((data_page_offset + page_length) > PAGE_SIZE) - page_length = PAGE_SIZE - data_page_offset; - slow_kernel_write(dev_priv->mm.gtt_mapping, - gtt_page_base, gtt_page_offset, - user_pages[data_page_index], - data_page_offset, - page_length); - - remain -= page_length; - offset += page_length; - data_ptr += page_length; - } + if (unlikely(page_do_bit17_swizzling)) + return -EINVAL; -out_unpin_pages: - for (i = 0; i < pinned_pages; i++) - page_cache_release(user_pages[i]); - drm_free_large(user_pages); + vaddr = kmap_atomic(page); + if (needs_clflush_before) + drm_clflush_virt_range(vaddr + shmem_page_offset, + page_length); + ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, + user_data, + page_length); + if (needs_clflush_after) + drm_clflush_virt_range(vaddr + shmem_page_offset, + page_length); + kunmap_atomic(vaddr); return ret; } -/** - * This is the fast shmem pwrite path, which attempts to directly - * copy_from_user into the kmapped pages backing the object. - */ +/* Only difference to the fast-path function is that this can handle bit17 + * and uses non-atomic copy and kmap functions. */ static int -i915_gem_shmem_pwrite_fast(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) +shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, + char __user *user_data, + bool page_do_bit17_swizzling, + bool needs_clflush_before, + bool needs_clflush_after) { - struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; - ssize_t remain; - loff_t offset; - char __user *user_data; - int page_offset, page_length; - - user_data = (char __user *) (uintptr_t) args->data_ptr; - remain = args->size; - - offset = args->offset; - obj->dirty = 1; - - while (remain > 0) { - struct page *page; - char *vaddr; - int ret; - - /* Operation in this page - * - * page_offset = offset within page - * page_length = bytes to copy for this page - */ - page_offset = offset_in_page(offset); - page_length = remain; - if ((page_offset + remain) > PAGE_SIZE) - page_length = PAGE_SIZE - page_offset; - - page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); - if (IS_ERR(page)) - return PTR_ERR(page); + char *vaddr; + int ret; - vaddr = kmap_atomic(page); - ret = __copy_from_user_inatomic(vaddr + page_offset, + vaddr = kmap(page); + if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) + shmem_clflush_swizzled_range(vaddr + shmem_page_offset, + page_length, + page_do_bit17_swizzling); + if (page_do_bit17_swizzling) + ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, user_data, page_length); - kunmap_atomic(vaddr); - - set_page_dirty(page); - mark_page_accessed(page); - page_cache_release(page); - - /* If we get a fault while copying data, then (presumably) our - * source page isn't available. Return the error and we'll - * retry in the slow path. - */ - if (ret) - return -EFAULT; - - remain -= page_length; - user_data += page_length; - offset += page_length; - } + else + ret = __copy_from_user(vaddr + shmem_page_offset, + user_data, + page_length); + if (needs_clflush_after) + shmem_clflush_swizzled_range(vaddr + shmem_page_offset, + page_length, + page_do_bit17_swizzling); + kunmap(page); - return 0; + return ret; } -/** - * This is the fallback shmem pwrite path, which uses get_user_pages to pin - * the memory and maps it using kmap_atomic for copying. - * - * This avoids taking mmap_sem for faulting on the user's address while the - * struct_mutex is held. - */ static int -i915_gem_shmem_pwrite_slow(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) +i915_gem_shmem_pwrite(struct drm_device *dev, + struct drm_i915_gem_object *obj, + struct drm_i915_gem_pwrite *args, + struct drm_file *file) { struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; ssize_t remain; loff_t offset; char __user *user_data; - int shmem_page_offset, page_length, ret; + int shmem_page_offset, page_length, ret = 0; int obj_do_bit17_swizzling, page_do_bit17_swizzling; + int hit_slowpath = 0; + int needs_clflush_after = 0; + int needs_clflush_before = 0; + int release_page; user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); + if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { + /* If we're not in the cpu write domain, set ourself into the gtt + * write domain and manually flush cachelines (if required). This + * optimizes for the case when the gpu will use the data + * right away and we therefore have to clflush anyway. */ + if (obj->cache_level == I915_CACHE_NONE) + needs_clflush_after = 1; + ret = i915_gem_object_set_to_gtt_domain(obj, true); + if (ret) + return ret; + } + /* Same trick applies for invalidate partially written cachelines before + * writing. */ + if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU) + && obj->cache_level == I915_CACHE_NONE) + needs_clflush_before = 1; + offset = args->offset; obj->dirty = 1; - mutex_unlock(&dev->struct_mutex); - while (remain > 0) { struct page *page; - char *vaddr; + int partial_cacheline_write; /* Operation in this page * @@ -822,29 +771,51 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, if ((shmem_page_offset + page_length) > PAGE_SIZE) page_length = PAGE_SIZE - shmem_page_offset; - page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); - if (IS_ERR(page)) { - ret = PTR_ERR(page); - goto out; + /* If we don't overwrite a cacheline completely we need to be + * careful to have up-to-date data by first clflushing. Don't + * overcomplicate things and flush the entire patch. */ + partial_cacheline_write = needs_clflush_before && + ((shmem_page_offset | page_length) + & (boot_cpu_data.x86_clflush_size - 1)); + + if (obj->pages) { + page = obj->pages[offset >> PAGE_SHIFT]; + release_page = 0; + } else { + page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); + if (IS_ERR(page)) { + ret = PTR_ERR(page); + goto out; + } + release_page = 1; } page_do_bit17_swizzling = obj_do_bit17_swizzling && (page_to_phys(page) & (1 << 17)) != 0; - vaddr = kmap(page); - if (page_do_bit17_swizzling) - ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, - user_data, - page_length); - else - ret = __copy_from_user(vaddr + shmem_page_offset, - user_data, - page_length); - kunmap(page); + ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, + user_data, page_do_bit17_swizzling, + partial_cacheline_write, + needs_clflush_after); + if (ret == 0) + goto next_page; + + hit_slowpath = 1; + page_cache_get(page); + mutex_unlock(&dev->struct_mutex); + + ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, + user_data, page_do_bit17_swizzling, + partial_cacheline_write, + needs_clflush_after); + mutex_lock(&dev->struct_mutex); + page_cache_release(page); +next_page: set_page_dirty(page); mark_page_accessed(page); - page_cache_release(page); + if (release_page) + page_cache_release(page); if (ret) { ret = -EFAULT; @@ -857,17 +828,21 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, } out: - mutex_lock(&dev->struct_mutex); - /* Fixup: Kill any reinstated backing storage pages */ - if (obj->madv == __I915_MADV_PURGED) - i915_gem_object_truncate(obj); - /* and flush dirty cachelines in case the object isn't in the cpu write - * domain anymore. */ - if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { - i915_gem_clflush_object(obj); - intel_gtt_chipset_flush(); + if (hit_slowpath) { + /* Fixup: Kill any reinstated backing storage pages */ + if (obj->madv == __I915_MADV_PURGED) + i915_gem_object_truncate(obj); + /* and flush dirty cachelines in case the object isn't in the cpu write + * domain anymore. */ + if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { + i915_gem_clflush_object(obj); + intel_gtt_chipset_flush(); + } } + if (needs_clflush_after) + intel_gtt_chipset_flush(); + return ret; } @@ -892,8 +867,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, args->size)) return -EFAULT; - ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr, - args->size); + ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr, + args->size); if (ret) return -EFAULT; @@ -914,8 +889,17 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, goto out; } + /* prime objects have no backing filp to GEM pread/pwrite + * pages from. + */ + if (!obj->base.filp) { + ret = -EINVAL; + goto out; + } + trace_i915_gem_object_pwrite(obj, args->offset, args->size); + ret = -EFAULT; /* We can only do the GTT pwrite on untiled buffers, as otherwise * it would end up going through the fenced access, and we'll get * different detiling behavior between reading and writing. @@ -928,42 +912,18 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, } if (obj->gtt_space && + obj->cache_level == I915_CACHE_NONE && + obj->tiling_mode == I915_TILING_NONE && + obj->map_and_fenceable && obj->base.write_domain != I915_GEM_DOMAIN_CPU) { - ret = i915_gem_object_pin(obj, 0, true); - if (ret) - goto out; - - ret = i915_gem_object_set_to_gtt_domain(obj, true); - if (ret) - goto out_unpin; - - ret = i915_gem_object_put_fence(obj); - if (ret) - goto out_unpin; - ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); - if (ret == -EFAULT) - ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file); - -out_unpin: - i915_gem_object_unpin(obj); - - if (ret != -EFAULT) - goto out; - /* Fall through to the shmfs paths because the gtt paths might - * fail with non-page-backed user pointers (e.g. gtt mappings - * when moving data between textures). */ + /* Note that the gtt paths might fail with non-page-backed user + * pointers (e.g. gtt mappings when moving data between + * textures). Fallback to the shmem path in that case. */ } - ret = i915_gem_object_set_to_cpu_domain(obj, 1); - if (ret) - goto out; - - ret = -EFAULT; - if (!i915_gem_object_needs_bit17_swizzle(obj)) - ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file); if (ret == -EFAULT) - ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file); + ret = i915_gem_shmem_pwrite(dev, obj, args, file); out: drm_gem_object_unreference(&obj->base); @@ -986,9 +946,6 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, uint32_t write_domain = args->write_domain; int ret; - if (!(dev->driver->driver_features & DRIVER_GEM)) - return -ENODEV; - /* Only handle setting domains to types used by the CPU. */ if (write_domain & I915_GEM_GPU_DOMAINS) return -EINVAL; @@ -1042,9 +999,6 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; int ret = 0; - if (!(dev->driver->driver_features & DRIVER_GEM)) - return -ENODEV; - ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; @@ -1080,13 +1034,18 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct drm_gem_object *obj; unsigned long addr; - if (!(dev->driver->driver_features & DRIVER_GEM)) - return -ENODEV; - obj = drm_gem_object_lookup(dev, file, args->handle); if (obj == NULL) return -ENOENT; + /* prime objects have no backing filp to GEM mmap + * pages from. + */ + if (!obj->filp) { + drm_gem_object_unreference_unlocked(obj); + return -EINVAL; + } + addr = vm_mmap(obj->filp, 0, args->size, PROT_READ | PROT_WRITE, MAP_SHARED, args->offset); @@ -1151,10 +1110,10 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) goto unlock; } - if (obj->tiling_mode == I915_TILING_NONE) - ret = i915_gem_object_put_fence(obj); - else - ret = i915_gem_object_get_fence(obj, NULL); + if (!obj->has_global_gtt_mapping) + i915_gem_gtt_bind_object(obj, obj->cache_level); + + ret = i915_gem_object_get_fence(obj); if (ret) goto unlock; @@ -1308,9 +1267,6 @@ i915_gem_mmap_gtt(struct drm_file *file, struct drm_i915_gem_object *obj; int ret; - if (!(dev->driver->driver_features & DRIVER_GEM)) - return -ENODEV; - ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; @@ -1368,14 +1324,10 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_mmap_gtt *args = data; - if (!(dev->driver->driver_features & DRIVER_GEM)) - return -ENODEV; - return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); } - -static int +int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, gfp_t gfpmask) { @@ -1384,6 +1336,9 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, struct inode *inode; struct page *page; + if (obj->pages || obj->sg_table) + return 0; + /* Get the list of pages out of our struct file. They'll be pinned * at this point until we release them. */ @@ -1425,6 +1380,9 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) int page_count = obj->base.size / PAGE_SIZE; int i; + if (!obj->pages) + return; + BUG_ON(obj->madv == __I915_MADV_PURGED); if (i915_gem_object_needs_bit17_swizzle(obj)) @@ -1473,7 +1431,6 @@ i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, if (obj->fenced_gpu_access) { obj->last_fenced_seqno = seqno; - obj->last_fenced_ring = ring; /* Bump MRU to take account of the delayed flush */ if (obj->fence_reg != I915_FENCE_REG_NONE) { @@ -1512,15 +1469,11 @@ i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - if (obj->pin_count != 0) - list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); - else - list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); + list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); BUG_ON(!list_empty(&obj->gpu_write_list)); BUG_ON(!obj->active); obj->ring = NULL; - obj->last_fenced_ring = NULL; i915_gem_object_move_off_active(obj); obj->fenced_gpu_access = false; @@ -1546,6 +1499,9 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj) inode = obj->base.filp->f_path.dentry->d_inode; shmem_truncate_range(inode, 0, (loff_t)-1); + if (obj->base.map_list.map) + drm_gem_free_mmap_offset(&obj->base); + obj->madv = __I915_MADV_PURGED; } @@ -1711,30 +1667,29 @@ static void i915_gem_reset_fences(struct drm_device *dev) for (i = 0; i < dev_priv->num_fence_regs; i++) { struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; - struct drm_i915_gem_object *obj = reg->obj; - if (!obj) - continue; + i915_gem_write_fence(dev, i, NULL); - if (obj->tiling_mode) - i915_gem_release_mmap(obj); + if (reg->obj) + i915_gem_object_fence_lost(reg->obj); - reg->obj->fence_reg = I915_FENCE_REG_NONE; - reg->obj->fenced_gpu_access = false; - reg->obj->last_fenced_seqno = 0; - reg->obj->last_fenced_ring = NULL; - i915_gem_clear_fence_reg(dev, reg); + reg->pin_count = 0; + reg->obj = NULL; + INIT_LIST_HEAD(®->lru_list); } + + INIT_LIST_HEAD(&dev_priv->mm.fence_list); } void i915_gem_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; + struct intel_ring_buffer *ring; int i; - for (i = 0; i < I915_NUM_RINGS; i++) - i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]); + for_each_ring(ring, dev_priv, i) + i915_gem_reset_ring_lists(dev_priv, ring); /* Remove anything from the flushing lists. The GPU cache is likely * to be lost on reset along with the data, so simply move the @@ -1839,24 +1794,11 @@ void i915_gem_retire_requests(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring; int i; - if (!list_empty(&dev_priv->mm.deferred_free_list)) { - struct drm_i915_gem_object *obj, *next; - - /* We must be careful that during unbind() we do not - * accidentally infinitely recurse into retire requests. - * Currently: - * retire -> free -> unbind -> wait -> retire_ring - */ - list_for_each_entry_safe(obj, next, - &dev_priv->mm.deferred_free_list, - mm_list) - i915_gem_free_object_tail(obj); - } - - for (i = 0; i < I915_NUM_RINGS; i++) - i915_gem_retire_requests_ring(&dev_priv->ring[i]); + for_each_ring(ring, dev_priv, i) + i915_gem_retire_requests_ring(ring); } static void @@ -1864,6 +1806,7 @@ i915_gem_retire_work_handler(struct work_struct *work) { drm_i915_private_t *dev_priv; struct drm_device *dev; + struct intel_ring_buffer *ring; bool idle; int i; @@ -1883,9 +1826,7 @@ i915_gem_retire_work_handler(struct work_struct *work) * objects indefinitely. */ idle = true; - for (i = 0; i < I915_NUM_RINGS; i++) { - struct intel_ring_buffer *ring = &dev_priv->ring[i]; - + for_each_ring(ring, dev_priv, i) { if (!list_empty(&ring->gpu_write_list)) { struct drm_i915_gem_request *request; int ret; @@ -1907,20 +1848,10 @@ i915_gem_retire_work_handler(struct work_struct *work) mutex_unlock(&dev->struct_mutex); } -/** - * Waits for a sequence number to be signaled, and cleans up the - * request and object lists appropriately for that event. - */ -int -i915_wait_request(struct intel_ring_buffer *ring, - uint32_t seqno, - bool do_retire) +static int +i915_gem_check_wedge(struct drm_i915_private *dev_priv) { - drm_i915_private_t *dev_priv = ring->dev->dev_private; - u32 ier; - int ret = 0; - - BUG_ON(seqno == 0); + BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); if (atomic_read(&dev_priv->mm.wedged)) { struct completion *x = &dev_priv->error_completion; @@ -1935,6 +1866,20 @@ i915_wait_request(struct intel_ring_buffer *ring, return recovery_complete ? -EIO : -EAGAIN; } + return 0; +} + +/* + * Compare seqno against outstanding lazy request. Emit a request if they are + * equal. + */ +static int +i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) +{ + int ret = 0; + + BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); + if (seqno == ring->outstanding_lazy_request) { struct drm_i915_gem_request *request; @@ -1948,54 +1893,67 @@ i915_wait_request(struct intel_ring_buffer *ring, return ret; } - seqno = request->seqno; + BUG_ON(seqno != request->seqno); } - if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { - if (HAS_PCH_SPLIT(ring->dev)) - ier = I915_READ(DEIER) | I915_READ(GTIER); - else - ier = I915_READ(IER); - if (!ier) { - DRM_ERROR("something (likely vbetool) disabled " - "interrupts, re-enabling\n"); - ring->dev->driver->irq_preinstall(ring->dev); - ring->dev->driver->irq_postinstall(ring->dev); - } + return ret; +} - trace_i915_gem_request_wait_begin(ring, seqno); - - ring->waiting_seqno = seqno; - if (ring->irq_get(ring)) { - if (dev_priv->mm.interruptible) - ret = wait_event_interruptible(ring->irq_queue, - i915_seqno_passed(ring->get_seqno(ring), seqno) - || atomic_read(&dev_priv->mm.wedged)); - else - wait_event(ring->irq_queue, - i915_seqno_passed(ring->get_seqno(ring), seqno) - || atomic_read(&dev_priv->mm.wedged)); - - ring->irq_put(ring); - } else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring), - seqno) || - atomic_read(&dev_priv->mm.wedged), 3000)) - ret = -EBUSY; - ring->waiting_seqno = 0; - - trace_i915_gem_request_wait_end(ring, seqno); - } +static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, + bool interruptible) +{ + drm_i915_private_t *dev_priv = ring->dev->dev_private; + int ret = 0; + + if (i915_seqno_passed(ring->get_seqno(ring), seqno)) + return 0; + + trace_i915_gem_request_wait_begin(ring, seqno); + if (WARN_ON(!ring->irq_get(ring))) + return -ENODEV; + +#define EXIT_COND \ + (i915_seqno_passed(ring->get_seqno(ring), seqno) || \ + atomic_read(&dev_priv->mm.wedged)) + + if (interruptible) + ret = wait_event_interruptible(ring->irq_queue, + EXIT_COND); + else + wait_event(ring->irq_queue, EXIT_COND); + + ring->irq_put(ring); + trace_i915_gem_request_wait_end(ring, seqno); +#undef EXIT_COND + + return ret; +} + +/** + * Waits for a sequence number to be signaled, and cleans up the + * request and object lists appropriately for that event. + */ +int +i915_wait_request(struct intel_ring_buffer *ring, + uint32_t seqno) +{ + drm_i915_private_t *dev_priv = ring->dev->dev_private; + int ret = 0; + + BUG_ON(seqno == 0); + + ret = i915_gem_check_wedge(dev_priv); + if (ret) + return ret; + + ret = i915_gem_check_olr(ring, seqno); + if (ret) + return ret; + + ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible); if (atomic_read(&dev_priv->mm.wedged)) ret = -EAGAIN; - /* Directly dispatch request retiring. While we have the work queue - * to handle this, the waiter on a request often wants an associated - * buffer to have made it to the inactive list, and we would need - * a separate wait queue to handle that. - */ - if (ret == 0 && do_retire) - i915_gem_retire_requests_ring(ring); - return ret; } @@ -2017,15 +1975,58 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) * it. */ if (obj->active) { - ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, - true); + ret = i915_wait_request(obj->ring, obj->last_rendering_seqno); if (ret) return ret; + i915_gem_retire_requests_ring(obj->ring); } return 0; } +/** + * i915_gem_object_sync - sync an object to a ring. + * + * @obj: object which may be in use on another ring. + * @to: ring we wish to use the object on. May be NULL. + * + * This code is meant to abstract object synchronization with the GPU. + * Calling with NULL implies synchronizing the object with the CPU + * rather than a particular GPU ring. + * + * Returns 0 if successful, else propagates up the lower layer error. + */ +int +i915_gem_object_sync(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *to) +{ + struct intel_ring_buffer *from = obj->ring; + u32 seqno; + int ret, idx; + + if (from == NULL || to == from) + return 0; + + if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) + return i915_gem_object_wait_rendering(obj); + + idx = intel_ring_sync_index(from, to); + + seqno = obj->last_rendering_seqno; + if (seqno <= from->sync_seqno[idx]) + return 0; + + ret = i915_gem_check_olr(obj->ring, seqno); + if (ret) + return ret; + + ret = to->sync_to(to, from, seqno); + if (!ret) + from->sync_seqno[idx] = seqno; + + return ret; +} + static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) { u32 old_write_domain, old_read_domains; @@ -2068,7 +2069,7 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj) } ret = i915_gem_object_finish_gpu(obj); - if (ret == -ERESTARTSYS) + if (ret) return ret; /* Continue on if we fail due to EIO, the GPU is hung so we * should be safe and we need to cleanup or else we might @@ -2095,16 +2096,18 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj) /* release the fence reg _after_ flushing */ ret = i915_gem_object_put_fence(obj); - if (ret == -ERESTARTSYS) + if (ret) return ret; trace_i915_gem_object_unbind(obj); - i915_gem_gtt_unbind_object(obj); + if (obj->has_global_gtt_mapping) + i915_gem_gtt_unbind_object(obj); if (obj->has_aliasing_ppgtt_mapping) { i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); obj->has_aliasing_ppgtt_mapping = 0; } + i915_gem_gtt_finish_object(obj); i915_gem_object_put_pages_gtt(obj); @@ -2145,7 +2148,7 @@ i915_gem_flush_ring(struct intel_ring_buffer *ring, return 0; } -static int i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) +static int i915_ring_idle(struct intel_ring_buffer *ring) { int ret; @@ -2159,208 +2162,201 @@ static int i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) return ret; } - return i915_wait_request(ring, i915_gem_next_request_seqno(ring), - do_retire); + return i915_wait_request(ring, i915_gem_next_request_seqno(ring)); } -int i915_gpu_idle(struct drm_device *dev, bool do_retire) +int i915_gpu_idle(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring; int ret, i; /* Flush everything onto the inactive list. */ - for (i = 0; i < I915_NUM_RINGS; i++) { - ret = i915_ring_idle(&dev_priv->ring[i], do_retire); + for_each_ring(ring, dev_priv, i) { + ret = i915_ring_idle(ring); if (ret) return ret; + + /* Is the device fubar? */ + if (WARN_ON(!list_empty(&ring->gpu_write_list))) + return -EBUSY; } return 0; } -static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *pipelined) +static void sandybridge_write_fence_reg(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - u32 size = obj->gtt_space->size; - int regnum = obj->fence_reg; uint64_t val; - val = (uint64_t)((obj->gtt_offset + size - 4096) & - 0xfffff000) << 32; - val |= obj->gtt_offset & 0xfffff000; - val |= (uint64_t)((obj->stride / 128) - 1) << - SANDYBRIDGE_FENCE_PITCH_SHIFT; + if (obj) { + u32 size = obj->gtt_space->size; - if (obj->tiling_mode == I915_TILING_Y) - val |= 1 << I965_FENCE_TILING_Y_SHIFT; - val |= I965_FENCE_REG_VALID; + val = (uint64_t)((obj->gtt_offset + size - 4096) & + 0xfffff000) << 32; + val |= obj->gtt_offset & 0xfffff000; + val |= (uint64_t)((obj->stride / 128) - 1) << + SANDYBRIDGE_FENCE_PITCH_SHIFT; - if (pipelined) { - int ret = intel_ring_begin(pipelined, 6); - if (ret) - return ret; - - intel_ring_emit(pipelined, MI_NOOP); - intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); - intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); - intel_ring_emit(pipelined, (u32)val); - intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); - intel_ring_emit(pipelined, (u32)(val >> 32)); - intel_ring_advance(pipelined); + if (obj->tiling_mode == I915_TILING_Y) + val |= 1 << I965_FENCE_TILING_Y_SHIFT; + val |= I965_FENCE_REG_VALID; } else - I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); + val = 0; - return 0; + I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); + POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); } -static int i965_write_fence_reg(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *pipelined) +static void i965_write_fence_reg(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - u32 size = obj->gtt_space->size; - int regnum = obj->fence_reg; uint64_t val; - val = (uint64_t)((obj->gtt_offset + size - 4096) & - 0xfffff000) << 32; - val |= obj->gtt_offset & 0xfffff000; - val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; - if (obj->tiling_mode == I915_TILING_Y) - val |= 1 << I965_FENCE_TILING_Y_SHIFT; - val |= I965_FENCE_REG_VALID; + if (obj) { + u32 size = obj->gtt_space->size; - if (pipelined) { - int ret = intel_ring_begin(pipelined, 6); - if (ret) - return ret; - - intel_ring_emit(pipelined, MI_NOOP); - intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); - intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); - intel_ring_emit(pipelined, (u32)val); - intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); - intel_ring_emit(pipelined, (u32)(val >> 32)); - intel_ring_advance(pipelined); + val = (uint64_t)((obj->gtt_offset + size - 4096) & + 0xfffff000) << 32; + val |= obj->gtt_offset & 0xfffff000; + val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; + if (obj->tiling_mode == I915_TILING_Y) + val |= 1 << I965_FENCE_TILING_Y_SHIFT; + val |= I965_FENCE_REG_VALID; } else - I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); + val = 0; - return 0; + I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); + POSTING_READ(FENCE_REG_965_0 + reg * 8); } -static int i915_write_fence_reg(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *pipelined) +static void i915_write_fence_reg(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - u32 size = obj->gtt_space->size; - u32 fence_reg, val, pitch_val; - int tile_width; - - if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || - (size & -size) != size || - (obj->gtt_offset & (size - 1)), - "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", - obj->gtt_offset, obj->map_and_fenceable, size)) - return -EINVAL; + u32 val; - if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) - tile_width = 128; - else - tile_width = 512; - - /* Note: pitch better be a power of two tile widths */ - pitch_val = obj->stride / tile_width; - pitch_val = ffs(pitch_val) - 1; - - val = obj->gtt_offset; - if (obj->tiling_mode == I915_TILING_Y) - val |= 1 << I830_FENCE_TILING_Y_SHIFT; - val |= I915_FENCE_SIZE_BITS(size); - val |= pitch_val << I830_FENCE_PITCH_SHIFT; - val |= I830_FENCE_REG_VALID; - - fence_reg = obj->fence_reg; - if (fence_reg < 8) - fence_reg = FENCE_REG_830_0 + fence_reg * 4; - else - fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; + if (obj) { + u32 size = obj->gtt_space->size; + int pitch_val; + int tile_width; - if (pipelined) { - int ret = intel_ring_begin(pipelined, 4); - if (ret) - return ret; + WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || + (size & -size) != size || + (obj->gtt_offset & (size - 1)), + "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", + obj->gtt_offset, obj->map_and_fenceable, size); - intel_ring_emit(pipelined, MI_NOOP); - intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(pipelined, fence_reg); - intel_ring_emit(pipelined, val); - intel_ring_advance(pipelined); + if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) + tile_width = 128; + else + tile_width = 512; + + /* Note: pitch better be a power of two tile widths */ + pitch_val = obj->stride / tile_width; + pitch_val = ffs(pitch_val) - 1; + + val = obj->gtt_offset; + if (obj->tiling_mode == I915_TILING_Y) + val |= 1 << I830_FENCE_TILING_Y_SHIFT; + val |= I915_FENCE_SIZE_BITS(size); + val |= pitch_val << I830_FENCE_PITCH_SHIFT; + val |= I830_FENCE_REG_VALID; } else - I915_WRITE(fence_reg, val); + val = 0; - return 0; + if (reg < 8) + reg = FENCE_REG_830_0 + reg * 4; + else + reg = FENCE_REG_945_8 + (reg - 8) * 4; + + I915_WRITE(reg, val); + POSTING_READ(reg); } -static int i830_write_fence_reg(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *pipelined) +static void i830_write_fence_reg(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - u32 size = obj->gtt_space->size; - int regnum = obj->fence_reg; uint32_t val; - uint32_t pitch_val; - if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || - (size & -size) != size || - (obj->gtt_offset & (size - 1)), - "object 0x%08x not 512K or pot-size 0x%08x aligned\n", - obj->gtt_offset, size)) - return -EINVAL; - - pitch_val = obj->stride / 128; - pitch_val = ffs(pitch_val) - 1; - - val = obj->gtt_offset; - if (obj->tiling_mode == I915_TILING_Y) - val |= 1 << I830_FENCE_TILING_Y_SHIFT; - val |= I830_FENCE_SIZE_BITS(size); - val |= pitch_val << I830_FENCE_PITCH_SHIFT; - val |= I830_FENCE_REG_VALID; + if (obj) { + u32 size = obj->gtt_space->size; + uint32_t pitch_val; + + WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || + (size & -size) != size || + (obj->gtt_offset & (size - 1)), + "object 0x%08x not 512K or pot-size 0x%08x aligned\n", + obj->gtt_offset, size); + + pitch_val = obj->stride / 128; + pitch_val = ffs(pitch_val) - 1; + + val = obj->gtt_offset; + if (obj->tiling_mode == I915_TILING_Y) + val |= 1 << I830_FENCE_TILING_Y_SHIFT; + val |= I830_FENCE_SIZE_BITS(size); + val |= pitch_val << I830_FENCE_PITCH_SHIFT; + val |= I830_FENCE_REG_VALID; + } else + val = 0; - if (pipelined) { - int ret = intel_ring_begin(pipelined, 4); - if (ret) - return ret; + I915_WRITE(FENCE_REG_830_0 + reg * 4, val); + POSTING_READ(FENCE_REG_830_0 + reg * 4); +} - intel_ring_emit(pipelined, MI_NOOP); - intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); - intel_ring_emit(pipelined, val); - intel_ring_advance(pipelined); - } else - I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); +static void i915_gem_write_fence(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj) +{ + switch (INTEL_INFO(dev)->gen) { + case 7: + case 6: sandybridge_write_fence_reg(dev, reg, obj); break; + case 5: + case 4: i965_write_fence_reg(dev, reg, obj); break; + case 3: i915_write_fence_reg(dev, reg, obj); break; + case 2: i830_write_fence_reg(dev, reg, obj); break; + default: break; + } +} - return 0; +static inline int fence_number(struct drm_i915_private *dev_priv, + struct drm_i915_fence_reg *fence) +{ + return fence - dev_priv->fence_regs; } -static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) +static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, + struct drm_i915_fence_reg *fence, + bool enable) { - return i915_seqno_passed(ring->get_seqno(ring), seqno); + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + int reg = fence_number(dev_priv, fence); + + i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); + + if (enable) { + obj->fence_reg = reg; + fence->obj = obj; + list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); + } else { + obj->fence_reg = I915_FENCE_REG_NONE; + fence->obj = NULL; + list_del_init(&fence->lru_list); + } } static int -i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *pipelined) +i915_gem_object_flush_fence(struct drm_i915_gem_object *obj) { int ret; if (obj->fenced_gpu_access) { if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { - ret = i915_gem_flush_ring(obj->last_fenced_ring, + ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); if (ret) return ret; @@ -2369,18 +2365,12 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, obj->fenced_gpu_access = false; } - if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { - if (!ring_passed_seqno(obj->last_fenced_ring, - obj->last_fenced_seqno)) { - ret = i915_wait_request(obj->last_fenced_ring, - obj->last_fenced_seqno, - true); - if (ret) - return ret; - } + if (obj->last_fenced_seqno) { + ret = i915_wait_request(obj->ring, obj->last_fenced_seqno); + if (ret) + return ret; obj->last_fenced_seqno = 0; - obj->last_fenced_ring = NULL; } /* Ensure that all CPU reads are completed before installing a fence @@ -2395,34 +2385,29 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, int i915_gem_object_put_fence(struct drm_i915_gem_object *obj) { + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; int ret; - if (obj->tiling_mode) - i915_gem_release_mmap(obj); - - ret = i915_gem_object_flush_fence(obj, NULL); + ret = i915_gem_object_flush_fence(obj); if (ret) return ret; - if (obj->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - - WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count); - i915_gem_clear_fence_reg(obj->base.dev, - &dev_priv->fence_regs[obj->fence_reg]); + if (obj->fence_reg == I915_FENCE_REG_NONE) + return 0; - obj->fence_reg = I915_FENCE_REG_NONE; - } + i915_gem_object_update_fence(obj, + &dev_priv->fence_regs[obj->fence_reg], + false); + i915_gem_object_fence_lost(obj); return 0; } static struct drm_i915_fence_reg * -i915_find_fence_reg(struct drm_device *dev, - struct intel_ring_buffer *pipelined) +i915_find_fence_reg(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_fence_reg *reg, *first, *avail; + struct drm_i915_fence_reg *reg, *avail; int i; /* First try to find a free reg */ @@ -2440,204 +2425,77 @@ i915_find_fence_reg(struct drm_device *dev, return NULL; /* None available, try to steal one or wait for a user to finish */ - avail = first = NULL; list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { if (reg->pin_count) continue; - if (first == NULL) - first = reg; - - if (!pipelined || - !reg->obj->last_fenced_ring || - reg->obj->last_fenced_ring == pipelined) { - avail = reg; - break; - } + return reg; } - if (avail == NULL) - avail = first; - - return avail; + return NULL; } /** - * i915_gem_object_get_fence - set up a fence reg for an object + * i915_gem_object_get_fence - set up fencing for an object * @obj: object to map through a fence reg - * @pipelined: ring on which to queue the change, or NULL for CPU access - * @interruptible: must we wait uninterruptibly for the register to retire? * * When mapping objects through the GTT, userspace wants to be able to write * to them without having to worry about swizzling if the object is tiled. - * * This function walks the fence regs looking for a free one for @obj, * stealing one if it can't find any. * * It then sets up the reg based on the object's properties: address, pitch * and tiling format. + * + * For an untiled surface, this removes any existing fence. */ int -i915_gem_object_get_fence(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *pipelined) +i915_gem_object_get_fence(struct drm_i915_gem_object *obj) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; + bool enable = obj->tiling_mode != I915_TILING_NONE; struct drm_i915_fence_reg *reg; int ret; - /* XXX disable pipelining. There are bugs. Shocking. */ - pipelined = NULL; + /* Have we updated the tiling parameters upon the object and so + * will need to serialise the write to the associated fence register? + */ + if (obj->fence_dirty) { + ret = i915_gem_object_flush_fence(obj); + if (ret) + return ret; + } /* Just update our place in the LRU if our fence is getting reused. */ if (obj->fence_reg != I915_FENCE_REG_NONE) { reg = &dev_priv->fence_regs[obj->fence_reg]; - list_move_tail(®->lru_list, &dev_priv->mm.fence_list); - - if (obj->tiling_changed) { - ret = i915_gem_object_flush_fence(obj, pipelined); - if (ret) - return ret; - - if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) - pipelined = NULL; - - if (pipelined) { - reg->setup_seqno = - i915_gem_next_request_seqno(pipelined); - obj->last_fenced_seqno = reg->setup_seqno; - obj->last_fenced_ring = pipelined; - } - - goto update; + if (!obj->fence_dirty) { + list_move_tail(®->lru_list, + &dev_priv->mm.fence_list); + return 0; } + } else if (enable) { + reg = i915_find_fence_reg(dev); + if (reg == NULL) + return -EDEADLK; - if (!pipelined) { - if (reg->setup_seqno) { - if (!ring_passed_seqno(obj->last_fenced_ring, - reg->setup_seqno)) { - ret = i915_wait_request(obj->last_fenced_ring, - reg->setup_seqno, - true); - if (ret) - return ret; - } + if (reg->obj) { + struct drm_i915_gem_object *old = reg->obj; - reg->setup_seqno = 0; - } - } else if (obj->last_fenced_ring && - obj->last_fenced_ring != pipelined) { - ret = i915_gem_object_flush_fence(obj, pipelined); + ret = i915_gem_object_flush_fence(old); if (ret) return ret; - } - - return 0; - } - - reg = i915_find_fence_reg(dev, pipelined); - if (reg == NULL) - return -EDEADLK; - - ret = i915_gem_object_flush_fence(obj, pipelined); - if (ret) - return ret; - - if (reg->obj) { - struct drm_i915_gem_object *old = reg->obj; - - drm_gem_object_reference(&old->base); - - if (old->tiling_mode) - i915_gem_release_mmap(old); - ret = i915_gem_object_flush_fence(old, pipelined); - if (ret) { - drm_gem_object_unreference(&old->base); - return ret; + i915_gem_object_fence_lost(old); } + } else + return 0; - if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) - pipelined = NULL; - - old->fence_reg = I915_FENCE_REG_NONE; - old->last_fenced_ring = pipelined; - old->last_fenced_seqno = - pipelined ? i915_gem_next_request_seqno(pipelined) : 0; - - drm_gem_object_unreference(&old->base); - } else if (obj->last_fenced_seqno == 0) - pipelined = NULL; - - reg->obj = obj; - list_move_tail(®->lru_list, &dev_priv->mm.fence_list); - obj->fence_reg = reg - dev_priv->fence_regs; - obj->last_fenced_ring = pipelined; - - reg->setup_seqno = - pipelined ? i915_gem_next_request_seqno(pipelined) : 0; - obj->last_fenced_seqno = reg->setup_seqno; - -update: - obj->tiling_changed = false; - switch (INTEL_INFO(dev)->gen) { - case 7: - case 6: - ret = sandybridge_write_fence_reg(obj, pipelined); - break; - case 5: - case 4: - ret = i965_write_fence_reg(obj, pipelined); - break; - case 3: - ret = i915_write_fence_reg(obj, pipelined); - break; - case 2: - ret = i830_write_fence_reg(obj, pipelined); - break; - } - - return ret; -} - -/** - * i915_gem_clear_fence_reg - clear out fence register info - * @obj: object to clear - * - * Zeroes out the fence register itself and clears out the associated - * data structures in dev_priv and obj. - */ -static void -i915_gem_clear_fence_reg(struct drm_device *dev, - struct drm_i915_fence_reg *reg) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - uint32_t fence_reg = reg - dev_priv->fence_regs; - - switch (INTEL_INFO(dev)->gen) { - case 7: - case 6: - I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); - break; - case 5: - case 4: - I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); - break; - case 3: - if (fence_reg >= 8) - fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; - else - case 2: - fence_reg = FENCE_REG_830_0 + fence_reg * 4; - - I915_WRITE(fence_reg, 0); - break; - } + i915_gem_object_update_fence(obj, reg, enable); + obj->fence_dirty = false; - list_del_init(®->lru_list); - reg->obj = NULL; - reg->setup_seqno = 0; - reg->pin_count = 0; + return 0; } /** @@ -2749,7 +2607,7 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, return ret; } - ret = i915_gem_gtt_bind_object(obj); + ret = i915_gem_gtt_prepare_object(obj); if (ret) { i915_gem_object_put_pages_gtt(obj); drm_mm_put_block(obj->gtt_space); @@ -2761,6 +2619,9 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, goto search_free; } + if (!dev_priv->mm.aliasing_ppgtt) + i915_gem_gtt_bind_object(obj, obj->cache_level); + list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); @@ -2878,6 +2739,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) int i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) { + drm_i915_private_t *dev_priv = obj->base.dev->dev_private; uint32_t old_write_domain, old_read_domains; int ret; @@ -2918,6 +2780,10 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) old_read_domains, old_write_domain); + /* And bump the LRU for this access */ + if (i915_gem_object_is_inactive(obj)) + list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); + return 0; } @@ -2953,7 +2819,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, return ret; } - i915_gem_gtt_rebind_object(obj, cache_level); + if (obj->has_global_gtt_mapping) + i915_gem_gtt_bind_object(obj, cache_level); if (obj->has_aliasing_ppgtt_mapping) i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, obj, cache_level); @@ -2990,11 +2857,6 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, * Prepare buffer for display plane (scanout, cursors, etc). * Can be called from an uninterruptible phase (modesetting) and allows * any flushes to be pipelined (for pageflips). - * - * For the display plane, we want to be in the GTT but out of any write - * domains. So in many ways this looks like set_to_gtt_domain() apart from the - * ability to pipeline the waits, pinning and any additional subtleties - * that may differentiate the display plane from ordinary buffers. */ int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, @@ -3009,8 +2871,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, return ret; if (pipelined != obj->ring) { - ret = i915_gem_object_wait_rendering(obj); - if (ret == -ERESTARTSYS) + ret = i915_gem_object_sync(obj, pipelined); + if (ret) return ret; } @@ -3082,7 +2944,7 @@ i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) * This function returns when the move is complete, including waiting on * flushes to occur. */ -static int +int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) { uint32_t old_write_domain, old_read_domains; @@ -3095,17 +2957,14 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) if (ret) return ret; - ret = i915_gem_object_wait_rendering(obj); - if (ret) - return ret; + if (write || obj->pending_gpu_write) { + ret = i915_gem_object_wait_rendering(obj); + if (ret) + return ret; + } i915_gem_object_flush_gtt_write_domain(obj); - /* If we have a partially-valid cache of the object in the CPU, - * finish invalidating it and free the per-page flags. - */ - i915_gem_object_set_to_full_cpu_read_domain(obj); - old_write_domain = obj->base.write_domain; old_read_domains = obj->base.read_domains; @@ -3136,113 +2995,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) return 0; } -/** - * Moves the object from a partially CPU read to a full one. - * - * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(), - * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). - */ -static void -i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) -{ - if (!obj->page_cpu_valid) - return; - - /* If we're partially in the CPU read domain, finish moving it in. - */ - if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) { - int i; - - for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { - if (obj->page_cpu_valid[i]) - continue; - drm_clflush_pages(obj->pages + i, 1); - } - } - - /* Free the page_cpu_valid mappings which are now stale, whether - * or not we've got I915_GEM_DOMAIN_CPU. - */ - kfree(obj->page_cpu_valid); - obj->page_cpu_valid = NULL; -} - -/** - * Set the CPU read domain on a range of the object. - * - * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's - * not entirely valid. The page_cpu_valid member of the object flags which - * pages have been flushed, and will be respected by - * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping - * of the whole object. - * - * This function returns when the move is complete, including waiting on - * flushes to occur. - */ -static int -i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, - uint64_t offset, uint64_t size) -{ - uint32_t old_read_domains; - int i, ret; - - if (offset == 0 && size == obj->base.size) - return i915_gem_object_set_to_cpu_domain(obj, 0); - - ret = i915_gem_object_flush_gpu_write_domain(obj); - if (ret) - return ret; - - ret = i915_gem_object_wait_rendering(obj); - if (ret) - return ret; - - i915_gem_object_flush_gtt_write_domain(obj); - - /* If we're already fully in the CPU read domain, we're done. */ - if (obj->page_cpu_valid == NULL && - (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) - return 0; - - /* Otherwise, create/clear the per-page CPU read domain flag if we're - * newly adding I915_GEM_DOMAIN_CPU - */ - if (obj->page_cpu_valid == NULL) { - obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE, - GFP_KERNEL); - if (obj->page_cpu_valid == NULL) - return -ENOMEM; - } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) - memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); - - /* Flush the cache on any pages that are still invalid from the CPU's - * perspective. - */ - for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; - i++) { - if (obj->page_cpu_valid[i]) - continue; - - drm_clflush_pages(obj->pages + i, 1); - - obj->page_cpu_valid[i] = 1; - } - - /* It should now be out of any other write domains, and we can update - * the domain values for our changes. - */ - BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); - - old_read_domains = obj->base.read_domains; - obj->base.read_domains |= I915_GEM_DOMAIN_CPU; - - trace_i915_gem_object_change_domain(obj, - old_read_domains, - obj->base.write_domain); - - return 0; -} - /* Throttle our rendering by waiting until the ring has completed our requests * emitted over 20 msec ago. * @@ -3280,28 +3032,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) if (seqno == 0) return 0; - ret = 0; - if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { - /* And wait for the seqno passing without holding any locks and - * causing extra latency for others. This is safe as the irq - * generation is designed to be run atomically and so is - * lockless. - */ - if (ring->irq_get(ring)) { - ret = wait_event_interruptible(ring->irq_queue, - i915_seqno_passed(ring->get_seqno(ring), seqno) - || atomic_read(&dev_priv->mm.wedged)); - ring->irq_put(ring); - - if (ret == 0 && atomic_read(&dev_priv->mm.wedged)) - ret = -EIO; - } else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring), - seqno) || - atomic_read(&dev_priv->mm.wedged), 3000)) { - ret = -EBUSY; - } - } - + ret = __wait_seqno(ring, seqno, true); if (ret == 0) queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); @@ -3313,12 +3044,9 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, bool map_and_fenceable) { - struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; int ret; BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); - WARN_ON(i915_verify_lists(dev)); if (obj->gtt_space != NULL) { if ((alignment && obj->gtt_offset & (alignment - 1)) || @@ -3343,34 +3071,23 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj, return ret; } - if (obj->pin_count++ == 0) { - if (!obj->active) - list_move_tail(&obj->mm_list, - &dev_priv->mm.pinned_list); - } + if (!obj->has_global_gtt_mapping && map_and_fenceable) + i915_gem_gtt_bind_object(obj, obj->cache_level); + + obj->pin_count++; obj->pin_mappable |= map_and_fenceable; - WARN_ON(i915_verify_lists(dev)); return 0; } void i915_gem_object_unpin(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->base.dev; - drm_i915_private_t *dev_priv = dev->dev_private; - - WARN_ON(i915_verify_lists(dev)); BUG_ON(obj->pin_count == 0); BUG_ON(obj->gtt_space == NULL); - if (--obj->pin_count == 0) { - if (!obj->active) - list_move_tail(&obj->mm_list, - &dev_priv->mm.inactive_list); + if (--obj->pin_count == 0) obj->pin_mappable = false; - } - WARN_ON(i915_verify_lists(dev)); } int @@ -3494,20 +3211,9 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); - } else if (obj->ring->outstanding_lazy_request == - obj->last_rendering_seqno) { - struct drm_i915_gem_request *request; - - /* This ring is not being cleared by active usage, - * so emit a request to do so. - */ - request = kzalloc(sizeof(*request), GFP_KERNEL); - if (request) { - ret = i915_add_request(obj->ring, NULL, request); - if (ret) - kfree(request); - } else - ret = -ENOMEM; + } else { + ret = i915_gem_check_olr(obj->ring, + obj->last_rendering_seqno); } /* Update the active list for the hardware's current position. @@ -3643,46 +3349,42 @@ int i915_gem_init_object(struct drm_gem_object *obj) return 0; } -static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj) +void i915_gem_free_object(struct drm_gem_object *gem_obj) { + struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - int ret; - - ret = i915_gem_object_unbind(obj); - if (ret == -ERESTARTSYS) { - list_move(&obj->mm_list, - &dev_priv->mm.deferred_free_list); - return; - } trace_i915_gem_object_destroy(obj); + if (gem_obj->import_attach) + drm_prime_gem_destroy(gem_obj, obj->sg_table); + + if (obj->phys_obj) + i915_gem_detach_phys_object(dev, obj); + + obj->pin_count = 0; + if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) { + bool was_interruptible; + + was_interruptible = dev_priv->mm.interruptible; + dev_priv->mm.interruptible = false; + + WARN_ON(i915_gem_object_unbind(obj)); + + dev_priv->mm.interruptible = was_interruptible; + } + if (obj->base.map_list.map) drm_gem_free_mmap_offset(&obj->base); drm_gem_object_release(&obj->base); i915_gem_info_remove_obj(dev_priv, obj->base.size); - kfree(obj->page_cpu_valid); kfree(obj->bit_17); kfree(obj); } -void i915_gem_free_object(struct drm_gem_object *gem_obj) -{ - struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); - struct drm_device *dev = obj->base.dev; - - while (obj->pin_count > 0) - i915_gem_object_unpin(obj); - - if (obj->phys_obj) - i915_gem_detach_phys_object(dev, obj); - - i915_gem_free_object_tail(obj); -} - int i915_gem_idle(struct drm_device *dev) { @@ -3696,20 +3398,16 @@ i915_gem_idle(struct drm_device *dev) return 0; } - ret = i915_gpu_idle(dev, true); + ret = i915_gpu_idle(dev); if (ret) { mutex_unlock(&dev->struct_mutex); return ret; } + i915_gem_retire_requests(dev); /* Under UMS, be paranoid and evict. */ - if (!drm_core_check_feature(dev, DRIVER_MODESET)) { - ret = i915_gem_evict_inactive(dev, false); - if (ret) { - mutex_unlock(&dev->struct_mutex); - return ret; - } - } + if (!drm_core_check_feature(dev, DRIVER_MODESET)) + i915_gem_evict_everything(dev, false); i915_gem_reset_fences(dev); @@ -3747,9 +3445,9 @@ void i915_gem_init_swizzling(struct drm_device *dev) I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); if (IS_GEN6(dev)) - I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); + I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); else - I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); + I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); } void i915_gem_init_ppgtt(struct drm_device *dev) @@ -3787,21 +3485,27 @@ void i915_gem_init_ppgtt(struct drm_device *dev) pd_offset <<= 16; if (INTEL_INFO(dev)->gen == 6) { - uint32_t ecochk = I915_READ(GAM_ECOCHK); + uint32_t ecochk, gab_ctl, ecobits; + + ecobits = I915_READ(GAC_ECO_BITS); + I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); + + gab_ctl = I915_READ(GAB_CTL); + I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); + + ecochk = I915_READ(GAM_ECOCHK); I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); - I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); + I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); } else if (INTEL_INFO(dev)->gen >= 7) { I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); /* GFX_MODE is per-ring on gen7+ */ } - for (i = 0; i < I915_NUM_RINGS; i++) { - ring = &dev_priv->ring[i]; - + for_each_ring(ring, dev_priv, i) { if (INTEL_INFO(dev)->gen >= 7) I915_WRITE(RING_MODE_GEN7(ring), - GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); + _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); @@ -3845,14 +3549,80 @@ cleanup_render_ring: return ret; } +static bool +intel_enable_ppgtt(struct drm_device *dev) +{ + if (i915_enable_ppgtt >= 0) + return i915_enable_ppgtt; + +#ifdef CONFIG_INTEL_IOMMU + /* Disable ppgtt on SNB if VT-d is on. */ + if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) + return false; +#endif + + return true; +} + +int i915_gem_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + unsigned long gtt_size, mappable_size; + int ret; + + gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; + mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; + + mutex_lock(&dev->struct_mutex); + if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { + /* PPGTT pdes are stolen from global gtt ptes, so shrink the + * aperture accordingly when using aliasing ppgtt. */ + gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; + + i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); + + ret = i915_gem_init_aliasing_ppgtt(dev); + if (ret) { + mutex_unlock(&dev->struct_mutex); + return ret; + } + } else { + /* Let GEM Manage all of the aperture. + * + * However, leave one page at the end still bound to the scratch + * page. There are a number of places where the hardware + * apparently prefetches past the end of the object, and we've + * seen multiple hangs with the GPU head pointer stuck in a + * batchbuffer bound at the last page of the aperture. One page + * should be enough to keep any prefetching inside of the + * aperture. + */ + i915_gem_init_global_gtt(dev, 0, mappable_size, + gtt_size); + } + + ret = i915_gem_init_hw(dev); + mutex_unlock(&dev->struct_mutex); + if (ret) { + i915_gem_cleanup_aliasing_ppgtt(dev); + return ret; + } + + /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ + if (!drm_core_check_feature(dev, DRIVER_MODESET)) + dev_priv->dri1.allow_batchbuffer = 1; + return 0; +} + void i915_gem_cleanup_ringbuffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring; int i; - for (i = 0; i < I915_NUM_RINGS; i++) - intel_cleanup_ring_buffer(&dev_priv->ring[i]); + for_each_ring(ring, dev_priv, i) + intel_cleanup_ring_buffer(ring); } int @@ -3860,7 +3630,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; - int ret, i; + int ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return 0; @@ -3882,10 +3652,6 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data, BUG_ON(!list_empty(&dev_priv->mm.active_list)); BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); - for (i = 0; i < I915_NUM_RINGS; i++) { - BUG_ON(!list_empty(&dev_priv->ring[i].active_list)); - BUG_ON(!list_empty(&dev_priv->ring[i].request_list)); - } mutex_unlock(&dev->struct_mutex); ret = drm_irq_install(dev); @@ -3944,9 +3710,7 @@ i915_gem_load(struct drm_device *dev) INIT_LIST_HEAD(&dev_priv->mm.active_list); INIT_LIST_HEAD(&dev_priv->mm.flushing_list); INIT_LIST_HEAD(&dev_priv->mm.inactive_list); - INIT_LIST_HEAD(&dev_priv->mm.pinned_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); - INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); INIT_LIST_HEAD(&dev_priv->mm.gtt_list); for (i = 0; i < I915_NUM_RINGS; i++) init_ring_lists(&dev_priv->ring[i]); @@ -3958,12 +3722,8 @@ i915_gem_load(struct drm_device *dev) /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ if (IS_GEN3(dev)) { - u32 tmp = I915_READ(MI_ARB_STATE); - if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { - /* arb state is a masked write, so set bit + bit in mask */ - tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); - I915_WRITE(MI_ARB_STATE, tmp); - } + I915_WRITE(MI_ARB_STATE, + _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); } dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; @@ -3978,9 +3738,7 @@ i915_gem_load(struct drm_device *dev) dev_priv->num_fence_regs = 8; /* Initialize fence registers to zero */ - for (i = 0; i < dev_priv->num_fence_regs; i++) { - i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); - } + i915_gem_reset_fences(dev); i915_gem_detect_bit_6_swizzle(dev); init_waitqueue_head(&dev_priv->pending_flip_queue); @@ -4268,7 +4026,7 @@ rescan: * This has a dramatic impact to reduce the number of * OOM-killer events whilst running the GPU aggressively. */ - if (i915_gpu_idle(dev, true) == 0) + if (i915_gpu_idle(dev) == 0) goto rescan; } mutex_unlock(&dev->struct_mutex); |