diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_gem.c | 303 |
1 files changed, 246 insertions, 57 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 80871c62a571..112699f71fa4 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -124,7 +124,7 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, args->size = roundup(args->size, PAGE_SIZE); /* Allocate the new object */ - obj = drm_gem_object_alloc(dev, args->size); + obj = i915_gem_alloc_object(dev, args->size); if (obj == NULL) return -ENOMEM; @@ -1051,7 +1051,9 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, * about to occur. */ if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { - list_move_tail(&obj_priv->fence_list, + struct drm_i915_fence_reg *reg = + &dev_priv->fence_regs[obj_priv->fence_reg]; + list_move_tail(®->lru_list, &dev_priv->mm.fence_list); } @@ -1566,7 +1568,7 @@ i915_gem_process_flushing_list(struct drm_device *dev, list_for_each_entry_safe(obj_priv, next, &dev_priv->mm.gpu_write_list, gpu_write_list) { - struct drm_gem_object *obj = obj_priv->obj; + struct drm_gem_object *obj = &obj_priv->base; if ((obj->write_domain & flush_domains) == obj->write_domain) { @@ -1577,9 +1579,12 @@ i915_gem_process_flushing_list(struct drm_device *dev, i915_gem_object_move_to_active(obj, seqno); /* update the fence lru list */ - if (obj_priv->fence_reg != I915_FENCE_REG_NONE) - list_move_tail(&obj_priv->fence_list, + if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { + struct drm_i915_fence_reg *reg = + &dev_priv->fence_regs[obj_priv->fence_reg]; + list_move_tail(®->lru_list, &dev_priv->mm.fence_list); + } trace_i915_gem_object_change_domain(obj, obj->read_domains, @@ -1588,6 +1593,13 @@ i915_gem_process_flushing_list(struct drm_device *dev, } } +#define PIPE_CONTROL_FLUSH(addr) \ + OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \ + PIPE_CONTROL_DEPTH_STALL); \ + OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \ + OUT_RING(0); \ + OUT_RING(0); \ + /** * Creates a new sequence number, emitting a write of it to the status page * plus an interrupt, which will trigger i915_user_interrupt_handler. @@ -1622,13 +1634,47 @@ i915_add_request(struct drm_device *dev, struct drm_file *file_priv, if (dev_priv->mm.next_gem_seqno == 0) dev_priv->mm.next_gem_seqno++; - BEGIN_LP_RING(4); - OUT_RING(MI_STORE_DWORD_INDEX); - OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - OUT_RING(seqno); + if (HAS_PIPE_CONTROL(dev)) { + u32 scratch_addr = dev_priv->seqno_gfx_addr + 128; - OUT_RING(MI_USER_INTERRUPT); - ADVANCE_LP_RING(); + /* + * Workaround qword write incoherence by flushing the + * PIPE_NOTIFY buffers out to memory before requesting + * an interrupt. + */ + BEGIN_LP_RING(32); + OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH); + OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT); + OUT_RING(seqno); + OUT_RING(0); + PIPE_CONTROL_FLUSH(scratch_addr); + scratch_addr += 128; /* write to separate cachelines */ + PIPE_CONTROL_FLUSH(scratch_addr); + scratch_addr += 128; + PIPE_CONTROL_FLUSH(scratch_addr); + scratch_addr += 128; + PIPE_CONTROL_FLUSH(scratch_addr); + scratch_addr += 128; + PIPE_CONTROL_FLUSH(scratch_addr); + scratch_addr += 128; + PIPE_CONTROL_FLUSH(scratch_addr); + OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH | + PIPE_CONTROL_NOTIFY); + OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT); + OUT_RING(seqno); + OUT_RING(0); + ADVANCE_LP_RING(); + } else { + BEGIN_LP_RING(4); + OUT_RING(MI_STORE_DWORD_INDEX); + OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + OUT_RING(seqno); + + OUT_RING(MI_USER_INTERRUPT); + ADVANCE_LP_RING(); + } DRM_DEBUG_DRIVER("%d\n", seqno); @@ -1704,7 +1750,7 @@ i915_gem_retire_request(struct drm_device *dev, obj_priv = list_first_entry(&dev_priv->mm.active_list, struct drm_i915_gem_object, list); - obj = obj_priv->obj; + obj = &obj_priv->base; /* If the seqno being retired doesn't match the oldest in the * list, then the oldest in the list must still be newer than @@ -1752,7 +1798,10 @@ i915_get_gem_seqno(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; - return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX); + if (HAS_PIPE_CONTROL(dev)) + return ((volatile u32 *)(dev_priv->seqno_page))[0]; + else + return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX); } /** @@ -2075,7 +2124,7 @@ i915_gem_find_inactive_object(struct drm_device *dev, int min_size) /* Try to find the smallest clean object */ list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) { - struct drm_gem_object *obj = obj_priv->obj; + struct drm_gem_object *obj = &obj_priv->base; if (obj->size >= min_size) { if ((!obj_priv->dirty || i915_gem_object_is_purgeable(obj_priv)) && @@ -2209,7 +2258,7 @@ i915_gem_evict_something(struct drm_device *dev, int min_size) /* Find an object that we can immediately reuse */ list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) { - obj = obj_priv->obj; + obj = &obj_priv->base; if (obj->size >= min_size) break; @@ -2362,6 +2411,12 @@ static void i915_write_fence_reg(struct drm_i915_fence_reg *reg) pitch_val = obj_priv->stride / tile_width; pitch_val = ffs(pitch_val) - 1; + if (obj_priv->tiling_mode == I915_TILING_Y && + HAS_128_BYTE_Y_TILING(dev)) + WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL); + else + WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL); + val = obj_priv->gtt_offset; if (obj_priv->tiling_mode == I915_TILING_Y) val |= 1 << I830_FENCE_TILING_Y_SHIFT; @@ -2435,9 +2490,10 @@ static int i915_find_fence_reg(struct drm_device *dev) /* None available, try to steal one or wait for a user to finish */ i = I915_FENCE_REG_NONE; - list_for_each_entry(obj_priv, &dev_priv->mm.fence_list, - fence_list) { - obj = obj_priv->obj; + list_for_each_entry(reg, &dev_priv->mm.fence_list, + lru_list) { + obj = reg->obj; + obj_priv = to_intel_bo(obj); if (obj_priv->pin_count) continue; @@ -2486,7 +2542,8 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj) /* Just update our place in the LRU if our fence is getting used. */ if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { - list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list); + reg = &dev_priv->fence_regs[obj_priv->fence_reg]; + list_move_tail(®->lru_list, &dev_priv->mm.fence_list); return 0; } @@ -2516,7 +2573,7 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj) obj_priv->fence_reg = ret; reg = &dev_priv->fence_regs[obj_priv->fence_reg]; - list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list); + list_add_tail(®->lru_list, &dev_priv->mm.fence_list); reg->obj = obj; @@ -2548,6 +2605,8 @@ i915_gem_clear_fence_reg(struct drm_gem_object *obj) struct drm_device *dev = obj->dev; drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct drm_i915_fence_reg *reg = + &dev_priv->fence_regs[obj_priv->fence_reg]; if (IS_GEN6(dev)) { I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + @@ -2566,9 +2625,9 @@ i915_gem_clear_fence_reg(struct drm_gem_object *obj) I915_WRITE(fence_reg, 0); } - dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL; + reg->obj = NULL; obj_priv->fence_reg = I915_FENCE_REG_NONE; - list_del_init(&obj_priv->fence_list); + list_del_init(®->lru_list); } /** @@ -4421,34 +4480,38 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, return 0; } -int i915_gem_init_object(struct drm_gem_object *obj) +struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, + size_t size) { - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; - obj_priv = kzalloc(sizeof(*obj_priv), GFP_KERNEL); - if (obj_priv == NULL) - return -ENOMEM; + obj = kzalloc(sizeof(*obj), GFP_KERNEL); + if (obj == NULL) + return NULL; - /* - * We've just allocated pages from the kernel, - * so they've just been written by the CPU with - * zeros. They'll need to be clflushed before we - * use them with the GPU. - */ - obj->write_domain = I915_GEM_DOMAIN_CPU; - obj->read_domains = I915_GEM_DOMAIN_CPU; + if (drm_gem_object_init(dev, &obj->base, size) != 0) { + kfree(obj); + return NULL; + } - obj_priv->agp_type = AGP_USER_MEMORY; + obj->base.write_domain = I915_GEM_DOMAIN_CPU; + obj->base.read_domains = I915_GEM_DOMAIN_CPU; - obj->driver_private = obj_priv; - obj_priv->obj = obj; - obj_priv->fence_reg = I915_FENCE_REG_NONE; - INIT_LIST_HEAD(&obj_priv->list); - INIT_LIST_HEAD(&obj_priv->gpu_write_list); - INIT_LIST_HEAD(&obj_priv->fence_list); - obj_priv->madv = I915_MADV_WILLNEED; + obj->agp_type = AGP_USER_MEMORY; + obj->base.driver_private = NULL; + obj->fence_reg = I915_FENCE_REG_NONE; + INIT_LIST_HEAD(&obj->list); + INIT_LIST_HEAD(&obj->gpu_write_list); + obj->madv = I915_MADV_WILLNEED; + + trace_i915_gem_object_create(&obj->base); - trace_i915_gem_object_create(obj); + return &obj->base; +} + +int i915_gem_init_object(struct drm_gem_object *obj) +{ + BUG(); return 0; } @@ -4471,9 +4534,11 @@ void i915_gem_free_object(struct drm_gem_object *obj) if (obj_priv->mmap_offset) i915_gem_free_mmap_offset(obj); + drm_gem_object_release(obj); + kfree(obj_priv->page_cpu_valid); kfree(obj_priv->bit_17); - kfree(obj->driver_private); + kfree(obj_priv); } /** Unbinds all inactive objects. */ @@ -4486,9 +4551,9 @@ i915_gem_evict_from_inactive_list(struct drm_device *dev) struct drm_gem_object *obj; int ret; - obj = list_first_entry(&dev_priv->mm.inactive_list, - struct drm_i915_gem_object, - list)->obj; + obj = &list_first_entry(&dev_priv->mm.inactive_list, + struct drm_i915_gem_object, + list)->base; ret = i915_gem_object_unbind(obj); if (ret != 0) { @@ -4546,6 +4611,49 @@ i915_gem_idle(struct drm_device *dev) return 0; } +/* + * 965+ support PIPE_CONTROL commands, which provide finer grained control + * over cache flushing. + */ +static int +i915_gem_init_pipe_control(struct drm_device *dev) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_gem_object *obj; + struct drm_i915_gem_object *obj_priv; + int ret; + + obj = i915_gem_alloc_object(dev, 4096); + if (obj == NULL) { + DRM_ERROR("Failed to allocate seqno page\n"); + ret = -ENOMEM; + goto err; + } + obj_priv = to_intel_bo(obj); + obj_priv->agp_type = AGP_USER_CACHED_MEMORY; + + ret = i915_gem_object_pin(obj, 4096); + if (ret) + goto err_unref; + + dev_priv->seqno_gfx_addr = obj_priv->gtt_offset; + dev_priv->seqno_page = kmap(obj_priv->pages[0]); + if (dev_priv->seqno_page == NULL) + goto err_unpin; + + dev_priv->seqno_obj = obj; + memset(dev_priv->seqno_page, 0, PAGE_SIZE); + + return 0; + +err_unpin: + i915_gem_object_unpin(obj); +err_unref: + drm_gem_object_unreference(obj); +err: + return ret; +} + static int i915_gem_init_hws(struct drm_device *dev) { @@ -4560,10 +4668,11 @@ i915_gem_init_hws(struct drm_device *dev) if (!I915_NEED_GFX_HWS(dev)) return 0; - obj = drm_gem_object_alloc(dev, 4096); + obj = i915_gem_alloc_object(dev, 4096); if (obj == NULL) { DRM_ERROR("Failed to allocate status page\n"); - return -ENOMEM; + ret = -ENOMEM; + goto err; } obj_priv = to_intel_bo(obj); obj_priv->agp_type = AGP_USER_CACHED_MEMORY; @@ -4571,7 +4680,7 @@ i915_gem_init_hws(struct drm_device *dev) ret = i915_gem_object_pin(obj, 4096); if (ret != 0) { drm_gem_object_unreference(obj); - return ret; + goto err_unref; } dev_priv->status_gfx_addr = obj_priv->gtt_offset; @@ -4580,10 +4689,16 @@ i915_gem_init_hws(struct drm_device *dev) if (dev_priv->hw_status_page == NULL) { DRM_ERROR("Failed to map status page.\n"); memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); - i915_gem_object_unpin(obj); - drm_gem_object_unreference(obj); - return -EINVAL; + ret = -EINVAL; + goto err_unpin; } + + if (HAS_PIPE_CONTROL(dev)) { + ret = i915_gem_init_pipe_control(dev); + if (ret) + goto err_unpin; + } + dev_priv->hws_obj = obj; memset(dev_priv->hw_status_page, 0, PAGE_SIZE); if (IS_GEN6(dev)) { @@ -4596,6 +4711,30 @@ i915_gem_init_hws(struct drm_device *dev) DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr); return 0; + +err_unpin: + i915_gem_object_unpin(obj); +err_unref: + drm_gem_object_unreference(obj); +err: + return 0; +} + +static void +i915_gem_cleanup_pipe_control(struct drm_device *dev) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_gem_object *obj; + struct drm_i915_gem_object *obj_priv; + + obj = dev_priv->seqno_obj; + obj_priv = to_intel_bo(obj); + kunmap(obj_priv->pages[0]); + i915_gem_object_unpin(obj); + drm_gem_object_unreference(obj); + dev_priv->seqno_obj = NULL; + + dev_priv->seqno_page = NULL; } static void @@ -4619,6 +4758,9 @@ i915_gem_cleanup_hws(struct drm_device *dev) memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); dev_priv->hw_status_page = NULL; + if (HAS_PIPE_CONTROL(dev)) + i915_gem_cleanup_pipe_control(dev); + /* Write high address into HWS_PGA when disabling. */ I915_WRITE(HWS_PGA, 0x1ffff000); } @@ -4637,7 +4779,7 @@ i915_gem_init_ringbuffer(struct drm_device *dev) if (ret != 0) return ret; - obj = drm_gem_object_alloc(dev, 128 * 1024); + obj = i915_gem_alloc_object(dev, 128 * 1024); if (obj == NULL) { DRM_ERROR("Failed to allocate ringbuffer\n"); i915_gem_cleanup_hws(dev); @@ -4830,6 +4972,8 @@ i915_gem_load(struct drm_device *dev) INIT_LIST_HEAD(&dev_priv->mm.inactive_list); INIT_LIST_HEAD(&dev_priv->mm.request_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); + for (i = 0; i < 16; i++) + INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); INIT_DELAYED_WORK(&dev_priv->mm.retire_work, i915_gem_retire_work_handler); dev_priv->mm.next_gem_seqno = 1; @@ -5058,6 +5202,20 @@ void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv) } static int +i915_gpu_is_active(struct drm_device *dev) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + int lists_empty; + + spin_lock(&dev_priv->mm.active_list_lock); + lists_empty = list_empty(&dev_priv->mm.flushing_list) && + list_empty(&dev_priv->mm.active_list); + spin_unlock(&dev_priv->mm.active_list_lock); + + return !lists_empty; +} + +static int i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask) { drm_i915_private_t *dev_priv, *next_dev; @@ -5086,6 +5244,7 @@ i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask) spin_lock(&shrink_list_lock); +rescan: /* first scan for clean buffers */ list_for_each_entry_safe(dev_priv, next_dev, &shrink_list, mm.shrink_list) { @@ -5102,7 +5261,7 @@ i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask) &dev_priv->mm.inactive_list, list) { if (i915_gem_object_is_purgeable(obj_priv)) { - i915_gem_object_unbind(obj_priv->obj); + i915_gem_object_unbind(&obj_priv->base); if (--nr_to_scan <= 0) break; } @@ -5131,7 +5290,7 @@ i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask) &dev_priv->mm.inactive_list, list) { if (nr_to_scan > 0) { - i915_gem_object_unbind(obj_priv->obj); + i915_gem_object_unbind(&obj_priv->base); nr_to_scan--; } else cnt++; @@ -5143,6 +5302,36 @@ i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask) would_deadlock = 0; } + if (nr_to_scan) { + int active = 0; + + /* + * We are desperate for pages, so as a last resort, wait + * for the GPU to finish and discard whatever we can. + * This has a dramatic impact to reduce the number of + * OOM-killer events whilst running the GPU aggressively. + */ + list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { + struct drm_device *dev = dev_priv->dev; + + if (!mutex_trylock(&dev->struct_mutex)) + continue; + + spin_unlock(&shrink_list_lock); + + if (i915_gpu_is_active(dev)) { + i915_gpu_idle(dev); + active++; + } + + spin_lock(&shrink_list_lock); + mutex_unlock(&dev->struct_mutex); + } + + if (active) + goto rescan; + } + spin_unlock(&shrink_list_lock); if (would_deadlock) |