diff options
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_bo.c')
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 290 |
1 files changed, 102 insertions, 188 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index eb4b7df02ca0..9a03c7834b1e 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -45,7 +45,7 @@ static void ttm_bo_global_kobj_release(struct kobject *kobj); -/** +/* * ttm_global_mutex - protecting the global BO state */ DEFINE_MUTEX(ttm_global_mutex); @@ -115,10 +115,7 @@ static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo, struct ttm_bo_device *bdev = bo->bdev; struct ttm_resource_manager *man; - if (!list_empty(&bo->lru)) - return; - - if (mem->placement & TTM_PL_FLAG_NO_EVICT) + if (!list_empty(&bo->lru) || bo->pin_count) return; man = ttm_manager_type(bdev, mem->mem_type); @@ -165,7 +162,7 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, ttm_bo_del_from_lru(bo); ttm_bo_add_mem_to_lru(bo, &bo->mem); - if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { + if (bulk && !bo->pin_count) { switch (bo->mem.mem_type) { case TTM_PL_TT: ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); @@ -234,7 +231,8 @@ EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail); static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_resource *mem, bool evict, - struct ttm_operation_ctx *ctx) + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type); @@ -255,49 +253,20 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (ret) goto out_err; - ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); - if (ret) - goto out_err; - if (mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_populate(bdev, bo->ttm, ctx); - if (ret) - goto out_err; - - ret = ttm_bo_tt_bind(bo, mem); + ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx); if (ret) goto out_err; } - - if (bo->mem.mem_type == TTM_PL_SYSTEM) { - if (bdev->driver->move_notify) - bdev->driver->move_notify(bo, evict, mem); - bo->mem = *mem; - goto moved; - } } - if (bdev->driver->move_notify) - bdev->driver->move_notify(bo, evict, mem); - - if (old_man->use_tt && new_man->use_tt) - ret = ttm_bo_move_ttm(bo, ctx, mem); - else if (bdev->driver->move) - ret = bdev->driver->move(bo, evict, ctx, mem); - else - ret = ttm_bo_move_memcpy(bo, ctx, mem); - + ret = bdev->driver->move(bo, evict, ctx, mem, hop); if (ret) { - if (bdev->driver->move_notify) { - swap(*mem, bo->mem); - bdev->driver->move_notify(bo, false, mem); - swap(*mem, bo->mem); - } - + if (ret == -EMULTIHOP) + return ret; goto out_err; } -moved: ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; return 0; @@ -309,7 +278,7 @@ out_err: return ret; } -/** +/* * Call bo::reserved. * Will release GPU memory type usage on destruction. * This is the place to put in driver specific hooks to release @@ -319,8 +288,8 @@ out_err: static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) { - if (bo->bdev->driver->move_notify) - bo->bdev->driver->move_notify(bo, false, NULL); + if (bo->bdev->driver->delete_mem_notify) + bo->bdev->driver->delete_mem_notify(bo); ttm_bo_tt_destroy(bo); ttm_resource_free(bo, &bo->mem); @@ -383,9 +352,10 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) * Must be called with lru_lock and reservation held, this function * will drop the lru lock and optionally the reservation lock before returning. * - * @interruptible Any sleeps should occur interruptibly. - * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. - * @unlock_resv Unlock the reservation lock as well. + * @bo: The buffer object to clean-up + * @interruptible: Any sleeps should occur interruptibly. + * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. + * @unlock_resv: Unlock the reservation lock as well. */ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, @@ -451,7 +421,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, return 0; } -/** +/* * Traverse the delayed list, and call ttm_bo_cleanup_refs on all * encountered buffers. */ @@ -540,12 +510,12 @@ static void ttm_bo_release(struct kref *kref) spin_lock(&ttm_bo_glob.lru_lock); /* - * Make NO_EVICT bos immediately available to + * Make pinned bos immediately available to * shrinkers, now that they are queued for * destruction. */ - if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { - bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; + if (bo->pin_count) { + bo->pin_count = 0; ttm_bo_del_from_lru(bo); ttm_bo_add_mem_to_lru(bo, &bo->mem); } @@ -601,8 +571,11 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, struct ttm_bo_device *bdev = bo->bdev; struct ttm_resource evict_mem; struct ttm_placement placement; + struct ttm_place hop; int ret = 0; + memset(&hop, 0, sizeof(hop)); + dma_resv_assert_held(bo->base.resv); placement.num_placement = 0; @@ -631,8 +604,9 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, goto out; } - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx, &hop); if (unlikely(ret)) { + WARN(ret == -EMULTIHOP, "Unexpected multihop in eviction - likely driver bug\n"); if (ret != -ERESTARTSYS) pr_err("Buffer eviction failed\n"); ttm_resource_free(bo, &evict_mem); @@ -655,7 +629,7 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_eviction_valuable); -/** +/* * Check the target bo is allowable to be evicted or swapout, including cases: * * a. if share same reservation object with ctx->resv, have assumption @@ -672,7 +646,7 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, if (bo->base.resv == ctx->resv) { dma_resv_assert_held(bo->base.resv); - if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT) + if (ctx->allow_res_evict) ret = true; *locked = false; if (busy) @@ -794,7 +768,7 @@ int ttm_mem_evict_first(struct ttm_bo_device *bdev, return ret; } -/** +/* * Add the last move fence to the BO and reserve a new shared slot. */ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, @@ -830,7 +804,7 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, return 0; } -/** +/* * Repeatedly evict memory from the LRU for @mem_type until we create enough * space, or we've evicted everything and there isn't enough space. */ @@ -860,35 +834,11 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu); } -static uint32_t ttm_bo_select_caching(struct ttm_resource_manager *man, - uint32_t cur_placement, - uint32_t proposed_placement) -{ - uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; - uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; - - /** - * Keep current caching if possible. - */ - - if ((cur_placement & caching) != 0) - result |= (cur_placement & caching); - else if ((TTM_PL_FLAG_CACHED & caching) != 0) - result |= TTM_PL_FLAG_CACHED; - else if ((TTM_PL_FLAG_WC & caching) != 0) - result |= TTM_PL_FLAG_WC; - else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) - result |= TTM_PL_FLAG_UNCACHED; - - return result; -} - /** * ttm_bo_mem_placement - check if placement is compatible * @bo: BO to find memory for * @place: where to search * @mem: the memory object to fill in - * @ctx: operation context * * Check if placement is compatible and fill in mem structure. * Returns -EBUSY if placement won't work or negative error code. @@ -896,23 +846,17 @@ static uint32_t ttm_bo_select_caching(struct ttm_resource_manager *man, */ static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, const struct ttm_place *place, - struct ttm_resource *mem, - struct ttm_operation_ctx *ctx) + struct ttm_resource *mem) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_resource_manager *man; - uint32_t cur_flags = 0; man = ttm_manager_type(bdev, place->mem_type); if (!man || !ttm_resource_manager_used(man)) return -EBUSY; - cur_flags = ttm_bo_select_caching(man, bo->mem.placement, - place->flags); - cur_flags |= place->flags & ~TTM_PL_MASK_CACHING; - mem->mem_type = place->mem_type; - mem->placement = cur_flags; + mem->placement = place->flags; spin_lock(&ttm_bo_glob.lru_lock); ttm_bo_del_from_lru(bo); @@ -922,7 +866,7 @@ static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, return 0; } -/** +/* * Creates space for memory region @mem according to its type. * * This function first searches for free space in compatible memory types in @@ -947,7 +891,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, const struct ttm_place *place = &placement->placement[i]; struct ttm_resource_manager *man; - ret = ttm_bo_mem_placement(bo, place, mem, ctx); + ret = ttm_bo_mem_placement(bo, place, mem); if (ret) continue; @@ -973,7 +917,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, for (i = 0; i < placement->num_busy_placement; ++i) { const struct ttm_place *place = &placement->busy_placement[i]; - ret = ttm_bo_mem_placement(bo, place, mem, ctx); + ret = ttm_bo_mem_placement(bo, place, mem); if (ret) continue; @@ -1001,15 +945,45 @@ error: } EXPORT_SYMBOL(ttm_bo_mem_space); +static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, + struct ttm_resource *mem, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) +{ + struct ttm_placement hop_placement; + int ret; + struct ttm_resource hop_mem = *mem; + + hop_mem.mm_node = NULL; + hop_mem.mem_type = TTM_PL_SYSTEM; + hop_mem.placement = 0; + + hop_placement.num_placement = hop_placement.num_busy_placement = 1; + hop_placement.placement = hop_placement.busy_placement = hop; + + /* find space in the bounce domain */ + ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); + if (ret) + return ret; + /* move to the bounce domain */ + ret = ttm_bo_handle_move_mem(bo, &hop_mem, false, ctx, NULL); + if (ret) + return ret; + return 0; +} + static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_operation_ctx *ctx) { int ret = 0; + struct ttm_place hop; struct ttm_resource mem; dma_resv_assert_held(bo->base.resv); + memset(&hop, 0, sizeof(hop)); + mem.num_pages = bo->num_pages; mem.size = mem.num_pages << PAGE_SHIFT; mem.page_alignment = bo->mem.page_alignment; @@ -1019,12 +993,25 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, /* * Determine where to move the buffer. + * + * If driver determines move is going to need + * an extra step then it will return -EMULTIHOP + * and the buffer will be moved to the temporary + * stop and the driver will be called to make + * the second hop. */ +bounce: ret = ttm_bo_mem_space(bo, placement, &mem, ctx); if (ret) - goto out_unlock; - ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); -out_unlock: + return ret; + ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx, &hop); + if (ret == -EMULTIHOP) { + ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop); + if (ret) + return ret; + /* try and move to final place now. */ + goto bounce; + } if (ret) ttm_resource_free(bo, &mem); return ret; @@ -1045,8 +1032,7 @@ static bool ttm_bo_places_compat(const struct ttm_place *places, continue; *new_flags = heap->flags; - if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && - (mem->mem_type == heap->mem_type) && + if ((mem->mem_type == heap->mem_type) && (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) return true; @@ -1100,9 +1086,6 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, ret = ttm_bo_move_buffer(bo, placement, ctx); if (ret) return ret; - } else { - bo->mem.placement &= TTM_PL_MASK_CACHING; - bo->mem.placement |= new_flags & ~TTM_PL_MASK_CACHING; } /* * We might need to add a TTM. @@ -1170,8 +1153,9 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, bo->mem.bus.offset = 0; bo->mem.bus.addr = NULL; bo->moving = NULL; - bo->mem.placement = TTM_PL_FLAG_CACHED; + bo->mem.placement = 0; bo->acc_size = acc_size; + bo->pin_count = 0; bo->sg = sg; if (resv) { bo->base.resv = resv; @@ -1251,19 +1235,6 @@ int ttm_bo_init(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_bo_init); -static size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, - unsigned long bo_size, - unsigned struct_size) -{ - unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; - size_t size = 0; - - size += ttm_round_pot(struct_size); - size += ttm_round_pot(npages * sizeof(void *)); - size += ttm_round_pot(sizeof(struct ttm_tt)); - return size; -} - size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, unsigned long bo_size, unsigned struct_size) @@ -1273,56 +1244,11 @@ size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, size += ttm_round_pot(struct_size); size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); - size += ttm_round_pot(sizeof(struct ttm_dma_tt)); + size += ttm_round_pot(sizeof(struct ttm_tt)); return size; } EXPORT_SYMBOL(ttm_bo_dma_acc_size); -int ttm_bo_create(struct ttm_bo_device *bdev, - unsigned long size, - enum ttm_bo_type type, - struct ttm_placement *placement, - uint32_t page_alignment, - bool interruptible, - struct ttm_buffer_object **p_bo) -{ - struct ttm_buffer_object *bo; - size_t acc_size; - int ret; - - bo = kzalloc(sizeof(*bo), GFP_KERNEL); - if (unlikely(bo == NULL)) - return -ENOMEM; - - acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); - ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, - interruptible, acc_size, - NULL, NULL, NULL); - if (likely(ret == 0)) - *p_bo = bo; - - return ret; -} -EXPORT_SYMBOL(ttm_bo_create); - -int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) -{ - struct ttm_resource_manager *man = ttm_manager_type(bdev, mem_type); - - if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { - pr_err("Illegal memory manager memory type %u\n", mem_type); - return -EINVAL; - } - - if (!man) { - pr_err("Memory type %u has not been initialized\n", mem_type); - return 0; - } - - return ttm_resource_manager_force_list_clean(bdev, man); -} -EXPORT_SYMBOL(ttm_bo_evict_mm); - static void ttm_bo_global_kobj_release(struct kobject *kobj) { struct ttm_bo_global *glob = @@ -1409,6 +1335,8 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) pr_debug("Swap list %d was clean\n", i); spin_unlock(&glob->lru_lock); + ttm_pool_fini(&bdev->pool); + if (!ret) ttm_bo_global_release(); @@ -1433,9 +1361,10 @@ static void ttm_bo_init_sysman(struct ttm_bo_device *bdev) int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_driver *driver, + struct device *dev, struct address_space *mapping, struct drm_vma_offset_manager *vma_manager, - bool need_dma32) + bool use_dma_alloc, bool use_dma32) { struct ttm_bo_global *glob = &ttm_bo_glob; int ret; @@ -1450,12 +1379,12 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, bdev->driver = driver; ttm_bo_init_sysman(bdev); + ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32); bdev->vma_manager = vma_manager; INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); INIT_LIST_HEAD(&bdev->ddestroy); bdev->dev_mapping = mapping; - bdev->need_dma32 = need_dma32; mutex_lock(&ttm_global_mutex); list_add_tail(&bdev->device_list, &glob->device_list); mutex_unlock(&ttm_global_mutex); @@ -1502,12 +1431,13 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_wait); -/** +/* * A buffer object shrink method that tries to swap out the first * buffer object on the bo_global::swap_lru list. */ -int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) +int ttm_bo_swapout(struct ttm_operation_ctx *ctx) { + struct ttm_bo_global *glob = &ttm_bo_glob; struct ttm_buffer_object *bo; int ret = -EBUSY; bool locked; @@ -1551,19 +1481,23 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) * Move to system cached */ - if (bo->mem.mem_type != TTM_PL_SYSTEM || - bo->ttm->caching_state != tt_cached) { + if (bo->mem.mem_type != TTM_PL_SYSTEM) { struct ttm_operation_ctx ctx = { false, false }; struct ttm_resource evict_mem; + struct ttm_place hop; + + memset(&hop, 0, sizeof(hop)); evict_mem = bo->mem; evict_mem.mm_node = NULL; - evict_mem.placement = TTM_PL_FLAG_CACHED; + evict_mem.placement = 0; evict_mem.mem_type = TTM_PL_SYSTEM; - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); - if (unlikely(ret != 0)) + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx, &hop); + if (unlikely(ret != 0)) { + WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n"); goto out; + } } /** @@ -1584,7 +1518,7 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) if (bo->bdev->driver->swap_notify) bo->bdev->driver->swap_notify(bo); - ret = ttm_tt_swapout(bo->bdev, bo->ttm, bo->persistent_swap_storage); + ret = ttm_tt_swapout(bo->bdev, bo->ttm); out: /** @@ -1599,17 +1533,6 @@ out: } EXPORT_SYMBOL(ttm_bo_swapout); -void ttm_bo_swapout_all(void) -{ - struct ttm_operation_ctx ctx = { - .interruptible = false, - .no_wait_gpu = false - }; - - while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0); -} -EXPORT_SYMBOL(ttm_bo_swapout_all); - void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) { if (bo->ttm == NULL) @@ -1619,12 +1542,3 @@ void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) bo->ttm = NULL; } -int ttm_bo_tt_bind(struct ttm_buffer_object *bo, struct ttm_resource *mem) -{ - return bo->bdev->driver->ttm_tt_bind(bo->bdev, bo->ttm, mem); -} - -void ttm_bo_tt_unbind(struct ttm_buffer_object *bo) -{ - bo->bdev->driver->ttm_tt_unbind(bo->bdev, bo->ttm); -} |