diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/Makefile | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 250 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 379 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 156 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_device.c | 25 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_module.c | 35 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_module.h | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_range_manager.c | 95 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_resource.c | 236 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_sys_manager.c | 48 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 67 |
12 files changed, 788 insertions, 516 deletions
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index 40e5e9da7953..f906b22959cf 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -4,7 +4,7 @@ ttm-y := ttm_tt.o ttm_bo.o ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ ttm_execbuf_util.o ttm_range_manager.o ttm_resource.o ttm_pool.o \ - ttm_device.o + ttm_device.o ttm_sys_manager.o ttm-$(CONFIG_AGP) += ttm_agp_backend.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 0226ae69d3ab..6ddc16f0fe2b 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -32,8 +32,9 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_bo_driver.h> -#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_tt.h> +#include <drm/ttm/ttm_resource.h> #include <linux/agp_backend.h> #include <linux/module.h> #include <linux/slab.h> @@ -50,7 +51,6 @@ int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_resource *bo_mem) { struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); struct page *dummy_read_page = ttm_glob.dummy_read_page; - struct drm_mm_node *node = bo_mem->mm_node; struct agp_memory *mem; int ret, cached = ttm->caching == ttm_cached; unsigned i; @@ -76,7 +76,7 @@ int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_resource *bo_mem) mem->is_flushed = 1; mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY; - ret = agp_bind_memory(mem, node->start); + ret = agp_bind_memory(mem, bo_mem->start); if (ret) pr_err("AGP Bind memory failed\n"); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index ebcffe794adb..1b950b45cf4b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -58,7 +58,7 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, int i, mem_type; drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n", - bo, bo->mem.num_pages, bo->base.size >> 10, + bo, bo->resource->num_pages, bo->base.size >> 10, bo->base.size >> 20); for (i = 0; i < placement->num_placement; i++) { mem_type = placement->placement[i].mem_type; @@ -109,7 +109,7 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, bdev->funcs->del_from_lru_notify(bo); if (bulk && !bo->pin_count) { - switch (bo->mem.mem_type) { + switch (bo->resource->mem_type) { case TTM_PL_TT: ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); break; @@ -163,11 +163,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, struct ttm_place *hop) { + struct ttm_resource_manager *old_man, *new_man; struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type); - struct ttm_resource_manager *new_man = ttm_manager_type(bdev, mem->mem_type); int ret; + old_man = ttm_manager_type(bdev, bo->resource->mem_type); + new_man = ttm_manager_type(bdev, mem->mem_type); + ttm_bo_unmap_virtual(bo); /* @@ -200,7 +202,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, return 0; out_err: - new_man = ttm_manager_type(bdev, bo->mem.mem_type); + new_man = ttm_manager_type(bdev, bo->resource->mem_type); if (!new_man->use_tt) ttm_bo_tt_destroy(bo); @@ -221,7 +223,7 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) bo->bdev->funcs->delete_mem_notify(bo); ttm_bo_tt_destroy(bo); - ttm_resource_free(bo, &bo->mem); + ttm_resource_free(bo, &bo->resource); } static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) @@ -259,8 +261,8 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) int i; rcu_read_lock(); - fobj = rcu_dereference(resv->fence); - fence = rcu_dereference(resv->fence_excl); + fobj = dma_resv_shared_list(resv); + fence = dma_resv_excl_fence(resv); if (fence && !fence->ops->signaled) dma_fence_enable_sw_signaling(fence); @@ -274,7 +276,7 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) } /** - * function ttm_bo_cleanup_refs + * ttm_bo_cleanup_refs * If bo idle, remove from lru lists, and unref. * If not idle, block if possible. * @@ -294,7 +296,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, struct dma_resv *resv = &bo->base._resv; int ret; - if (dma_resv_test_signaled_rcu(resv, true)) + if (dma_resv_test_signaled(resv, true)) ret = 0; else ret = -EBUSY; @@ -306,8 +308,8 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, dma_resv_unlock(bo->base.resv); spin_unlock(&bo->bdev->lru_lock); - lret = dma_resv_wait_timeout_rcu(resv, true, interruptible, - 30 * HZ); + lret = dma_resv_wait_timeout(resv, true, interruptible, + 30 * HZ); if (lret < 0) return lret; @@ -401,24 +403,26 @@ static void ttm_bo_release(struct kref *kref) struct ttm_device *bdev = bo->bdev; int ret; + WARN_ON_ONCE(bo->pin_count); + if (!bo->deleted) { ret = ttm_bo_individualize_resv(bo); if (ret) { /* Last resort, if we fail to allocate memory for the * fences block for the BO to become idle */ - dma_resv_wait_timeout_rcu(bo->base.resv, true, false, - 30 * HZ); + dma_resv_wait_timeout(bo->base.resv, true, false, + 30 * HZ); } if (bo->bdev->funcs->release_notify) bo->bdev->funcs->release_notify(bo); drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); - ttm_mem_io_free(bdev, &bo->mem); + ttm_mem_io_free(bdev, bo->resource); } - if (!dma_resv_test_signaled_rcu(bo->base.resv, true) || + if (!dma_resv_test_signaled(bo->base.resv, true) || !dma_resv_trylock(bo->base.resv)) { /* The BO is not idle, resurrect it for delayed destroy */ ttm_bo_flush_all_fences(bo); @@ -434,9 +438,9 @@ static void ttm_bo_release(struct kref *kref) * FIXME: QXL is triggering this. Can be removed when the * driver is fixed. */ - if (WARN_ON_ONCE(bo->pin_count)) { + if (bo->pin_count) { bo->pin_count = 0; - ttm_bo_move_to_lru_tail(bo, &bo->mem, NULL); + ttm_bo_move_to_lru_tail(bo, bo->resource, NULL); } kref_init(&bo->kref); @@ -458,8 +462,6 @@ static void ttm_bo_release(struct kref *kref) atomic_dec(&ttm_glob.bo_count); dma_fence_put(bo->moving); - if (!ttm_bo_uses_embedded_gem_object(bo)) - dma_resv_fini(&bo->base._resv); bo->destroy(bo); } @@ -487,7 +489,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { struct ttm_device *bdev = bo->bdev; - struct ttm_resource evict_mem; + struct ttm_resource *evict_mem; struct ttm_placement placement; struct ttm_place hop; int ret = 0; @@ -501,17 +503,17 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bdev->funcs->evict_flags(bo, &placement); if (!placement.num_placement && !placement.num_busy_placement) { - ttm_bo_wait(bo, false, false); + ret = ttm_bo_wait(bo, true, false); + if (ret) + return ret; - ttm_bo_cleanup_memtype_use(bo); - return ttm_tt_create(bo, false); + /* + * Since we've already synced, this frees backing store + * immediately. + */ + return ttm_bo_pipeline_gutting(bo); } - evict_mem = bo->mem; - evict_mem.mm_node = NULL; - evict_mem.bus.offset = 0; - evict_mem.bus.addr = NULL; - ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); if (ret) { if (ret != -ERESTARTSYS) { @@ -522,7 +524,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, goto out; } - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx, &hop); + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); if (unlikely(ret)) { WARN(ret == -EMULTIHOP, "Unexpected multihop in eviction - likely driver bug\n"); if (ret != -ERESTARTSYS) @@ -536,11 +538,15 @@ out: bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) { + dma_resv_assert_held(bo->base.resv); + if (bo->resource->mem_type == TTM_PL_SYSTEM) + return true; + /* Don't evict this BO if it's outside of the * requested placement range */ - if (place->fpfn >= (bo->mem.start + bo->mem.num_pages) || - (place->lpfn && place->lpfn <= bo->mem.start)) + if (place->fpfn >= (bo->resource->start + bo->resource->num_pages) || + (place->lpfn && place->lpfn <= bo->resource->start)) return false; return true; @@ -558,7 +564,9 @@ EXPORT_SYMBOL(ttm_bo_eviction_valuable); * b. Otherwise, trylock it. */ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, - struct ttm_operation_ctx *ctx, bool *locked, bool *busy) + struct ttm_operation_ctx *ctx, + const struct ttm_place *place, + bool *locked, bool *busy) { bool ret = false; @@ -576,6 +584,14 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, *busy = !ret; } + if (ret && place && !bo->bdev->funcs->eviction_valuable(bo, place)) { + ret = false; + if (*locked) { + dma_resv_unlock(bo->base.resv); + *locked = false; + } + } + return ret; } @@ -630,20 +646,14 @@ int ttm_mem_evict_first(struct ttm_device *bdev, list_for_each_entry(bo, &man->lru[i], lru) { bool busy; - if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, - &busy)) { + if (!ttm_bo_evict_swapout_allowable(bo, ctx, place, + &locked, &busy)) { if (busy && !busy_bo && ticket != dma_resv_locking_ctx(bo->base.resv)) busy_bo = bo; continue; } - if (place && !bdev->funcs->eviction_valuable(bo, - place)) { - if (locked) - dma_resv_unlock(bo->base.resv); - continue; - } if (!ttm_bo_get_unless_zero(bo)) { if (locked) dma_resv_unlock(bo->base.resv); @@ -687,7 +697,9 @@ int ttm_mem_evict_first(struct ttm_device *bdev, } /* - * Add the last move fence to the BO and reserve a new shared slot. + * Add the last move fence to the BO and reserve a new shared slot. We only use + * a shared slot to avoid unecessary sync and rely on the subsequent bo move to + * either stall or use an exclusive fence respectively set bo->moving. */ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, struct ttm_resource_manager *man, @@ -729,14 +741,15 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, */ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, const struct ttm_place *place, - struct ttm_resource *mem, + struct ttm_resource **mem, struct ttm_operation_ctx *ctx) { struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *man = ttm_manager_type(bdev, mem->mem_type); + struct ttm_resource_manager *man; struct ww_acquire_ctx *ticket; int ret; + man = ttm_manager_type(bdev, place->mem_type); ticket = dma_resv_locking_ctx(bo->base.resv); do { ret = ttm_resource_alloc(bo, place, mem); @@ -750,37 +763,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ret; } while (1); - return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu); -} - -/** - * ttm_bo_mem_placement - check if placement is compatible - * @bo: BO to find memory for - * @place: where to search - * @mem: the memory object to fill in - * - * Check if placement is compatible and fill in mem structure. - * Returns -EBUSY if placement won't work or negative error code. - * 0 when placement can be used. - */ -static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, - const struct ttm_place *place, - struct ttm_resource *mem) -{ - struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *man; - - man = ttm_manager_type(bdev, place->mem_type); - if (!man || !ttm_resource_manager_used(man)) - return -EBUSY; - - mem->mem_type = place->mem_type; - mem->placement = place->flags; - - spin_lock(&bo->bdev->lru_lock); - ttm_bo_move_to_lru_tail(bo, mem, NULL); - spin_unlock(&bo->bdev->lru_lock); - return 0; + return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); } /* @@ -793,7 +776,7 @@ static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, */ int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, - struct ttm_resource *mem, + struct ttm_resource **mem, struct ttm_operation_ctx *ctx) { struct ttm_device *bdev = bo->bdev; @@ -808,8 +791,8 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, const struct ttm_place *place = &placement->placement[i]; struct ttm_resource_manager *man; - ret = ttm_bo_mem_placement(bo, place, mem); - if (ret) + man = ttm_manager_type(bdev, place->mem_type); + if (!man || !ttm_resource_manager_used(man)) continue; type_found = true; @@ -819,8 +802,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, if (unlikely(ret)) goto error; - man = ttm_manager_type(bdev, mem->mem_type); - ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu); + ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); if (unlikely(ret)) { ttm_resource_free(bo, mem); if (ret == -EBUSY) @@ -833,9 +815,10 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, for (i = 0; i < placement->num_busy_placement; ++i) { const struct ttm_place *place = &placement->busy_placement[i]; + struct ttm_resource_manager *man; - ret = ttm_bo_mem_placement(bo, place, mem); - if (ret) + man = ttm_manager_type(bdev, place->mem_type); + if (!man || !ttm_resource_manager_used(man)) continue; type_found = true; @@ -854,7 +837,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, } error: - if (bo->mem.mem_type == TTM_PL_SYSTEM && !bo->pin_count) + if (bo->resource->mem_type == TTM_PL_SYSTEM && !bo->pin_count) ttm_bo_move_to_lru_tail_unlocked(bo); return ret; @@ -862,17 +845,13 @@ error: EXPORT_SYMBOL(ttm_bo_mem_space); static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, - struct ttm_resource *mem, + struct ttm_resource **mem, struct ttm_operation_ctx *ctx, struct ttm_place *hop) { struct ttm_placement hop_placement; + struct ttm_resource *hop_mem; int ret; - struct ttm_resource hop_mem = *mem; - - hop_mem.mm_node = NULL; - hop_mem.mem_type = TTM_PL_SYSTEM; - hop_mem.placement = 0; hop_placement.num_placement = hop_placement.num_busy_placement = 1; hop_placement.placement = hop_placement.busy_placement = hop; @@ -882,7 +861,7 @@ static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, if (ret) return ret; /* move to the bounce domain */ - ret = ttm_bo_handle_move_mem(bo, &hop_mem, false, ctx, NULL); + ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); if (ret) { ttm_resource_free(bo, &hop_mem); return ret; @@ -894,20 +873,12 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_operation_ctx *ctx) { - int ret = 0; + struct ttm_resource *mem; struct ttm_place hop; - struct ttm_resource mem; + int ret; dma_resv_assert_held(bo->base.resv); - memset(&hop, 0, sizeof(hop)); - - mem.num_pages = PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT; - mem.page_alignment = bo->mem.page_alignment; - mem.bus.offset = 0; - mem.bus.addr = NULL; - mem.mm_node = NULL; - /* * Determine where to move the buffer. * @@ -921,7 +892,7 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, if (ret) return ret; bounce: - ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx, &hop); + ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop); if (ret == -EMULTIHOP) { ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop); if (ret) @@ -989,18 +960,13 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, /* * Remove the backing store if no placement is given. */ - if (!placement->num_placement && !placement->num_busy_placement) { - ret = ttm_bo_pipeline_gutting(bo); - if (ret) - return ret; - - return ttm_tt_create(bo, false); - } + if (!placement->num_placement && !placement->num_busy_placement) + return ttm_bo_pipeline_gutting(bo); /* * Check whether we need to move buffer. */ - if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { + if (!ttm_bo_mem_compat(placement, bo->resource, &new_flags)) { ret = ttm_bo_move_buffer(bo, placement, ctx); if (ret) return ret; @@ -1008,7 +974,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, /* * We might need to add a TTM. */ - if (bo->mem.mem_type == TTM_PL_SYSTEM) { + if (bo->resource->mem_type == TTM_PL_SYSTEM) { ret = ttm_tt_create(bo, true); if (ret) return ret; @@ -1028,8 +994,9 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { + static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM }; bool locked; - int ret = 0; + int ret; bo->destroy = destroy ? destroy : ttm_bo_default_destroy; @@ -1038,14 +1005,8 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, INIT_LIST_HEAD(&bo->ddestroy); bo->bdev = bdev; bo->type = type; - bo->mem.mem_type = TTM_PL_SYSTEM; - bo->mem.num_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; - bo->mem.mm_node = NULL; - bo->mem.page_alignment = page_alignment; - bo->mem.bus.offset = 0; - bo->mem.bus.addr = NULL; + bo->page_alignment = page_alignment; bo->moving = NULL; - bo->mem.placement = 0; bo->pin_count = 0; bo->sg = sg; if (resv) { @@ -1054,17 +1015,14 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, } else { bo->base.resv = &bo->base._resv; } - if (!ttm_bo_uses_embedded_gem_object(bo)) { - /* - * bo.base is not initialized, so we have to setup the - * struct elements we want use regardless. - */ - bo->base.size = size; - dma_resv_init(&bo->base._resv); - drm_vma_node_reset(&bo->base.vma_node); - } atomic_inc(&ttm_glob.bo_count); + ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource); + if (unlikely(ret)) { + ttm_bo_put(bo); + return ret; + } + /* * For ttm_bo_type_device buffers, allocate * address space from the device. @@ -1072,7 +1030,7 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, - bo->mem.num_pages); + bo->resource->num_pages); /* passed reservation objects should already be locked, * since otherwise lockdep will be angered in radeon. @@ -1134,7 +1092,7 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) struct ttm_device *bdev = bo->bdev; drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); - ttm_mem_io_free(bdev, &bo->mem); + ttm_mem_io_free(bdev, bo->resource); } EXPORT_SYMBOL(ttm_bo_unmap_virtual); @@ -1144,14 +1102,14 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, long timeout = 15 * HZ; if (no_wait) { - if (dma_resv_test_signaled_rcu(bo->base.resv, true)) + if (dma_resv_test_signaled(bo->base.resv, true)) return 0; else return -EBUSY; } - timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true, - interruptible, timeout); + timeout = dma_resv_wait_timeout(bo->base.resv, true, interruptible, + timeout); if (timeout < 0) return timeout; @@ -1166,10 +1124,19 @@ EXPORT_SYMBOL(ttm_bo_wait); int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, gfp_t gfp_flags) { + struct ttm_place place; bool locked; int ret; - if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, NULL)) + /* + * While the bo may already reside in SYSTEM placement, set + * SYSTEM as new placement to cover also the move further below. + * The driver may use the fact that we're moving from SYSTEM + * as an indication that we're about to swap out. + */ + memset(&place, 0, sizeof(place)); + place.mem_type = TTM_PL_SYSTEM; + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL)) return -EBUSY; if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || @@ -1194,19 +1161,17 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, /* * Move to system cached */ - if (bo->mem.mem_type != TTM_PL_SYSTEM) { + if (bo->resource->mem_type != TTM_PL_SYSTEM) { struct ttm_operation_ctx ctx = { false, false }; - struct ttm_resource evict_mem; + struct ttm_resource *evict_mem; struct ttm_place hop; memset(&hop, 0, sizeof(hop)); + ret = ttm_resource_alloc(bo, &place, &evict_mem); + if (unlikely(ret)) + goto out; - evict_mem = bo->mem; - evict_mem.mm_node = NULL; - evict_mem.placement = 0; - evict_mem.mem_type = TTM_PL_SYSTEM; - - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx, &hop); + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop); if (unlikely(ret != 0)) { WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n"); goto out; @@ -1229,7 +1194,8 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, if (bo->bdev->funcs->swap_notify) bo->bdev->funcs->swap_notify(bo); - ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags); + if (ttm_tt_is_populated(bo->ttm)) + ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags); out: /* diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index efb7e9c34ab4..2f57f824e6db 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -31,6 +31,7 @@ #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> +#include <drm/drm_cache.h> #include <drm/drm_vma_manager.h> #include <linux/dma-buf-map.h> #include <linux/io.h> @@ -72,190 +73,112 @@ void ttm_mem_io_free(struct ttm_device *bdev, mem->bus.addr = NULL; } -static int ttm_resource_ioremap(struct ttm_device *bdev, - struct ttm_resource *mem, - void **virtual) +/** + * ttm_move_memcpy - Helper to perform a memcpy ttm move operation. + * @bo: The struct ttm_buffer_object. + * @new_mem: The struct ttm_resource we're moving to (copy destination). + * @new_iter: A struct ttm_kmap_iter representing the destination resource. + * @src_iter: A struct ttm_kmap_iter representing the source resource. + * + * This function is intended to be able to move out async under a + * dma-fence if desired. + */ +void ttm_move_memcpy(struct ttm_buffer_object *bo, + u32 num_pages, + struct ttm_kmap_iter *dst_iter, + struct ttm_kmap_iter *src_iter) { - int ret; - void *addr; - - *virtual = NULL; - ret = ttm_mem_io_reserve(bdev, mem); - if (ret || !mem->bus.is_iomem) - return ret; + const struct ttm_kmap_iter_ops *dst_ops = dst_iter->ops; + const struct ttm_kmap_iter_ops *src_ops = src_iter->ops; + struct ttm_tt *ttm = bo->ttm; + struct dma_buf_map src_map, dst_map; + pgoff_t i; - if (mem->bus.addr) { - addr = mem->bus.addr; - } else { - size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; + /* Single TTM move. NOP */ + if (dst_ops->maps_tt && src_ops->maps_tt) + return; - if (mem->bus.caching == ttm_write_combined) - addr = ioremap_wc(mem->bus.offset, bus_size); -#ifdef CONFIG_X86 - else if (mem->bus.caching == ttm_cached) - addr = ioremap_cache(mem->bus.offset, bus_size); -#endif - else - addr = ioremap(mem->bus.offset, bus_size); - if (!addr) { - ttm_mem_io_free(bdev, mem); - return -ENOMEM; + /* Don't move nonexistent data. Clear destination instead. */ + if (src_ops->maps_tt && (!ttm || !ttm_tt_is_populated(ttm))) { + if (ttm && !(ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC)) + return; + + for (i = 0; i < num_pages; ++i) { + dst_ops->map_local(dst_iter, &dst_map, i); + if (dst_map.is_iomem) + memset_io(dst_map.vaddr_iomem, 0, PAGE_SIZE); + else + memset(dst_map.vaddr, 0, PAGE_SIZE); + if (dst_ops->unmap_local) + dst_ops->unmap_local(dst_iter, &dst_map); } + return; } - *virtual = addr; - return 0; -} - -static void ttm_resource_iounmap(struct ttm_device *bdev, - struct ttm_resource *mem, - void *virtual) -{ - if (virtual && mem->bus.addr == NULL) - iounmap(virtual); - ttm_mem_io_free(bdev, mem); -} - -static int ttm_copy_io_page(void *dst, void *src, unsigned long page) -{ - uint32_t *dstP = - (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); - uint32_t *srcP = - (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); - - int i; - for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) - iowrite32(ioread32(srcP++), dstP++); - return 0; -} - -static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, - unsigned long page, - pgprot_t prot) -{ - struct page *d = ttm->pages[page]; - void *dst; - - if (!d) - return -ENOMEM; - - src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); - dst = kmap_atomic_prot(d, prot); - if (!dst) - return -ENOMEM; - - memcpy_fromio(dst, src, PAGE_SIZE); - - kunmap_atomic(dst); - - return 0; -} - -static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, - unsigned long page, - pgprot_t prot) -{ - struct page *s = ttm->pages[page]; - void *src; - - if (!s) - return -ENOMEM; - - dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); - src = kmap_atomic_prot(s, prot); - if (!src) - return -ENOMEM; - memcpy_toio(dst, src, PAGE_SIZE); + for (i = 0; i < num_pages; ++i) { + dst_ops->map_local(dst_iter, &dst_map, i); + src_ops->map_local(src_iter, &src_map, i); - kunmap_atomic(src); + drm_memcpy_from_wc(&dst_map, &src_map, PAGE_SIZE); - return 0; + if (src_ops->unmap_local) + src_ops->unmap_local(src_iter, &src_map); + if (dst_ops->unmap_local) + dst_ops->unmap_local(dst_iter, &dst_map); + } } +EXPORT_SYMBOL(ttm_move_memcpy); int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, - struct ttm_resource *new_mem) + struct ttm_resource *dst_mem) { struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); + struct ttm_resource_manager *dst_man = + ttm_manager_type(bo->bdev, dst_mem->mem_type); struct ttm_tt *ttm = bo->ttm; - struct ttm_resource *old_mem = &bo->mem; - struct ttm_resource old_copy = *old_mem; - void *old_iomap; - void *new_iomap; - int ret; - unsigned long i; - - ret = ttm_bo_wait_ctx(bo, ctx); - if (ret) - return ret; - - ret = ttm_resource_ioremap(bdev, old_mem, &old_iomap); - if (ret) - return ret; - ret = ttm_resource_ioremap(bdev, new_mem, &new_iomap); - if (ret) - goto out; - - /* - * Single TTM move. NOP. - */ - if (old_iomap == NULL && new_iomap == NULL) - goto out2; - - /* - * Don't move nonexistent data. Clear destination instead. - */ - if (old_iomap == NULL && - (ttm == NULL || (!ttm_tt_is_populated(ttm) && - !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { - memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); - goto out2; - } + struct ttm_resource *src_mem = bo->resource; + struct ttm_resource_manager *src_man = + ttm_manager_type(bdev, src_mem->mem_type); + struct ttm_resource src_copy = *src_mem; + union { + struct ttm_kmap_iter_tt tt; + struct ttm_kmap_iter_linear_io io; + } _dst_iter, _src_iter; + struct ttm_kmap_iter *dst_iter, *src_iter; + int ret = 0; - /* - * TTM might be null for moves within the same region. - */ - if (ttm) { + if (ttm && ((ttm->page_flags & TTM_PAGE_FLAG_SWAPPED) || + dst_man->use_tt)) { ret = ttm_tt_populate(bdev, ttm, ctx); if (ret) - goto out1; + return ret; } - for (i = 0; i < new_mem->num_pages; ++i) { - if (old_iomap == NULL) { - pgprot_t prot = ttm_io_prot(bo, old_mem, PAGE_KERNEL); - ret = ttm_copy_ttm_io_page(ttm, new_iomap, i, - prot); - } else if (new_iomap == NULL) { - pgprot_t prot = ttm_io_prot(bo, new_mem, PAGE_KERNEL); - ret = ttm_copy_io_ttm_page(ttm, old_iomap, i, - prot); - } else { - ret = ttm_copy_io_page(new_iomap, old_iomap, i); - } - if (ret) - goto out1; + dst_iter = ttm_kmap_iter_linear_io_init(&_dst_iter.io, bdev, dst_mem); + if (PTR_ERR(dst_iter) == -EINVAL && dst_man->use_tt) + dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm); + if (IS_ERR(dst_iter)) + return PTR_ERR(dst_iter); + + src_iter = ttm_kmap_iter_linear_io_init(&_src_iter.io, bdev, src_mem); + if (PTR_ERR(src_iter) == -EINVAL && src_man->use_tt) + src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm); + if (IS_ERR(src_iter)) { + ret = PTR_ERR(src_iter); + goto out_src_iter; } - mb(); -out2: - old_copy = *old_mem; - ttm_bo_assign_mem(bo, new_mem); + ttm_move_memcpy(bo, dst_mem->num_pages, dst_iter, src_iter); + src_copy = *src_mem; + ttm_bo_move_sync_cleanup(bo, dst_mem); - if (!man->use_tt) - ttm_bo_tt_destroy(bo); + if (!src_iter->ops->maps_tt) + ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, &src_copy); +out_src_iter: + if (!dst_iter->ops->maps_tt) + ttm_kmap_iter_linear_io_fini(&_dst_iter.io, bdev, dst_mem); -out1: - ttm_resource_iounmap(bdev, old_mem, new_iomap); -out: - ttm_resource_iounmap(bdev, &old_copy, old_iomap); - - /* - * On error, keep the mm node! - */ - if (!ret) - ttm_resource_free(bo, &old_copy); return ret; } EXPORT_SYMBOL(ttm_bo_move_memcpy); @@ -336,27 +259,7 @@ pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, man = ttm_manager_type(bo->bdev, res->mem_type); caching = man->use_tt ? bo->ttm->caching : res->bus.caching; - /* Cached mappings need no adjustment */ - if (caching == ttm_cached) - return tmp; - -#if defined(__i386__) || defined(__x86_64__) - if (caching == ttm_write_combined) - tmp = pgprot_writecombine(tmp); - else if (boot_cpu_data.x86 > 3) - tmp = pgprot_noncached(tmp); -#endif -#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ - defined(__powerpc__) || defined(__mips__) - if (caching == ttm_write_combined) - tmp = pgprot_writecombine(tmp); - else - tmp = pgprot_noncached(tmp); -#endif -#if defined(__sparc__) - tmp = pgprot_noncached(tmp); -#endif - return tmp; + return ttm_prot_from_caching(caching, tmp); } EXPORT_SYMBOL(ttm_io_prot); @@ -365,24 +268,23 @@ static int ttm_bo_ioremap(struct ttm_buffer_object *bo, unsigned long size, struct ttm_bo_kmap_obj *map) { - struct ttm_resource *mem = &bo->mem; + struct ttm_resource *mem = bo->resource; - if (bo->mem.bus.addr) { + if (bo->resource->bus.addr) { map->bo_kmap_type = ttm_bo_map_premapped; - map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); + map->virtual = ((u8 *)bo->resource->bus.addr) + offset; } else { + resource_size_t res = bo->resource->bus.offset + offset; + map->bo_kmap_type = ttm_bo_map_iomap; if (mem->bus.caching == ttm_write_combined) - map->virtual = ioremap_wc(bo->mem.bus.offset + offset, - size); + map->virtual = ioremap_wc(res, size); #ifdef CONFIG_X86 else if (mem->bus.caching == ttm_cached) - map->virtual = ioremap_cache(bo->mem.bus.offset + offset, - size); + map->virtual = ioremap_cache(res, size); #endif else - map->virtual = ioremap(bo->mem.bus.offset + offset, - size); + map->virtual = ioremap(res, size); } return (!map->virtual) ? -ENOMEM : 0; } @@ -392,7 +294,7 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, unsigned long num_pages, struct ttm_bo_kmap_obj *map) { - struct ttm_resource *mem = &bo->mem; + struct ttm_resource *mem = bo->resource; struct ttm_operation_ctx ctx = { .interruptible = false, .no_wait_gpu = false @@ -438,15 +340,15 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, map->virtual = NULL; map->bo = bo; - if (num_pages > bo->mem.num_pages) + if (num_pages > bo->resource->num_pages) return -EINVAL; - if ((start_page + num_pages) > bo->mem.num_pages) + if ((start_page + num_pages) > bo->resource->num_pages) return -EINVAL; - ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); + ret = ttm_mem_io_reserve(bo->bdev, bo->resource); if (ret) return ret; - if (!bo->mem.bus.is_iomem) { + if (!bo->resource->bus.is_iomem) { return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); } else { offset = start_page << PAGE_SHIFT; @@ -475,7 +377,7 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) default: BUG(); } - ttm_mem_io_free(map->bo->bdev, &map->bo->mem); + ttm_mem_io_free(map->bo->bdev, map->bo->resource); map->virtual = NULL; map->page = NULL; } @@ -483,7 +385,7 @@ EXPORT_SYMBOL(ttm_bo_kunmap); int ttm_bo_vmap(struct ttm_buffer_object *bo, struct dma_buf_map *map) { - struct ttm_resource *mem = &bo->mem; + struct ttm_resource *mem = bo->resource; int ret; ret = ttm_mem_io_reserve(bo->bdev, mem); @@ -542,7 +444,7 @@ EXPORT_SYMBOL(ttm_bo_vmap); void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct dma_buf_map *map) { - struct ttm_resource *mem = &bo->mem; + struct ttm_resource *mem = bo->resource; if (dma_buf_map_is_null(map)) return; @@ -553,7 +455,7 @@ void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct dma_buf_map *map) iounmap(map->vaddr_iomem); dma_buf_map_clear(map); - ttm_mem_io_free(bo->bdev, &bo->mem); + ttm_mem_io_free(bo->bdev, bo->resource); } EXPORT_SYMBOL(ttm_bo_vunmap); @@ -567,7 +469,7 @@ static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, if (!dst_use_tt) ttm_bo_tt_destroy(bo); - ttm_resource_free(bo, &bo->mem); + ttm_resource_free(bo, &bo->resource); return 0; } @@ -605,6 +507,7 @@ static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, ghost_obj->ttm = NULL; else bo->ttm = NULL; + bo->resource = NULL; dma_resv_unlock(&ghost_obj->base._resv); ttm_bo_put(ghost_obj); @@ -615,7 +518,9 @@ static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, struct dma_fence *fence) { struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); + struct ttm_resource_manager *from; + + from = ttm_manager_type(bdev, bo->resource->mem_type); /** * BO doesn't have a TTM we need to bind/unbind. Just remember @@ -628,7 +533,7 @@ static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, } spin_unlock(&from->move_lock); - ttm_resource_free(bo, &bo->mem); + ttm_resource_free(bo, &bo->resource); dma_fence_put(bo->moving); bo->moving = dma_fence_get(fence); @@ -641,7 +546,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, struct ttm_resource *new_mem) { struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); + struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->resource->mem_type); struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); int ret = 0; @@ -662,26 +567,82 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); +/** + * ttm_bo_pipeline_gutting - purge the contents of a bo + * @bo: The buffer object + * + * Purge the contents of a bo, async if the bo is not idle. + * After a successful call, the bo is left unpopulated in + * system placement. The function may wait uninterruptible + * for idle on OOM. + * + * Return: 0 if successful, negative error code on failure. + */ int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) { + static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM }; struct ttm_buffer_object *ghost; + struct ttm_resource *sys_res; + struct ttm_tt *ttm; int ret; - ret = ttm_buffer_object_transfer(bo, &ghost); + ret = ttm_resource_alloc(bo, &sys_mem, &sys_res); if (ret) return ret; + /* If already idle, no need for ghost object dance. */ + ret = ttm_bo_wait(bo, false, true); + if (ret != -EBUSY) { + if (!bo->ttm) { + /* See comment below about clearing. */ + ret = ttm_tt_create(bo, true); + if (ret) + goto error_free_sys_mem; + } else { + ttm_tt_unpopulate(bo->bdev, bo->ttm); + if (bo->type == ttm_bo_type_device) + ttm_tt_mark_for_clear(bo->ttm); + } + ttm_resource_free(bo, &bo->resource); + ttm_bo_assign_mem(bo, sys_res); + return 0; + } + + /* + * We need an unpopulated ttm_tt after giving our current one, + * if any, to the ghost object. And we can't afford to fail + * creating one *after* the operation. If the bo subsequently gets + * resurrected, make sure it's cleared (if ttm_bo_type_device) + * to avoid leaking sensitive information to user-space. + */ + + ttm = bo->ttm; + bo->ttm = NULL; + ret = ttm_tt_create(bo, true); + swap(bo->ttm, ttm); + if (ret) + goto error_free_sys_mem; + + ret = ttm_buffer_object_transfer(bo, &ghost); + if (ret) + goto error_destroy_tt; + ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); /* Last resort, wait for the BO to be idle when we are OOM */ if (ret) ttm_bo_wait(bo, false, false); - memset(&bo->mem, 0, sizeof(bo->mem)); - bo->mem.mem_type = TTM_PL_SYSTEM; - bo->ttm = NULL; - dma_resv_unlock(&ghost->base._resv); ttm_bo_put(ghost); - + bo->ttm = ttm; + bo->resource = NULL; + ttm_bo_assign_mem(bo, sys_res); return 0; + +error_destroy_tt: + ttm_tt_destroy(bo->bdev, ttm); + +error_free_sys_mem: + ttm_resource_free(bo, &sys_res); + return ret; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index b31b18058965..f56be5bc0861 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -34,6 +34,8 @@ #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> #include <drm/drm_vma_manager.h> +#include <drm/drm_drv.h> +#include <drm/drm_managed.h> #include <linux/mm.h> #include <linux/pfn_t.h> #include <linux/rbtree.h> @@ -100,7 +102,7 @@ static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, if (bdev->funcs->io_mem_pfn) return bdev->funcs->io_mem_pfn(bo, page_offset); - return (bo->mem.bus.offset >> PAGE_SHIFT) + page_offset; + return (bo->resource->bus.offset >> PAGE_SHIFT) + page_offset; } /** @@ -198,10 +200,10 @@ static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf, /* Fault should not cross bo boundary. */ page_offset &= ~(fault_page_size - 1); - if (page_offset + fault_page_size > bo->mem.num_pages) + if (page_offset + fault_page_size > bo->resource->num_pages) goto out_fallback; - if (bo->mem.bus.is_iomem) + if (bo->resource->bus.is_iomem) pfn = ttm_bo_io_mem_pfn(bo, page_offset); else pfn = page_to_pfn(ttm->pages[page_offset]); @@ -211,7 +213,7 @@ static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf, goto out_fallback; /* Check that memory is contiguous. */ - if (!bo->mem.bus.is_iomem) { + if (!bo->resource->bus.is_iomem) { for (i = 1; i < fault_page_size; ++i) { if (page_to_pfn(ttm->pages[page_offset + i]) != pfn + i) goto out_fallback; @@ -297,7 +299,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, if (unlikely(ret != 0)) return ret; - err = ttm_mem_io_reserve(bdev, &bo->mem); + err = ttm_mem_io_reserve(bdev, bo->resource); if (unlikely(err != 0)) return VM_FAULT_SIGBUS; @@ -306,11 +308,11 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, page_last = vma_pages(vma) + vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); - if (unlikely(page_offset >= bo->mem.num_pages)) + if (unlikely(page_offset >= bo->resource->num_pages)) return VM_FAULT_SIGBUS; - prot = ttm_io_prot(bo, &bo->mem, prot); - if (!bo->mem.bus.is_iomem) { + prot = ttm_io_prot(bo, bo->resource, prot); + if (!bo->resource->bus.is_iomem) { struct ttm_operation_ctx ctx = { .interruptible = false, .no_wait_gpu = false, @@ -335,7 +337,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, * first page. */ for (i = 0; i < num_prefault; ++i) { - if (bo->mem.bus.is_iomem) { + if (bo->resource->bus.is_iomem) { pfn = ttm_bo_io_mem_pfn(bo, page_offset); } else { page = ttm->pages[page_offset]; @@ -357,12 +359,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, * at arbitrary times while the data is mmap'ed. * See vmf_insert_mixed_prot() for a discussion. */ - if (vma->vm_flags & VM_MIXEDMAP) - ret = vmf_insert_mixed_prot(vma, address, - __pfn_to_pfn_t(pfn, PFN_DEV), - prot); - else - ret = vmf_insert_pfn_prot(vma, address, pfn, prot); + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); /* Never error on prefaulted PTEs */ if (unlikely((ret & VM_FAULT_ERROR))) { @@ -380,19 +377,63 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, } EXPORT_SYMBOL(ttm_bo_vm_fault_reserved); +static void ttm_bo_release_dummy_page(struct drm_device *dev, void *res) +{ + struct page *dummy_page = (struct page *)res; + + __free_page(dummy_page); +} + +vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + struct drm_device *ddev = bo->base.dev; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long address; + unsigned long pfn; + struct page *page; + + /* Allocate new dummy page to map all the VA range in this VMA to it*/ + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return VM_FAULT_OOM; + + /* Set the page to be freed using drmm release action */ + if (drmm_add_action_or_reset(ddev, ttm_bo_release_dummy_page, page)) + return VM_FAULT_OOM; + + pfn = page_to_pfn(page); + + /* Prefault the entire VMA range right away to avoid further faults */ + for (address = vma->vm_start; address < vma->vm_end; + address += PAGE_SIZE) + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_dummy_page); + vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; pgprot_t prot; struct ttm_buffer_object *bo = vma->vm_private_data; + struct drm_device *ddev = bo->base.dev; vm_fault_t ret; + int idx; ret = ttm_bo_vm_reserve(bo, vmf); if (ret) return ret; prot = vma->vm_page_prot; - ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); + if (drm_dev_enter(ddev, &idx)) { + ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); + drm_dev_exit(idx); + } else { + ret = ttm_bo_vm_dummy_page(vmf, prot); + } if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) return ret; @@ -469,14 +510,14 @@ int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, << PAGE_SHIFT); int ret; - if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->mem.num_pages) + if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->resource->num_pages) return -EIO; ret = ttm_bo_reserve(bo, true, false, NULL); if (ret) return ret; - switch (bo->mem.mem_type) { + switch (bo->resource->mem_type) { case TTM_PL_SYSTEM: if (unlikely(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { ret = ttm_tt_swapin(bo->ttm); @@ -508,33 +549,20 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { .access = ttm_bo_vm_access, }; -static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_device *bdev, - unsigned long offset, - unsigned long pages) +int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) { - struct drm_vma_offset_node *node; - struct ttm_buffer_object *bo = NULL; - - drm_vma_offset_lock_lookup(bdev->vma_manager); - - node = drm_vma_offset_lookup_locked(bdev->vma_manager, offset, pages); - if (likely(node)) { - bo = container_of(node, struct ttm_buffer_object, - base.vma_node); - bo = ttm_bo_get_unless_zero(bo); - } - - drm_vma_offset_unlock_lookup(bdev->vma_manager); - - if (!bo) - pr_err("Could not find buffer object to map\n"); + /* Enforce no COW since would have really strange behavior with it. */ + if (is_cow_mapping(vma->vm_flags)) + return -EINVAL; - return bo; -} + ttm_bo_get(bo); -static void ttm_bo_mmap_vma_setup(struct ttm_buffer_object *bo, struct vm_area_struct *vma) -{ - vma->vm_ops = &ttm_bo_vm_ops; + /* + * Drivers may want to override the vm_ops field. Otherwise we + * use TTM's default callbacks. + */ + if (!vma->vm_ops) + vma->vm_ops = &ttm_bo_vm_ops; /* * Note: We're transferring the bo reference to @@ -543,50 +571,8 @@ static void ttm_bo_mmap_vma_setup(struct ttm_buffer_object *bo, struct vm_area_s vma->vm_private_data = bo; - /* - * We'd like to use VM_PFNMAP on shared mappings, where - * (vma->vm_flags & VM_SHARED) != 0, for performance reasons, - * but for some reason VM_PFNMAP + x86 PAT + write-combine is very - * bad for performance. Until that has been sorted out, use - * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 - */ - vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_PFNMAP; vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; -} - -int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, - struct ttm_device *bdev) -{ - struct ttm_buffer_object *bo; - int ret; - - if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET_START)) - return -EINVAL; - - bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); - if (unlikely(!bo)) - return -EINVAL; - - if (unlikely(!bo->bdev->funcs->verify_access)) { - ret = -EPERM; - goto out_unref; - } - ret = bo->bdev->funcs->verify_access(bo, filp); - if (unlikely(ret != 0)) - goto out_unref; - - ttm_bo_mmap_vma_setup(bo, vma); - return 0; -out_unref: - ttm_bo_put(bo); - return ret; -} -EXPORT_SYMBOL(ttm_bo_mmap); - -int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) -{ - ttm_bo_get(bo); - ttm_bo_mmap_vma_setup(bo, vma); return 0; } EXPORT_SYMBOL(ttm_bo_mmap_obj); diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c index 3d9c62b93e29..5f31acec3ad7 100644 --- a/drivers/gpu/drm/ttm/ttm_device.c +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -36,11 +36,11 @@ #include "ttm_module.h" -/** +/* * ttm_global_mutex - protecting the global state */ -DEFINE_MUTEX(ttm_global_mutex); -unsigned ttm_glob_use_count; +static DEFINE_MUTEX(ttm_global_mutex); +static unsigned ttm_glob_use_count; struct ttm_global ttm_glob; EXPORT_SYMBOL(ttm_glob); @@ -104,7 +104,7 @@ out: return ret; } -/** +/* * A buffer object shrink method that tries to swap out the first * buffer object on the global::swap_lru list. */ @@ -159,21 +159,6 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, } EXPORT_SYMBOL(ttm_device_swapout); -static void ttm_init_sysman(struct ttm_device *bdev) -{ - struct ttm_resource_manager *man = &bdev->sysman; - - /* - * Initialize the system memory buffer type. - * Other types need to be driver / IOCTL initialized. - */ - man->use_tt = true; - - ttm_resource_manager_init(man, 0); - ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man); - ttm_resource_manager_set_used(man, true); -} - static void ttm_device_delayed_workqueue(struct work_struct *work) { struct ttm_device *bdev = @@ -216,7 +201,7 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs, bdev->funcs = funcs; - ttm_init_sysman(bdev); + ttm_sys_man_init(bdev); ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32); bdev->vma_manager = vma_manager; diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c index 56b0efdba1a9..997c458f68a9 100644 --- a/drivers/gpu/drm/ttm/ttm_module.c +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -31,12 +31,47 @@ */ #include <linux/module.h> #include <linux/device.h> +#include <linux/pgtable.h> #include <linux/sched.h> #include <linux/debugfs.h> #include <drm/drm_sysfs.h> +#include <drm/ttm/ttm_caching.h> #include "ttm_module.h" +/** + * ttm_prot_from_caching - Modify the page protection according to the + * ttm cacing mode + * @caching: The ttm caching mode + * @tmp: The original page protection + * + * Return: The modified page protection + */ +pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp) +{ + /* Cached mappings need no adjustment */ + if (caching == ttm_cached) + return tmp; + +#if defined(__i386__) || defined(__x86_64__) + if (caching == ttm_write_combined) + tmp = pgprot_writecombine(tmp); + else if (boot_cpu_data.x86 > 3) + tmp = pgprot_noncached(tmp); +#endif +#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ + defined(__powerpc__) || defined(__mips__) + if (caching == ttm_write_combined) + tmp = pgprot_writecombine(tmp); + else + tmp = pgprot_noncached(tmp); +#endif +#if defined(__sparc__) + tmp = pgprot_noncached(tmp); +#endif + return tmp; +} + struct dentry *ttm_debugfs_root; static int __init ttm_init(void) diff --git a/drivers/gpu/drm/ttm/ttm_module.h b/drivers/gpu/drm/ttm/ttm_module.h index d7cac5d4b835..767fe22aed48 100644 --- a/drivers/gpu/drm/ttm/ttm_module.h +++ b/drivers/gpu/drm/ttm/ttm_module.h @@ -34,7 +34,10 @@ #define TTM_PFX "[TTM] " struct dentry; +struct ttm_device; extern struct dentry *ttm_debugfs_root; +void ttm_sys_man_init(struct ttm_device *bdev); + #endif /* _TTM_MODULE_H_ */ diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c index 707e5c152896..03395386e8a7 100644 --- a/drivers/gpu/drm/ttm/ttm_range_manager.c +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -29,12 +29,13 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_device.h> #include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_range_manager.h> +#include <drm/ttm/ttm_bo_api.h> #include <drm/drm_mm.h> #include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/module.h> /* * Currently we use a spinlock for the lock, but a mutex *may* be @@ -48,7 +49,8 @@ struct ttm_range_manager { spinlock_t lock; }; -static inline struct ttm_range_manager *to_range_manager(struct ttm_resource_manager *man) +static inline struct ttm_range_manager * +to_range_manager(struct ttm_resource_manager *man) { return container_of(man, struct ttm_range_manager, manager); } @@ -56,11 +58,11 @@ static inline struct ttm_range_manager *to_range_manager(struct ttm_resource_man static int ttm_range_man_alloc(struct ttm_resource_manager *man, struct ttm_buffer_object *bo, const struct ttm_place *place, - struct ttm_resource *mem) + struct ttm_resource **res) { struct ttm_range_manager *rman = to_range_manager(man); + struct ttm_range_mgr_node *node; struct drm_mm *mm = &rman->mm; - struct drm_mm_node *node; enum drm_mm_insert_mode mode; unsigned long lpfn; int ret; @@ -69,7 +71,7 @@ static int ttm_range_man_alloc(struct ttm_resource_manager *man, if (!lpfn) lpfn = man->size; - node = kzalloc(sizeof(*node), GFP_KERNEL); + node = kzalloc(struct_size(node, mm_nodes, 1), GFP_KERNEL); if (!node) return -ENOMEM; @@ -77,40 +79,65 @@ static int ttm_range_man_alloc(struct ttm_resource_manager *man, if (place->flags & TTM_PL_FLAG_TOPDOWN) mode = DRM_MM_INSERT_HIGH; + ttm_resource_init(bo, place, &node->base); + spin_lock(&rman->lock); - ret = drm_mm_insert_node_in_range(mm, node, - mem->num_pages, - mem->page_alignment, 0, + ret = drm_mm_insert_node_in_range(mm, &node->mm_nodes[0], + node->base.num_pages, + bo->page_alignment, 0, place->fpfn, lpfn, mode); spin_unlock(&rman->lock); if (unlikely(ret)) { kfree(node); - } else { - mem->mm_node = node; - mem->start = node->start; + return ret; } - return ret; + node->base.start = node->mm_nodes[0].start; + *res = &node->base; + return 0; } static void ttm_range_man_free(struct ttm_resource_manager *man, - struct ttm_resource *mem) + struct ttm_resource *res) { + struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res); struct ttm_range_manager *rman = to_range_manager(man); - if (mem->mm_node) { - spin_lock(&rman->lock); - drm_mm_remove_node(mem->mm_node); - spin_unlock(&rman->lock); + spin_lock(&rman->lock); + drm_mm_remove_node(&node->mm_nodes[0]); + spin_unlock(&rman->lock); - kfree(mem->mm_node); - mem->mm_node = NULL; - } + kfree(node); +} + +static void ttm_range_man_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + struct ttm_range_manager *rman = to_range_manager(man); + + spin_lock(&rman->lock); + drm_mm_print(&rman->mm, printer); + spin_unlock(&rman->lock); } -static const struct ttm_resource_manager_func ttm_range_manager_func; +static const struct ttm_resource_manager_func ttm_range_manager_func = { + .alloc = ttm_range_man_alloc, + .free = ttm_range_man_free, + .debug = ttm_range_man_debug +}; +/** + * ttm_range_man_init + * + * @bdev: ttm device + * @type: memory manager type + * @use_tt: if the memory manager uses tt + * @p_size: size of area to be managed in pages. + * + * Initialise a generic range manager for the selected memory type. + * The range manager is installed for this device in the type slot. + */ int ttm_range_man_init(struct ttm_device *bdev, unsigned type, bool use_tt, unsigned long p_size) @@ -138,6 +165,14 @@ int ttm_range_man_init(struct ttm_device *bdev, } EXPORT_SYMBOL(ttm_range_man_init); +/** + * ttm_range_man_fini + * + * @bdev: ttm device + * @type: memory manager type + * + * Remove the generic range manager from a slot and tear it down. + */ int ttm_range_man_fini(struct ttm_device *bdev, unsigned type) { @@ -163,19 +198,3 @@ int ttm_range_man_fini(struct ttm_device *bdev, return 0; } EXPORT_SYMBOL(ttm_range_man_fini); - -static void ttm_range_man_debug(struct ttm_resource_manager *man, - struct drm_printer *printer) -{ - struct ttm_range_manager *rman = to_range_manager(man); - - spin_lock(&rman->lock); - drm_mm_print(&rman->mm, printer); - spin_unlock(&rman->lock); -} - -static const struct ttm_resource_manager_func ttm_range_manager_func = { - .alloc = ttm_range_man_alloc, - .free = ttm_range_man_free, - .debug = ttm_range_man_debug -}; diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 04f2eef653ab..2431717376e7 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -22,33 +22,48 @@ * Authors: Christian König */ +#include <linux/dma-buf-map.h> +#include <linux/io-mapping.h> +#include <linux/scatterlist.h> + #include <drm/ttm/ttm_resource.h> #include <drm/ttm/ttm_bo_driver.h> +void ttm_resource_init(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource *res) +{ + res->start = 0; + res->num_pages = PFN_UP(bo->base.size); + res->mem_type = place->mem_type; + res->placement = place->flags; + res->bus.addr = NULL; + res->bus.offset = 0; + res->bus.is_iomem = false; + res->bus.caching = ttm_cached; +} +EXPORT_SYMBOL(ttm_resource_init); + int ttm_resource_alloc(struct ttm_buffer_object *bo, const struct ttm_place *place, - struct ttm_resource *res) + struct ttm_resource **res_ptr) { struct ttm_resource_manager *man = - ttm_manager_type(bo->bdev, res->mem_type); + ttm_manager_type(bo->bdev, place->mem_type); - res->mm_node = NULL; - if (!man->func || !man->func->alloc) - return 0; - - return man->func->alloc(man, bo, place, res); + return man->func->alloc(man, bo, place, res_ptr); } -void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource *res) +void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res) { - struct ttm_resource_manager *man = - ttm_manager_type(bo->bdev, res->mem_type); + struct ttm_resource_manager *man; - if (man->func && man->func->free) - man->func->free(man, res); + if (!*res) + return; - res->mm_node = NULL; - res->mem_type = TTM_PL_SYSTEM; + man = ttm_manager_type(bo->bdev, (*res)->mem_type); + man->func->free(man, *res); + *res = NULL; } EXPORT_SYMBOL(ttm_resource_free); @@ -139,7 +154,196 @@ void ttm_resource_manager_debug(struct ttm_resource_manager *man, drm_printf(p, " use_type: %d\n", man->use_type); drm_printf(p, " use_tt: %d\n", man->use_tt); drm_printf(p, " size: %llu\n", man->size); - if (man->func && man->func->debug) - (*man->func->debug)(man, p); + if (man->func->debug) + man->func->debug(man, p); } EXPORT_SYMBOL(ttm_resource_manager_debug); + +static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter, + struct dma_buf_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_iomap *iter_io = + container_of(iter, typeof(*iter_io), base); + void __iomem *addr; + +retry: + while (i >= iter_io->cache.end) { + iter_io->cache.sg = iter_io->cache.sg ? + sg_next(iter_io->cache.sg) : iter_io->st->sgl; + iter_io->cache.i = iter_io->cache.end; + iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >> + PAGE_SHIFT; + iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) - + iter_io->start; + } + + if (i < iter_io->cache.i) { + iter_io->cache.end = 0; + iter_io->cache.sg = NULL; + goto retry; + } + + addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs + + (((resource_size_t)i - iter_io->cache.i) + << PAGE_SHIFT)); + dma_buf_map_set_vaddr_iomem(dmap, addr); +} + +static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter, + struct dma_buf_map *map) +{ + io_mapping_unmap_local(map->vaddr_iomem); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = { + .map_local = ttm_kmap_iter_iomap_map_local, + .unmap_local = ttm_kmap_iter_iomap_unmap_local, + .maps_tt = false, +}; + +/** + * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap + * @iter_io: The struct ttm_kmap_iter_iomap to initialize. + * @iomap: The struct io_mapping representing the underlying linear io_memory. + * @st: sg_table into @iomap, representing the memory of the struct + * ttm_resource. + * @start: Offset that needs to be subtracted from @st to make + * sg_dma_address(st->sgl) - @start == 0 for @iomap start. + * + * Return: Pointer to the embedded struct ttm_kmap_iter. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, + struct io_mapping *iomap, + struct sg_table *st, + resource_size_t start) +{ + iter_io->base.ops = &ttm_kmap_iter_io_ops; + iter_io->iomap = iomap; + iter_io->st = st; + iter_io->start = start; + memset(&iter_io->cache, 0, sizeof(iter_io->cache)); + + return &iter_io->base; +} +EXPORT_SYMBOL(ttm_kmap_iter_iomap_init); + +/** + * DOC: Linear io iterator + * + * This code should die in the not too near future. Best would be if we could + * make io-mapping use memremap for all io memory, and have memremap + * implement a kmap_local functionality. We could then strip a huge amount of + * code. These linear io iterators are implemented to mimic old functionality, + * and they don't use kmap_local semantics at all internally. Rather ioremap or + * friends, and at least on 32-bit they add global TLB flushes and points + * of failure. + */ + +static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter, + struct dma_buf_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_linear_io *iter_io = + container_of(iter, typeof(*iter_io), base); + + *dmap = iter_io->dmap; + dma_buf_map_incr(dmap, i * PAGE_SIZE); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = { + .map_local = ttm_kmap_iter_linear_io_map_local, + .maps_tt = false, +}; + +/** + * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory + * @iter_io: The iterator to initialize + * @bdev: The TTM device + * @mem: The ttm resource representing the iomap. + * + * This function is for internal TTM use only. It sets up a memcpy kmap iterator + * pointing at a linear chunk of io memory. + * + * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on + * failure. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, + struct ttm_device *bdev, + struct ttm_resource *mem) +{ + int ret; + + ret = ttm_mem_io_reserve(bdev, mem); + if (ret) + goto out_err; + if (!mem->bus.is_iomem) { + ret = -EINVAL; + goto out_io_free; + } + + if (mem->bus.addr) { + dma_buf_map_set_vaddr(&iter_io->dmap, mem->bus.addr); + iter_io->needs_unmap = false; + } else { + size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; + + iter_io->needs_unmap = true; + memset(&iter_io->dmap, 0, sizeof(iter_io->dmap)); + if (mem->bus.caching == ttm_write_combined) + dma_buf_map_set_vaddr_iomem(&iter_io->dmap, + ioremap_wc(mem->bus.offset, + bus_size)); + else if (mem->bus.caching == ttm_cached) + dma_buf_map_set_vaddr(&iter_io->dmap, + memremap(mem->bus.offset, bus_size, + MEMREMAP_WB | + MEMREMAP_WT | + MEMREMAP_WC)); + + /* If uncached requested or if mapping cached or wc failed */ + if (dma_buf_map_is_null(&iter_io->dmap)) + dma_buf_map_set_vaddr_iomem(&iter_io->dmap, + ioremap(mem->bus.offset, + bus_size)); + + if (dma_buf_map_is_null(&iter_io->dmap)) { + ret = -ENOMEM; + goto out_io_free; + } + } + + iter_io->base.ops = &ttm_kmap_iter_linear_io_ops; + return &iter_io->base; + +out_io_free: + ttm_mem_io_free(bdev, mem); +out_err: + return ERR_PTR(ret); +} + +/** + * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory + * @iter_io: The iterator to initialize + * @bdev: The TTM device + * @mem: The ttm resource representing the iomap. + * + * This function is for internal TTM use only. It cleans up a memcpy kmap + * iterator initialized by ttm_kmap_iter_linear_io_init. + */ +void +ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, + struct ttm_device *bdev, + struct ttm_resource *mem) +{ + if (iter_io->needs_unmap && dma_buf_map_is_set(&iter_io->dmap)) { + if (iter_io->dmap.is_iomem) + iounmap(iter_io->dmap.vaddr_iomem); + else + memunmap(iter_io->dmap.vaddr); + } + + ttm_mem_io_free(bdev, mem); +} diff --git a/drivers/gpu/drm/ttm/ttm_sys_manager.c b/drivers/gpu/drm/ttm/ttm_sys_manager.c new file mode 100644 index 000000000000..63aca52f75e1 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_sys_manager.c @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ + +#include <drm/ttm/ttm_resource.h> +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_placement.h> +#include <linux/slab.h> + +#include "ttm_module.h" + +static int ttm_sys_man_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + *res = kzalloc(sizeof(**res), GFP_KERNEL); + if (!*res) + return -ENOMEM; + + ttm_resource_init(bo, place, *res); + return 0; +} + +static void ttm_sys_man_free(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + kfree(res); +} + +static const struct ttm_resource_manager_func ttm_sys_manager_func = { + .alloc = ttm_sys_man_alloc, + .free = ttm_sys_man_free, +}; + +void ttm_sys_man_init(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man = &bdev->sysman; + + /* + * Initialize the system memory buffer type. + * Other types need to be driver / IOCTL initialized. + */ + man->use_tt = true; + man->func = &ttm_sys_manager_func; + + ttm_resource_manager_init(man, 0); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man); + ttm_resource_manager_set_used(man, true); +} diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index a1a25410ec74..24031a8acd2d 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -400,16 +400,81 @@ void ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) ttm->page_flags &= ~TTM_PAGE_FLAG_PRIV_POPULATED; } -/** +#ifdef CONFIG_DEBUG_FS + +/* Test the shrinker functions and dump the result */ +static int ttm_tt_debugfs_shrink_show(struct seq_file *m, void *data) +{ + struct ttm_operation_ctx ctx = { false, false }; + + seq_printf(m, "%d\n", ttm_global_swapout(&ctx, GFP_KERNEL)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_tt_debugfs_shrink); + +#endif + + +/* * ttm_tt_mgr_init - register with the MM shrinker * * Register with the MM shrinker for swapping out BOs. */ void ttm_tt_mgr_init(unsigned long num_pages, unsigned long num_dma32_pages) { +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("tt_shrink", 0400, ttm_debugfs_root, NULL, + &ttm_tt_debugfs_shrink_fops); +#endif + if (!ttm_pages_limit) ttm_pages_limit = num_pages; if (!ttm_dma32_pages_limit) ttm_dma32_pages_limit = num_dma32_pages; } + +static void ttm_kmap_iter_tt_map_local(struct ttm_kmap_iter *iter, + struct dma_buf_map *dmap, + pgoff_t i) +{ + struct ttm_kmap_iter_tt *iter_tt = + container_of(iter, typeof(*iter_tt), base); + + dma_buf_map_set_vaddr(dmap, kmap_local_page_prot(iter_tt->tt->pages[i], + iter_tt->prot)); +} + +static void ttm_kmap_iter_tt_unmap_local(struct ttm_kmap_iter *iter, + struct dma_buf_map *map) +{ + kunmap_local(map->vaddr); +} + +static const struct ttm_kmap_iter_ops ttm_kmap_iter_tt_ops = { + .map_local = ttm_kmap_iter_tt_map_local, + .unmap_local = ttm_kmap_iter_tt_unmap_local, + .maps_tt = true, +}; + +/** + * ttm_kmap_iter_tt_init - Initialize a struct ttm_kmap_iter_tt + * @iter_tt: The struct ttm_kmap_iter_tt to initialize. + * @tt: Struct ttm_tt holding page pointers of the struct ttm_resource. + * + * Return: Pointer to the embedded struct ttm_kmap_iter. + */ +struct ttm_kmap_iter * +ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt, + struct ttm_tt *tt) +{ + iter_tt->base.ops = &ttm_kmap_iter_tt_ops; + iter_tt->tt = tt; + if (tt) + iter_tt->prot = ttm_prot_from_caching(tt->caching, PAGE_KERNEL); + else + iter_tt->prot = PAGE_KERNEL; + + return &iter_tt->base; +} +EXPORT_SYMBOL(ttm_kmap_iter_tt_init); |