diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/Makefile | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_bo_test.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c | 60 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_mock_manager.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_backup.c | 182 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 36 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_internal.h | 58 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 351 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_device.c | 24 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_pool.c | 749 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_range_manager.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_resource.c | 55 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 84 |
17 files changed, 1358 insertions, 263 deletions
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index dad298127226..40d07a35293a 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -4,7 +4,7 @@ ttm-y := ttm_tt.o ttm_bo.o ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ ttm_execbuf_util.o ttm_range_manager.o ttm_resource.o ttm_pool.o \ - ttm_device.o ttm_sys_manager.o + ttm_device.o ttm_sys_manager.o ttm_backup.o ttm-$(CONFIG_AGP) += ttm_agp_backend.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c index f8f20d2f6174..6c77550c51af 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c @@ -175,7 +175,7 @@ struct signal_timer { static void signal_for_ttm_bo_reserve(struct timer_list *t) { - struct signal_timer *s_timer = from_timer(s_timer, t, timer); + struct signal_timer *s_timer = timer_container_of(s_timer, t, timer); struct task_struct *task = s_timer->ctx->task; do_send_sig_info(SIGTERM, SEND_SIG_PRIV, task, PIDTYPE_PID); @@ -201,7 +201,7 @@ static int threaded_ttm_bo_reserve(void *arg) err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); timer_delete_sync(&s_timer.timer); - destroy_timer_on_stack(&s_timer.timer); + timer_destroy_on_stack(&s_timer.timer); ww_acquire_fini(&ctx); @@ -340,7 +340,7 @@ static void ttm_bo_unreserve_bulk(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, ttm_dev); resv = kunit_kzalloc(test, sizeof(*resv), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, ttm_dev); + KUNIT_ASSERT_NOT_NULL(test, resv); err = ttm_device_kunit_init(priv, ttm_dev, false, false); KUNIT_ASSERT_EQ(test, err, 0); diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c index 3148f5d3dbd6..1bcc67977f48 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c @@ -542,14 +542,15 @@ static void ttm_bo_validate_no_placement_signaled(struct kunit *test) bo->ttm = old_tt; } - err = ttm_resource_alloc(bo, place, &bo->resource, NULL); - KUNIT_EXPECT_EQ(test, err, 0); - KUNIT_ASSERT_EQ(test, man->usage, size); - placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, placement); ttm_bo_reserve(bo, false, false, NULL); + + err = ttm_resource_alloc(bo, place, &bo->resource, NULL); + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, man->usage, size); + err = ttm_bo_validate(bo, placement, &ctx); ttm_bo_unreserve(bo); @@ -757,56 +758,6 @@ static void ttm_bo_validate_move_fence_not_signaled(struct kunit *test) ttm_mock_manager_fini(priv->ttm_dev, snd_mem); } -static void ttm_bo_validate_swapout(struct kunit *test) -{ - unsigned long size_big, size = ALIGN(BO_SIZE, PAGE_SIZE); - enum ttm_bo_type bo_type = ttm_bo_type_device; - struct ttm_buffer_object *bo_small, *bo_big; - struct ttm_test_devices *priv = test->priv; - struct ttm_operation_ctx ctx = { }; - struct ttm_placement *placement; - u32 mem_type = TTM_PL_TT; - struct ttm_place *place; - struct sysinfo si; - int err; - - si_meminfo(&si); - size_big = ALIGN(((u64)si.totalram * si.mem_unit / 2), PAGE_SIZE); - - ttm_mock_manager_init(priv->ttm_dev, mem_type, size_big + size); - - place = ttm_place_kunit_init(test, mem_type, 0); - placement = ttm_placement_kunit_init(test, place, 1); - - bo_small = kunit_kzalloc(test, sizeof(*bo_small), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, bo_small); - - drm_gem_private_object_init(priv->drm, &bo_small->base, size); - - err = ttm_bo_init_reserved(priv->ttm_dev, bo_small, bo_type, placement, - PAGE_SIZE, &ctx, NULL, NULL, - &dummy_ttm_bo_destroy); - KUNIT_EXPECT_EQ(test, err, 0); - dma_resv_unlock(bo_small->base.resv); - - bo_big = ttm_bo_kunit_init(test, priv, size_big, NULL); - - dma_resv_lock(bo_big->base.resv, NULL); - err = ttm_bo_validate(bo_big, placement, &ctx); - dma_resv_unlock(bo_big->base.resv); - - KUNIT_EXPECT_EQ(test, err, 0); - KUNIT_EXPECT_NOT_NULL(test, bo_big->resource); - KUNIT_EXPECT_EQ(test, bo_big->resource->mem_type, mem_type); - KUNIT_EXPECT_EQ(test, bo_small->resource->mem_type, TTM_PL_SYSTEM); - KUNIT_EXPECT_TRUE(test, bo_small->ttm->page_flags & TTM_TT_FLAG_SWAPPED); - - ttm_bo_put(bo_big); - ttm_bo_put(bo_small); - - ttm_mock_manager_fini(priv->ttm_dev, mem_type); -} - static void ttm_bo_validate_happy_evict(struct kunit *test) { u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT, @@ -1201,7 +1152,6 @@ static struct kunit_case ttm_bo_validate_test_cases[] = { KUNIT_CASE(ttm_bo_validate_move_fence_signaled), KUNIT_CASE_PARAM(ttm_bo_validate_move_fence_not_signaled, ttm_bo_validate_wait_gen_params), - KUNIT_CASE(ttm_bo_validate_swapout), KUNIT_CASE(ttm_bo_validate_happy_evict), KUNIT_CASE(ttm_bo_validate_all_pinned_evict), KUNIT_CASE(ttm_bo_validate_allowed_only_evict), diff --git a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c index b91c13f46225..7aaf0d1395ff 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c +++ b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c @@ -2,6 +2,9 @@ /* * Copyright © 2023 Intel Corporation */ + +#include <linux/export.h> + #include <drm/ttm/ttm_tt.h> #include "ttm_kunit_helpers.h" diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c index f6d1c8a2845d..d7eb6471f2ed 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c @@ -2,6 +2,9 @@ /* * Copyright © 2023 Intel Corporation */ + +#include <linux/export.h> + #include <drm/ttm/ttm_resource.h> #include <drm/ttm/ttm_device.h> #include <drm/ttm/ttm_placement.h> diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index d27691f2e451..fca0a1a3c6fd 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -36,6 +36,7 @@ #include <drm/ttm/ttm_tt.h> #include <drm/ttm/ttm_resource.h> #include <linux/agp_backend.h> +#include <linux/export.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/io.h> diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c new file mode 100644 index 000000000000..32530c75f038 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2024 Intel Corporation + */ + +#include <drm/ttm/ttm_backup.h> + +#include <linux/export.h> +#include <linux/page-flags.h> +#include <linux/swap.h> + +/* + * Need to map shmem indices to handle since a handle value + * of 0 means error, following the swp_entry_t convention. + */ +static unsigned long ttm_backup_shmem_idx_to_handle(pgoff_t idx) +{ + return (unsigned long)idx + 1; +} + +static pgoff_t ttm_backup_handle_to_shmem_idx(pgoff_t handle) +{ + return handle - 1; +} + +/** + * ttm_backup_drop() - release memory associated with a handle + * @backup: The struct backup pointer used to obtain the handle + * @handle: The handle obtained from the @backup_page function. + */ +void ttm_backup_drop(struct file *backup, pgoff_t handle) +{ + loff_t start = ttm_backup_handle_to_shmem_idx(handle); + + start <<= PAGE_SHIFT; + shmem_truncate_range(file_inode(backup), start, + start + PAGE_SIZE - 1); +} + +/** + * ttm_backup_copy_page() - Copy the contents of a previously backed + * up page + * @backup: The struct backup pointer used to back up the page. + * @dst: The struct page to copy into. + * @handle: The handle returned when the page was backed up. + * @intr: Try to perform waits interruptible or at least killable. + * + * Return: 0 on success, Negative error code on failure, notably + * -EINTR if @intr was set to true and a signal is pending. + */ +int ttm_backup_copy_page(struct file *backup, struct page *dst, + pgoff_t handle, bool intr) +{ + struct address_space *mapping = backup->f_mapping; + struct folio *from_folio; + pgoff_t idx = ttm_backup_handle_to_shmem_idx(handle); + + from_folio = shmem_read_folio(mapping, idx); + if (IS_ERR(from_folio)) + return PTR_ERR(from_folio); + + copy_highpage(dst, folio_file_page(from_folio, idx)); + folio_put(from_folio); + + return 0; +} + +/** + * ttm_backup_backup_page() - Backup a page + * @backup: The struct backup pointer to use. + * @page: The page to back up. + * @writeback: Whether to perform immediate writeback of the page. + * This may have performance implications. + * @idx: A unique integer for each page and each struct backup. + * This allows the backup implementation to avoid managing + * its address space separately. + * @page_gfp: The gfp value used when the page was allocated. + * This is used for accounting purposes. + * @alloc_gfp: The gfp to be used when allocating memory. + * + * Context: If called from reclaim context, the caller needs to + * assert that the shrinker gfp has __GFP_FS set, to avoid + * deadlocking on lock_page(). If @writeback is set to true and + * called from reclaim context, the caller also needs to assert + * that the shrinker gfp has __GFP_IO set, since without it, + * we're not allowed to start backup IO. + * + * Return: A handle on success. Negative error code on failure. + * + * Note: This function could be extended to back up a folio and + * implementations would then split the folio internally if needed. + * Drawback is that the caller would then have to keep track of + * the folio size- and usage. + */ +s64 +ttm_backup_backup_page(struct file *backup, struct page *page, + bool writeback, pgoff_t idx, gfp_t page_gfp, + gfp_t alloc_gfp) +{ + struct address_space *mapping = backup->f_mapping; + unsigned long handle = 0; + struct folio *to_folio; + int ret; + + to_folio = shmem_read_folio_gfp(mapping, idx, alloc_gfp); + if (IS_ERR(to_folio)) + return PTR_ERR(to_folio); + + folio_mark_accessed(to_folio); + folio_lock(to_folio); + folio_mark_dirty(to_folio); + copy_highpage(folio_file_page(to_folio, idx), page); + handle = ttm_backup_shmem_idx_to_handle(idx); + + if (writeback && !folio_mapped(to_folio) && + folio_clear_dirty_for_io(to_folio)) { + folio_set_reclaim(to_folio); + ret = shmem_writeout(to_folio, NULL, NULL); + if (!folio_test_writeback(to_folio)) + folio_clear_reclaim(to_folio); + /* + * If writeout succeeds, it unlocks the folio. errors + * are otherwise dropped, since writeout is only best + * effort here. + */ + if (ret) + folio_unlock(to_folio); + } else { + folio_unlock(to_folio); + } + + folio_put(to_folio); + + return handle; +} + +/** + * ttm_backup_fini() - Free the struct backup resources after last use. + * @backup: Pointer to the struct backup whose resources to free. + * + * After a call to this function, it's illegal to use the @backup pointer. + */ +void ttm_backup_fini(struct file *backup) +{ + fput(backup); +} + +/** + * ttm_backup_bytes_avail() - Report the approximate number of bytes of backup space + * left for backup. + * + * This function is intended also for driver use to indicate whether a + * backup attempt is meaningful. + * + * Return: An approximate size of backup space available. + */ +u64 ttm_backup_bytes_avail(void) +{ + /* + * The idea behind backing up to shmem is that shmem objects may + * eventually be swapped out. So no point swapping out if there + * is no or low swap-space available. But the accuracy of this + * number also depends on shmem actually swapping out backed-up + * shmem objects without too much buffering. + */ + return (u64)get_nr_swap_pages() << PAGE_SHIFT; +} +EXPORT_SYMBOL_GPL(ttm_backup_bytes_avail); + +/** + * ttm_backup_shmem_create() - Create a shmem-based struct backup. + * @size: The maximum size (in bytes) to back up. + * + * Create a backup utilizing shmem objects. + * + * Return: A pointer to a struct file on success, + * an error pointer on error. + */ +struct file *ttm_backup_shmem_create(loff_t size) +{ + return shmem_file_setup("ttm shmem backup", size, 0); +} diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index ea5e49858857..f4d9e68b21e7 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -35,6 +35,7 @@ #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_tt.h> +#include <linux/export.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> @@ -46,6 +47,7 @@ #include <linux/dma-resv.h> #include "ttm_module.h" +#include "ttm_bo_internal.h" static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_placement *placement) @@ -235,7 +237,7 @@ static void ttm_bo_delayed_delete(struct work_struct *work) bo = container_of(work, typeof(*bo), delayed_delete); - dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false, + dma_resv_wait_timeout(&bo->base._resv, DMA_RESV_USAGE_BOOKKEEP, false, MAX_SCHEDULE_TIMEOUT); dma_resv_lock(bo->base.resv, NULL); ttm_bo_cleanup_memtype_use(bo); @@ -270,7 +272,7 @@ static void ttm_bo_release(struct kref *kref) drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); ttm_mem_io_free(bdev, bo->resource); - if (!dma_resv_test_signaled(bo->base.resv, + if (!dma_resv_test_signaled(&bo->base._resv, DMA_RESV_USAGE_BOOKKEEP) || (want_init_on_free() && (bo->ttm != NULL)) || bo->type == ttm_bo_type_sg || @@ -451,7 +453,8 @@ int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man int ret = 0; spin_lock(&bdev->lru_lock); - res = ttm_resource_manager_first(man, &cursor); + ttm_resource_cursor_init(&cursor, man); + res = ttm_resource_manager_first(&cursor); ttm_resource_cursor_fini(&cursor); if (!res) { ret = -ENOENT; @@ -523,11 +526,11 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object * return 0; if (bo->deleted) { - lret = ttm_bo_wait_ctx(bo, walk->ctx); + lret = ttm_bo_wait_ctx(bo, walk->arg.ctx); if (!lret) ttm_bo_cleanup_memtype_use(bo); } else { - lret = ttm_bo_evict(bo, walk->ctx); + lret = ttm_bo_evict(bo, walk->arg.ctx); } if (lret) @@ -563,8 +566,10 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, struct ttm_bo_evict_walk evict_walk = { .walk = { .ops = &ttm_evict_walk_ops, - .ctx = ctx, - .ticket = ticket, + .arg = { + .ctx = ctx, + .ticket = ticket, + } }, .place = place, .evictor = evictor, @@ -573,7 +578,7 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, }; s64 lret; - evict_walk.walk.trylock_only = true; + evict_walk.walk.arg.trylock_only = true; lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); /* One more attempt if we hit low limit? */ @@ -587,12 +592,12 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, /* Reset low limit */ evict_walk.try_low = evict_walk.hit_low = false; /* If ticket-locking, repeat while making progress. */ - evict_walk.walk.trylock_only = false; + evict_walk.walk.arg.trylock_only = false; retry: do { /* The walk may clear the evict_walk.walk.ticket field */ - evict_walk.walk.ticket = ticket; + evict_walk.walk.arg.ticket = ticket; evict_walk.evicted = 0; lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); } while (!lret && evict_walk.evicted); @@ -1092,7 +1097,8 @@ struct ttm_bo_swapout_walk { struct ttm_lru_walk walk; /** @gfp_flags: The gfp flags to use for ttm_tt_swapout() */ gfp_t gfp_flags; - + /** @hit_low: Whether we should attempt to swap BO's with low watermark threshold */ + /** @evict_low: If we cannot swap a bo when @try_low is false (first pass) */ bool hit_low, evict_low; }; @@ -1102,7 +1108,7 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) struct ttm_place place = {.mem_type = bo->resource->mem_type}; struct ttm_bo_swapout_walk *swapout_walk = container_of(walk, typeof(*swapout_walk), walk); - struct ttm_operation_ctx *ctx = walk->ctx; + struct ttm_operation_ctx *ctx = walk->arg.ctx; s64 ret; /* @@ -1213,8 +1219,10 @@ s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, struct ttm_bo_swapout_walk swapout_walk = { .walk = { .ops = &ttm_swap_ops, - .ctx = ctx, - .trylock_only = true, + .arg = { + .ctx = ctx, + .trylock_only = true, + }, }, .gfp_flags = gfp_flags, }; diff --git a/drivers/gpu/drm/ttm/ttm_bo_internal.h b/drivers/gpu/drm/ttm/ttm_bo_internal.h new file mode 100644 index 000000000000..9d8b747a34db --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_internal.h @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ + +#ifndef _TTM_BO_INTERNAL_H_ +#define _TTM_BO_INTERNAL_H_ + +#include <drm/ttm/ttm_bo.h> + +/** + * ttm_bo_get - reference a struct ttm_buffer_object + * + * @bo: The buffer object. + */ +static inline void ttm_bo_get(struct ttm_buffer_object *bo) +{ + kref_get(&bo->kref); +} + +/** + * ttm_bo_get_unless_zero - reference a struct ttm_buffer_object unless + * its refcount has already reached zero. + * @bo: The buffer object. + * + * Used to reference a TTM buffer object in lookups where the object is removed + * from the lookup structure during the destructor and for RCU lookups. + * + * Returns: @bo if the referencing was successful, NULL otherwise. + */ +static inline __must_check struct ttm_buffer_object * +ttm_bo_get_unless_zero(struct ttm_buffer_object *bo) +{ + if (!kref_get_unless_zero(&bo->kref)) + return NULL; + return bo; +} + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index d939925efa81..acbbca9d5c92 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -29,6 +29,8 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ +#include <linux/export.h> +#include <linux/swap.h> #include <linux/vmalloc.h> #include <drm/ttm/ttm_bo.h> @@ -37,6 +39,8 @@ #include <drm/drm_cache.h> +#include "ttm_bo_internal.h" + struct ttm_transfer_obj { struct ttm_buffer_object base; struct ttm_buffer_object *bo; @@ -254,6 +258,13 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, ret = dma_resv_trylock(&fbo->base.base._resv); WARN_ON(!ret); + ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1); + if (ret) { + dma_resv_unlock(&fbo->base.base._resv); + kfree(fbo); + return ret; + } + if (fbo->base.resource) { ttm_resource_set_bo(fbo->base.resource, &fbo->base); bo->resource = NULL; @@ -262,12 +273,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->base.bulk_move = NULL; } - ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1); - if (ret) { - kfree(fbo); - return ret; - } - ttm_bo_get(bo); fbo->bo = bo; @@ -378,6 +383,32 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, } /** + * ttm_bo_kmap_try_from_panic + * + * @bo: The buffer object + * @page: The page to map + * + * Sets up a kernel virtual mapping using kmap_local_page_try_from_panic(). + * This should only be called from the panic handler, if you make sure the bo + * is the one being displayed, so is properly allocated, and protected. + * + * Returns the vaddr, that you can use to write to the bo, and that you should + * pass to kunmap_local() when you're done with this page, or NULL if the bo + * is in iomem. + */ +void *ttm_bo_kmap_try_from_panic(struct ttm_buffer_object *bo, unsigned long page) +{ + if (page + 1 > PFN_UP(bo->resource->size)) + return NULL; + + if (!bo->resource->bus.is_iomem && bo->ttm->pages && bo->ttm->pages[page]) + return kmap_local_page_try_from_panic(bo->ttm->pages[page]); + + return NULL; +} +EXPORT_SYMBOL(ttm_bo_kmap_try_from_panic); + +/** * ttm_bo_kmap * * @bo: The buffer object. @@ -769,16 +800,15 @@ error_destroy_tt: return ret; } -static bool ttm_lru_walk_trylock(struct ttm_lru_walk *walk, - struct ttm_buffer_object *bo, - bool *needs_unlock) +static bool ttm_lru_walk_trylock(struct ttm_bo_lru_cursor *curs, + struct ttm_buffer_object *bo) { - struct ttm_operation_ctx *ctx = walk->ctx; + struct ttm_operation_ctx *ctx = curs->arg->ctx; - *needs_unlock = false; + curs->needs_unlock = false; if (dma_resv_trylock(bo->base.resv)) { - *needs_unlock = true; + curs->needs_unlock = true; return true; } @@ -790,27 +820,27 @@ static bool ttm_lru_walk_trylock(struct ttm_lru_walk *walk, return false; } -static int ttm_lru_walk_ticketlock(struct ttm_lru_walk *walk, - struct ttm_buffer_object *bo, - bool *needs_unlock) +static int ttm_lru_walk_ticketlock(struct ttm_bo_lru_cursor *curs, + struct ttm_buffer_object *bo) { + struct ttm_lru_walk_arg *arg = curs->arg; struct dma_resv *resv = bo->base.resv; int ret; - if (walk->ctx->interruptible) - ret = dma_resv_lock_interruptible(resv, walk->ticket); + if (arg->ctx->interruptible) + ret = dma_resv_lock_interruptible(resv, arg->ticket); else - ret = dma_resv_lock(resv, walk->ticket); + ret = dma_resv_lock(resv, arg->ticket); if (!ret) { - *needs_unlock = true; + curs->needs_unlock = true; /* * Only a single ticketlock per loop. Ticketlocks are prone * to return -EDEADLK causing the eviction to fail, so * after waiting for the ticketlock, revert back to * trylocking for this walk. */ - walk->ticket = NULL; + arg->ticket = NULL; } else if (ret == -EDEADLK) { /* Caller needs to exit the ww transaction. */ ret = -ENOSPC; @@ -819,12 +849,6 @@ static int ttm_lru_walk_ticketlock(struct ttm_lru_walk *walk, return ret; } -static void ttm_lru_walk_unlock(struct ttm_buffer_object *bo, bool locked) -{ - if (locked) - dma_resv_unlock(bo->base.resv); -} - /** * ttm_lru_walk_for_evict() - Perform a LRU list walk, with actions taken on * valid items. @@ -859,40 +883,118 @@ static void ttm_lru_walk_unlock(struct ttm_buffer_object *bo, bool locked) s64 ttm_lru_walk_for_evict(struct ttm_lru_walk *walk, struct ttm_device *bdev, struct ttm_resource_manager *man, s64 target) { - struct ttm_resource_cursor cursor; - struct ttm_resource *res; + struct ttm_bo_lru_cursor cursor; + struct ttm_buffer_object *bo; s64 progress = 0; s64 lret; - spin_lock(&bdev->lru_lock); - ttm_resource_manager_for_each_res(man, &cursor, res) { - struct ttm_buffer_object *bo = res->bo; - bool bo_needs_unlock = false; + ttm_bo_lru_for_each_reserved_guarded(&cursor, man, &walk->arg, bo) { + lret = walk->ops->process_bo(walk, bo); + if (lret == -EBUSY || lret == -EALREADY) + lret = 0; + progress = (lret < 0) ? lret : progress + lret; + if (progress < 0 || progress >= target) + break; + } + if (IS_ERR(bo)) + return PTR_ERR(bo); + + return progress; +} +EXPORT_SYMBOL(ttm_lru_walk_for_evict); + +static void ttm_bo_lru_cursor_cleanup_bo(struct ttm_bo_lru_cursor *curs) +{ + struct ttm_buffer_object *bo = curs->bo; + + if (bo) { + if (curs->needs_unlock) + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); + curs->bo = NULL; + } +} + +/** + * ttm_bo_lru_cursor_fini() - Stop using a struct ttm_bo_lru_cursor + * and clean up any iteration it was used for. + * @curs: The cursor. + */ +void ttm_bo_lru_cursor_fini(struct ttm_bo_lru_cursor *curs) +{ + spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; + + ttm_bo_lru_cursor_cleanup_bo(curs); + spin_lock(lru_lock); + ttm_resource_cursor_fini(&curs->res_curs); + spin_unlock(lru_lock); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_fini); + +/** + * ttm_bo_lru_cursor_init() - Initialize a struct ttm_bo_lru_cursor + * @curs: The ttm_bo_lru_cursor to initialize. + * @man: The ttm resource_manager whose LRU lists to iterate over. + * @arg: The ttm_lru_walk_arg to govern the walk. + * + * Initialize a struct ttm_bo_lru_cursor. + * + * Return: Pointer to @curs. The function does not fail. + */ +struct ttm_bo_lru_cursor * +ttm_bo_lru_cursor_init(struct ttm_bo_lru_cursor *curs, + struct ttm_resource_manager *man, + struct ttm_lru_walk_arg *arg) +{ + memset(curs, 0, sizeof(*curs)); + ttm_resource_cursor_init(&curs->res_curs, man); + curs->arg = arg; + + return curs; +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_init); + +static struct ttm_buffer_object * +__ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) +{ + spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; + struct ttm_resource *res = NULL; + struct ttm_buffer_object *bo; + struct ttm_lru_walk_arg *arg = curs->arg; + bool first = !curs->bo; + + ttm_bo_lru_cursor_cleanup_bo(curs); + + spin_lock(lru_lock); + for (;;) { + int mem_type, ret = 0; bool bo_locked = false; - int mem_type; - /* - * Attempt a trylock before taking a reference on the bo, - * since if we do it the other way around, and the trylock fails, - * we need to drop the lru lock to put the bo. - */ - if (ttm_lru_walk_trylock(walk, bo, &bo_needs_unlock)) + if (first) { + res = ttm_resource_manager_first(&curs->res_curs); + first = false; + } else { + res = ttm_resource_manager_next(&curs->res_curs); + } + if (!res) + break; + + bo = res->bo; + if (ttm_lru_walk_trylock(curs, bo)) bo_locked = true; - else if (!walk->ticket || walk->ctx->no_wait_gpu || - walk->trylock_only) + else if (!arg->ticket || arg->ctx->no_wait_gpu || arg->trylock_only) continue; if (!ttm_bo_get_unless_zero(bo)) { - ttm_lru_walk_unlock(bo, bo_needs_unlock); + if (curs->needs_unlock) + dma_resv_unlock(bo->base.resv); continue; } mem_type = res->mem_type; - spin_unlock(&bdev->lru_lock); - - lret = 0; + spin_unlock(lru_lock); if (!bo_locked) - lret = ttm_lru_walk_ticketlock(walk, bo, &bo_needs_unlock); + ret = ttm_lru_walk_ticketlock(curs, bo); /* * Note that in between the release of the lru lock and the @@ -901,21 +1003,152 @@ s64 ttm_lru_walk_for_evict(struct ttm_lru_walk *walk, struct ttm_device *bdev, * freed and allocated again with a different memory type. * In that case, just skip it. */ - if (!lret && bo->resource && bo->resource->mem_type == mem_type) - lret = walk->ops->process_bo(walk, bo); + curs->bo = bo; + if (!ret && bo->resource && bo->resource->mem_type == mem_type) + return bo; - ttm_lru_walk_unlock(bo, bo_needs_unlock); - ttm_bo_put(bo); - if (lret == -EBUSY || lret == -EALREADY) - lret = 0; - progress = (lret < 0) ? lret : progress + lret; + ttm_bo_lru_cursor_cleanup_bo(curs); + if (ret && ret != -EALREADY) + return ERR_PTR(ret); - spin_lock(&bdev->lru_lock); - if (progress < 0 || progress >= target) - break; + spin_lock(lru_lock); } - ttm_resource_cursor_fini(&cursor); - spin_unlock(&bdev->lru_lock); - return progress; + spin_unlock(lru_lock); + return res ? bo : NULL; +} + +/** + * ttm_bo_lru_cursor_next() - Continue iterating a manager's LRU lists + * to find and lock buffer object. + * @curs: The cursor initialized using ttm_bo_lru_cursor_init() and + * ttm_bo_lru_cursor_first(). + * + * Return: A pointer to a locked and reference-counted buffer object, + * or NULL if none could be found and looping should be terminated. + */ +struct ttm_buffer_object *ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) +{ + return __ttm_bo_lru_cursor_next(curs); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_next); + +/** + * ttm_bo_lru_cursor_first() - Start iterating a manager's LRU lists + * to find and lock buffer object. + * @curs: The cursor initialized using ttm_bo_lru_cursor_init(). + * + * Return: A pointer to a locked and reference-counted buffer object, + * or NULL if none could be found and looping should be terminated. + */ +struct ttm_buffer_object *ttm_bo_lru_cursor_first(struct ttm_bo_lru_cursor *curs) +{ + ttm_bo_lru_cursor_cleanup_bo(curs); + return __ttm_bo_lru_cursor_next(curs); +} +EXPORT_SYMBOL(ttm_bo_lru_cursor_first); + +/** + * ttm_bo_shrink() - Helper to shrink a ttm buffer object. + * @ctx: The struct ttm_operation_ctx used for the shrinking operation. + * @bo: The buffer object. + * @flags: Flags governing the shrinking behaviour. + * + * The function uses the ttm_tt_back_up functionality to back up or + * purge a struct ttm_tt. If the bo is not in system, it's first + * moved there. + * + * Return: The number of pages shrunken or purged, or + * negative error code on failure. + */ +long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, + const struct ttm_bo_shrink_flags flags) +{ + static const struct ttm_place sys_placement_flags = { + .fpfn = 0, + .lpfn = 0, + .mem_type = TTM_PL_SYSTEM, + .flags = 0, + }; + static struct ttm_placement sys_placement = { + .num_placement = 1, + .placement = &sys_placement_flags, + }; + struct ttm_tt *tt = bo->ttm; + long lret; + + dma_resv_assert_held(bo->base.resv); + + if (flags.allow_move && bo->resource->mem_type != TTM_PL_SYSTEM) { + int ret = ttm_bo_validate(bo, &sys_placement, ctx); + + /* Consider -ENOMEM and -ENOSPC non-fatal. */ + if (ret) { + if (ret == -ENOMEM || ret == -ENOSPC) + ret = -EBUSY; + return ret; + } + } + + ttm_bo_unmap_virtual(bo); + lret = ttm_bo_wait_ctx(bo, ctx); + if (lret < 0) + return lret; + + if (bo->bulk_move) { + spin_lock(&bo->bdev->lru_lock); + ttm_resource_del_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); + } + + lret = ttm_tt_backup(bo->bdev, tt, (struct ttm_backup_flags) + {.purge = flags.purge, + .writeback = flags.writeback}); + + if (lret <= 0 && bo->bulk_move) { + spin_lock(&bo->bdev->lru_lock); + ttm_resource_add_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); + } + + if (lret < 0 && lret != -EINTR) + return -EBUSY; + + return lret; +} +EXPORT_SYMBOL(ttm_bo_shrink); + +/** + * ttm_bo_shrink_suitable() - Whether a bo is suitable for shinking + * @ctx: The struct ttm_operation_ctx governing the shrinking. + * @bo: The candidate for shrinking. + * + * Check whether the object, given the information available to TTM, + * is suitable for shinking, This function can and should be used + * before attempting to shrink an object. + * + * Return: true if suitable. false if not. + */ +bool ttm_bo_shrink_suitable(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) +{ + return bo->ttm && ttm_tt_is_populated(bo->ttm) && !bo->pin_count && + (!ctx->no_wait_gpu || + dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)); +} +EXPORT_SYMBOL(ttm_bo_shrink_suitable); + +/** + * ttm_bo_shrink_avoid_wait() - Whether to avoid waiting for GPU + * during shrinking + * + * In some situations, like direct reclaim, waiting (in particular gpu waiting) + * should be avoided since it may stall a system that could otherwise make progress + * shrinking something else less time consuming. + * + * Return: true if gpu waiting should be avoided, false if not. + */ +bool ttm_bo_shrink_avoid_wait(void) +{ + return !current_is_kswapd(); } +EXPORT_SYMBOL(ttm_bo_shrink_avoid_wait); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index a194db83421d..b47020fca199 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -31,6 +31,8 @@ #define pr_fmt(fmt) "[TTM] " fmt +#include <linux/export.h> + #include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_tt.h> @@ -220,7 +222,6 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, struct ttm_operation_ctx ctx = { .interruptible = true, .no_wait_gpu = false, - .force_alloc = true }; ttm = bo->ttm; diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c index 02e797fd1891..c3e2fcbdd2cc 100644 --- a/drivers/gpu/drm/ttm/ttm_device.c +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -28,6 +28,7 @@ #define pr_fmt(fmt) "[TTM DEVICE] " fmt #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/mm.h> #include <drm/ttm/ttm_bo.h> @@ -36,6 +37,7 @@ #include <drm/ttm/ttm_placement.h> #include "ttm_module.h" +#include "ttm_bo_internal.h" /* * ttm_global_mutex - protecting the global state @@ -123,6 +125,28 @@ out: return ret; } +/** + * ttm_device_prepare_hibernation - move GTT BOs to shmem for hibernation. + * + * @bdev: A pointer to a struct ttm_device to prepare hibernation for. + * + * Return: 0 on success, negative number on failure. + */ +int ttm_device_prepare_hibernation(struct ttm_device *bdev) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false, + }; + int ret; + + do { + ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL); + } while (ret > 0); + return ret; +} +EXPORT_SYMBOL(ttm_device_prepare_hibernation); + /* * A buffer object shrink method that tries to swap out the first * buffer object on the global::swap_lru list. diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index f1c60fa80c2d..bc7a83a9fe44 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -26,6 +26,8 @@ * **************************************************************************/ +#include <linux/export.h> + #include <drm/ttm/ttm_execbuf_util.h> #include <drm/ttm/ttm_bo.h> diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index 8504dbe19c1a..baf27c70a419 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -31,6 +31,7 @@ * cause they are rather slow compared to alloc_pages+map. */ +#include <linux/export.h> #include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/debugfs.h> @@ -41,12 +42,20 @@ #include <asm/set_memory.h> #endif +#include <drm/ttm/ttm_backup.h> #include <drm/ttm/ttm_pool.h> #include <drm/ttm/ttm_tt.h> #include <drm/ttm/ttm_bo.h> #include "ttm_module.h" +#ifdef CONFIG_FAULT_INJECTION +#include <linux/fault-inject.h> +static DECLARE_FAULT_ATTR(backup_fault_inject); +#else +#define should_fail(...) false +#endif + /** * struct ttm_pool_dma - Helper object for coherent DMA mappings * @@ -58,6 +67,52 @@ struct ttm_pool_dma { unsigned long vaddr; }; +/** + * struct ttm_pool_alloc_state - Current state of the tt page allocation process + * @pages: Pointer to the next tt page pointer to populate. + * @caching_divide: Pointer to the first page pointer whose page has a staged but + * not committed caching transition from write-back to @tt_caching. + * @dma_addr: Pointer to the next tt dma_address entry to populate if any. + * @remaining_pages: Remaining pages to populate. + * @tt_caching: The requested cpu-caching for the pages allocated. + */ +struct ttm_pool_alloc_state { + struct page **pages; + struct page **caching_divide; + dma_addr_t *dma_addr; + pgoff_t remaining_pages; + enum ttm_caching tt_caching; +}; + +/** + * struct ttm_pool_tt_restore - State representing restore from backup + * @pool: The pool used for page allocation while restoring. + * @snapshot_alloc: A snapshot of the most recent struct ttm_pool_alloc_state. + * @alloced_page: Pointer to the page most recently allocated from a pool or system. + * @first_dma: The dma address corresponding to @alloced_page if dma_mapping + * is requested. + * @alloced_pages: The number of allocated pages present in the struct ttm_tt + * page vector from this restore session. + * @restored_pages: The number of 4K pages restored for @alloced_page (which + * is typically a multi-order page). + * @page_caching: The struct ttm_tt requested caching + * @order: The order of @alloced_page. + * + * Recovery from backup might fail when we've recovered less than the + * full ttm_tt. In order not to loose any data (yet), keep information + * around that allows us to restart a failed ttm backup recovery. + */ +struct ttm_pool_tt_restore { + struct ttm_pool *pool; + struct ttm_pool_alloc_state snapshot_alloc; + struct page *alloced_page; + dma_addr_t first_dma; + pgoff_t alloced_pages; + pgoff_t restored_pages; + enum ttm_caching page_caching; + unsigned int order; +}; + static unsigned long page_pool_size; MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool"); @@ -160,34 +215,33 @@ static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching, kfree(dma); } -/* Apply a new caching to an array of pages */ -static int ttm_pool_apply_caching(struct page **first, struct page **last, - enum ttm_caching caching) +/* Apply any cpu-caching deferred during page allocation */ +static int ttm_pool_apply_caching(struct ttm_pool_alloc_state *alloc) { #ifdef CONFIG_X86 - unsigned int num_pages = last - first; + unsigned int num_pages = alloc->pages - alloc->caching_divide; if (!num_pages) return 0; - switch (caching) { + switch (alloc->tt_caching) { case ttm_cached: break; case ttm_write_combined: - return set_pages_array_wc(first, num_pages); + return set_pages_array_wc(alloc->caching_divide, num_pages); case ttm_uncached: - return set_pages_array_uc(first, num_pages); + return set_pages_array_uc(alloc->caching_divide, num_pages); } #endif + alloc->caching_divide = alloc->pages; return 0; } -/* Map pages of 1 << order size and fill the DMA address array */ +/* DMA Map pages of 1 << order size and return the resulting dma_address. */ static int ttm_pool_map(struct ttm_pool *pool, unsigned int order, - struct page *p, dma_addr_t **dma_addr) + struct page *p, dma_addr_t *dma_addr) { dma_addr_t addr; - unsigned int i; if (pool->use_dma_alloc) { struct ttm_pool_dma *dma = (void *)p->private; @@ -201,10 +255,7 @@ static int ttm_pool_map(struct ttm_pool *pool, unsigned int order, return -EFAULT; } - for (i = 1 << order; i ; --i) { - *(*dma_addr)++ = addr; - addr += PAGE_SIZE; - } + *dma_addr = addr; return 0; } @@ -354,24 +405,235 @@ static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p) return p->private; } -/* Called when we got a page, either from a pool or newly allocated */ +/* + * Split larger pages so that we can free each PAGE_SIZE page as soon + * as it has been backed up, in order to avoid memory pressure during + * reclaim. + */ +static void ttm_pool_split_for_swap(struct ttm_pool *pool, struct page *p) +{ + unsigned int order = ttm_pool_page_order(pool, p); + pgoff_t nr; + + if (!order) + return; + + split_page(p, order); + nr = 1UL << order; + while (nr--) + (p++)->private = 0; +} + +/** + * DOC: Partial backup and restoration of a struct ttm_tt. + * + * Swapout using ttm_backup_backup_page() and swapin using + * ttm_backup_copy_page() may fail. + * The former most likely due to lack of swap-space or memory, the latter due + * to lack of memory or because of signal interruption during waits. + * + * Backup failure is easily handled by using a ttm_tt pages vector that holds + * both backup handles and page pointers. This has to be taken into account when + * restoring such a ttm_tt from backup, and when freeing it while backed up. + * When restoring, for simplicity, new pages are actually allocated from the + * pool and the contents of any old pages are copied in and then the old pages + * are released. + * + * For restoration failures, the struct ttm_pool_tt_restore holds sufficient state + * to be able to resume an interrupted restore, and that structure is freed once + * the restoration is complete. If the struct ttm_tt is destroyed while there + * is a valid struct ttm_pool_tt_restore attached, that is also properly taken + * care of. + */ + +/* Is restore ongoing for the currently allocated page? */ +static bool ttm_pool_restore_valid(const struct ttm_pool_tt_restore *restore) +{ + return restore && restore->restored_pages < (1 << restore->order); +} + +/* DMA unmap and free a multi-order page, either to the relevant pool or to system. */ +static pgoff_t ttm_pool_unmap_and_free(struct ttm_pool *pool, struct page *page, + const dma_addr_t *dma_addr, enum ttm_caching caching) +{ + struct ttm_pool_type *pt = NULL; + unsigned int order; + pgoff_t nr; + + if (pool) { + order = ttm_pool_page_order(pool, page); + nr = (1UL << order); + if (dma_addr) + ttm_pool_unmap(pool, *dma_addr, nr); + + pt = ttm_pool_select_type(pool, caching, order); + } else { + order = page->private; + nr = (1UL << order); + } + + if (pt) + ttm_pool_type_give(pt, page); + else + ttm_pool_free_page(pool, caching, order, page); + + return nr; +} + +/* Populate the page-array using the most recent allocated multi-order page. */ +static void ttm_pool_allocated_page_commit(struct page *allocated, + dma_addr_t first_dma, + struct ttm_pool_alloc_state *alloc, + pgoff_t nr) +{ + pgoff_t i; + + for (i = 0; i < nr; ++i) + *alloc->pages++ = allocated++; + + alloc->remaining_pages -= nr; + + if (!alloc->dma_addr) + return; + + for (i = 0; i < nr; ++i) { + *alloc->dma_addr++ = first_dma; + first_dma += PAGE_SIZE; + } +} + +/* + * When restoring, restore backed-up content to the newly allocated page and + * if successful, populate the page-table and dma-address arrays. + */ +static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore, + struct file *backup, + const struct ttm_operation_ctx *ctx, + struct ttm_pool_alloc_state *alloc) + +{ + pgoff_t i, nr = 1UL << restore->order; + struct page **first_page = alloc->pages; + struct page *p; + int ret = 0; + + for (i = restore->restored_pages; i < nr; ++i) { + p = first_page[i]; + if (ttm_backup_page_ptr_is_handle(p)) { + unsigned long handle = ttm_backup_page_ptr_to_handle(p); + + if (IS_ENABLED(CONFIG_FAULT_INJECTION) && ctx->interruptible && + should_fail(&backup_fault_inject, 1)) { + ret = -EINTR; + break; + } + + if (handle == 0) { + restore->restored_pages++; + continue; + } + + ret = ttm_backup_copy_page(backup, restore->alloced_page + i, + handle, ctx->interruptible); + if (ret) + break; + + ttm_backup_drop(backup, handle); + } else if (p) { + /* + * We could probably avoid splitting the old page + * using clever logic, but ATM we don't care, as + * we prioritize releasing memory ASAP. Note that + * here, the old retained page is always write-back + * cached. + */ + ttm_pool_split_for_swap(restore->pool, p); + copy_highpage(restore->alloced_page + i, p); + __free_pages(p, 0); + } + + restore->restored_pages++; + first_page[i] = ttm_backup_handle_to_page_ptr(0); + } + + if (ret) { + if (!restore->restored_pages) { + dma_addr_t *dma_addr = alloc->dma_addr ? &restore->first_dma : NULL; + + ttm_pool_unmap_and_free(restore->pool, restore->alloced_page, + dma_addr, restore->page_caching); + restore->restored_pages = nr; + } + return ret; + } + + ttm_pool_allocated_page_commit(restore->alloced_page, restore->first_dma, + alloc, nr); + if (restore->page_caching == alloc->tt_caching || PageHighMem(restore->alloced_page)) + alloc->caching_divide = alloc->pages; + restore->snapshot_alloc = *alloc; + restore->alloced_pages += nr; + + return 0; +} + +/* If restoring, save information needed for ttm_pool_restore_commit(). */ +static void +ttm_pool_page_allocated_restore(struct ttm_pool *pool, unsigned int order, + struct page *p, + enum ttm_caching page_caching, + dma_addr_t first_dma, + struct ttm_pool_tt_restore *restore, + const struct ttm_pool_alloc_state *alloc) +{ + restore->pool = pool; + restore->order = order; + restore->restored_pages = 0; + restore->page_caching = page_caching; + restore->first_dma = first_dma; + restore->alloced_page = p; + restore->snapshot_alloc = *alloc; +} + +/* + * Called when we got a page, either from a pool or newly allocated. + * if needed, dma map the page and populate the dma address array. + * Populate the page address array. + * If the caching is consistent, update any deferred caching. Otherwise + * stage this page for an upcoming deferred caching update. + */ static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order, - struct page *p, dma_addr_t **dma_addr, - unsigned long *num_pages, - struct page ***pages) + struct page *p, enum ttm_caching page_caching, + struct ttm_pool_alloc_state *alloc, + struct ttm_pool_tt_restore *restore) { - unsigned int i; - int r; + bool caching_consistent; + dma_addr_t first_dma; + int r = 0; - if (*dma_addr) { - r = ttm_pool_map(pool, order, p, dma_addr); + caching_consistent = (page_caching == alloc->tt_caching) || PageHighMem(p); + + if (caching_consistent) { + r = ttm_pool_apply_caching(alloc); + if (r) + return r; + } + + if (alloc->dma_addr) { + r = ttm_pool_map(pool, order, p, &first_dma); if (r) return r; } - *num_pages -= 1 << order; - for (i = 1 << order; i; --i, ++(*pages), ++p) - **pages = p; + if (restore) { + ttm_pool_page_allocated_restore(pool, order, p, page_caching, + first_dma, restore, alloc); + } else { + ttm_pool_allocated_page_commit(p, first_dma, alloc, 1UL << order); + + if (caching_consistent) + alloc->caching_divide = alloc->pages; + } return 0; } @@ -394,53 +656,62 @@ static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt, pgoff_t start_page, pgoff_t end_page) { struct page **pages = &tt->pages[start_page]; - unsigned int order; + struct file *backup = tt->backup; pgoff_t i, nr; for (i = start_page; i < end_page; i += nr, pages += nr) { - struct ttm_pool_type *pt = NULL; + struct page *p = *pages; - order = ttm_pool_page_order(pool, *pages); - nr = (1UL << order); - if (tt->dma_address) - ttm_pool_unmap(pool, tt->dma_address[i], nr); + nr = 1; + if (ttm_backup_page_ptr_is_handle(p)) { + unsigned long handle = ttm_backup_page_ptr_to_handle(p); - pt = ttm_pool_select_type(pool, caching, order); - if (pt) - ttm_pool_type_give(pt, *pages); - else - ttm_pool_free_page(pool, caching, order, *pages); + if (handle != 0) + ttm_backup_drop(backup, handle); + } else if (p) { + dma_addr_t *dma_addr = tt->dma_address ? + tt->dma_address + i : NULL; + + nr = ttm_pool_unmap_and_free(pool, p, dma_addr, caching); + } } } -/** - * ttm_pool_alloc - Fill a ttm_tt object - * - * @pool: ttm_pool to use - * @tt: ttm_tt object to fill - * @ctx: operation context - * - * Fill the ttm_tt object with pages and also make sure to DMA map them when - * necessary. - * - * Returns: 0 on successe, negative error code otherwise. +static void ttm_pool_alloc_state_init(const struct ttm_tt *tt, + struct ttm_pool_alloc_state *alloc) +{ + alloc->pages = tt->pages; + alloc->caching_divide = tt->pages; + alloc->dma_addr = tt->dma_address; + alloc->remaining_pages = tt->num_pages; + alloc->tt_caching = tt->caching; +} + +/* + * Find a suitable allocation order based on highest desired order + * and number of remaining pages */ -int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, - struct ttm_operation_ctx *ctx) +static unsigned int ttm_pool_alloc_find_order(unsigned int highest, + const struct ttm_pool_alloc_state *alloc) +{ + return min_t(unsigned int, highest, __fls(alloc->remaining_pages)); +} + +static int __ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx, + struct ttm_pool_alloc_state *alloc, + struct ttm_pool_tt_restore *restore) { - pgoff_t num_pages = tt->num_pages; - dma_addr_t *dma_addr = tt->dma_address; - struct page **caching = tt->pages; - struct page **pages = tt->pages; enum ttm_caching page_caching; gfp_t gfp_flags = GFP_USER; pgoff_t caching_divide; unsigned int order; + bool allow_pools; struct page *p; int r; - WARN_ON(!num_pages || ttm_tt_is_populated(tt)); - WARN_ON(dma_addr && !pool->dev); + WARN_ON(!alloc->remaining_pages || ttm_tt_is_populated(tt)); + WARN_ON(alloc->dma_addr && !pool->dev); if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC) gfp_flags |= __GFP_ZERO; @@ -453,86 +724,155 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, else gfp_flags |= GFP_HIGHUSER; - for (order = min_t(unsigned int, MAX_PAGE_ORDER, __fls(num_pages)); - num_pages; - order = min_t(unsigned int, order, __fls(num_pages))) { + page_caching = tt->caching; + allow_pools = true; + for (order = ttm_pool_alloc_find_order(MAX_PAGE_ORDER, alloc); + alloc->remaining_pages; + order = ttm_pool_alloc_find_order(order, alloc)) { struct ttm_pool_type *pt; - page_caching = tt->caching; - pt = ttm_pool_select_type(pool, tt->caching, order); - p = pt ? ttm_pool_type_take(pt) : NULL; - if (p) { - r = ttm_pool_apply_caching(caching, pages, - tt->caching); - if (r) - goto error_free_page; - - caching = pages; - do { - r = ttm_pool_page_allocated(pool, order, p, - &dma_addr, - &num_pages, - &pages); - if (r) - goto error_free_page; - - caching = pages; - if (num_pages < (1 << order)) - break; - - p = ttm_pool_type_take(pt); - } while (p); - } - - page_caching = ttm_cached; - while (num_pages >= (1 << order) && - (p = ttm_pool_alloc_page(pool, gfp_flags, order))) { - - if (PageHighMem(p)) { - r = ttm_pool_apply_caching(caching, pages, - tt->caching); - if (r) - goto error_free_page; - caching = pages; - } - r = ttm_pool_page_allocated(pool, order, p, &dma_addr, - &num_pages, &pages); - if (r) - goto error_free_page; - if (PageHighMem(p)) - caching = pages; + /* First, try to allocate a page from a pool if one exists. */ + p = NULL; + pt = ttm_pool_select_type(pool, page_caching, order); + if (pt && allow_pools) + p = ttm_pool_type_take(pt); + /* + * If that fails or previously failed, allocate from system. + * Note that this also disallows additional pool allocations using + * write-back cached pools of the same order. Consider removing + * that behaviour. + */ + if (!p) { + page_caching = ttm_cached; + allow_pools = false; + p = ttm_pool_alloc_page(pool, gfp_flags, order); } - + /* If that fails, lower the order if possible and retry. */ if (!p) { if (order) { --order; + page_caching = tt->caching; + allow_pools = true; continue; } r = -ENOMEM; goto error_free_all; } + r = ttm_pool_page_allocated(pool, order, p, page_caching, alloc, + restore); + if (r) + goto error_free_page; + + if (ttm_pool_restore_valid(restore)) { + r = ttm_pool_restore_commit(restore, tt->backup, ctx, alloc); + if (r) + goto error_free_all; + } } - r = ttm_pool_apply_caching(caching, pages, tt->caching); + r = ttm_pool_apply_caching(alloc); if (r) goto error_free_all; + kfree(tt->restore); + tt->restore = NULL; + return 0; error_free_page: ttm_pool_free_page(pool, page_caching, order, p); error_free_all: - num_pages = tt->num_pages - num_pages; - caching_divide = caching - tt->pages; + if (tt->restore) + return r; + + caching_divide = alloc->caching_divide - tt->pages; ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide); - ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, num_pages); + ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, + tt->num_pages - alloc->remaining_pages); return r; } + +/** + * ttm_pool_alloc - Fill a ttm_tt object + * + * @pool: ttm_pool to use + * @tt: ttm_tt object to fill + * @ctx: operation context + * + * Fill the ttm_tt object with pages and also make sure to DMA map them when + * necessary. + * + * Returns: 0 on successe, negative error code otherwise. + */ +int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + struct ttm_operation_ctx *ctx) +{ + struct ttm_pool_alloc_state alloc; + + if (WARN_ON(ttm_tt_is_backed_up(tt))) + return -EINVAL; + + ttm_pool_alloc_state_init(tt, &alloc); + + return __ttm_pool_alloc(pool, tt, ctx, &alloc, NULL); +} EXPORT_SYMBOL(ttm_pool_alloc); /** + * ttm_pool_restore_and_alloc - Fill a ttm_tt, restoring previously backed-up + * content. + * + * @pool: ttm_pool to use + * @tt: ttm_tt object to fill + * @ctx: operation context + * + * Fill the ttm_tt object with pages and also make sure to DMA map them when + * necessary. Read in backed-up content. + * + * Returns: 0 on successe, negative error code otherwise. + */ +int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx) +{ + struct ttm_pool_alloc_state alloc; + + if (WARN_ON(!ttm_tt_is_backed_up(tt))) + return -EINVAL; + + if (!tt->restore) { + gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; + + ttm_pool_alloc_state_init(tt, &alloc); + if (ctx->gfp_retry_mayfail) + gfp |= __GFP_RETRY_MAYFAIL; + + tt->restore = kzalloc(sizeof(*tt->restore), gfp); + if (!tt->restore) + return -ENOMEM; + + tt->restore->snapshot_alloc = alloc; + tt->restore->pool = pool; + tt->restore->restored_pages = 1; + } else { + struct ttm_pool_tt_restore *restore = tt->restore; + int ret; + + alloc = restore->snapshot_alloc; + if (ttm_pool_restore_valid(tt->restore)) { + ret = ttm_pool_restore_commit(restore, tt->backup, ctx, &alloc); + if (ret) + return ret; + } + if (!alloc.remaining_pages) + return 0; + } + + return __ttm_pool_alloc(pool, tt, ctx, &alloc, tt->restore); +} + +/** * ttm_pool_free - Free the backing pages from a ttm_tt object * * @pool: Pool to give pages back to. @@ -550,6 +890,169 @@ void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) EXPORT_SYMBOL(ttm_pool_free); /** + * ttm_pool_drop_backed_up() - Release content of a swapped-out struct ttm_tt + * @tt: The struct ttm_tt. + * + * Release handles with associated content or any remaining pages of + * a backed-up struct ttm_tt. + */ +void ttm_pool_drop_backed_up(struct ttm_tt *tt) +{ + struct ttm_pool_tt_restore *restore; + pgoff_t start_page = 0; + + WARN_ON(!ttm_tt_is_backed_up(tt)); + + restore = tt->restore; + + /* + * Unmap and free any uncommitted restore page. + * any tt page-array backup entries already read back has + * been cleared already + */ + if (ttm_pool_restore_valid(restore)) { + dma_addr_t *dma_addr = tt->dma_address ? &restore->first_dma : NULL; + + ttm_pool_unmap_and_free(restore->pool, restore->alloced_page, + dma_addr, restore->page_caching); + restore->restored_pages = 1UL << restore->order; + } + + /* + * If a restore is ongoing, part of the tt pages may have a + * caching different than writeback. + */ + if (restore) { + pgoff_t mid = restore->snapshot_alloc.caching_divide - tt->pages; + + start_page = restore->alloced_pages; + WARN_ON(mid > start_page); + /* Pages that might be dma-mapped and non-cached */ + ttm_pool_free_range(restore->pool, tt, tt->caching, + 0, mid); + /* Pages that might be dma-mapped but cached */ + ttm_pool_free_range(restore->pool, tt, ttm_cached, + mid, restore->alloced_pages); + kfree(restore); + tt->restore = NULL; + } + + ttm_pool_free_range(NULL, tt, ttm_cached, start_page, tt->num_pages); +} + +/** + * ttm_pool_backup() - Back up or purge a struct ttm_tt + * @pool: The pool used when allocating the struct ttm_tt. + * @tt: The struct ttm_tt. + * @flags: Flags to govern the backup behaviour. + * + * Back up or purge a struct ttm_tt. If @purge is true, then + * all pages will be freed directly to the system rather than to the pool + * they were allocated from, making the function behave similarly to + * ttm_pool_free(). If @purge is false the pages will be backed up instead, + * exchanged for handles. + * A subsequent call to ttm_pool_restore_and_alloc() will then read back the content and + * a subsequent call to ttm_pool_drop_backed_up() will drop it. + * If backup of a page fails for whatever reason, @ttm will still be + * partially backed up, retaining those pages for which backup fails. + * In that case, this function can be retried, possibly after freeing up + * memory resources. + * + * Return: Number of pages actually backed up or freed, or negative + * error code on error. + */ +long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, + const struct ttm_backup_flags *flags) +{ + struct file *backup = tt->backup; + struct page *page; + unsigned long handle; + gfp_t alloc_gfp; + gfp_t gfp; + int ret = 0; + pgoff_t shrunken = 0; + pgoff_t i, num_pages; + + if (WARN_ON(ttm_tt_is_backed_up(tt))) + return -EINVAL; + + if ((!ttm_backup_bytes_avail() && !flags->purge) || + pool->use_dma_alloc || ttm_tt_is_backed_up(tt)) + return -EBUSY; + +#ifdef CONFIG_X86 + /* Anything returned to the system needs to be cached. */ + if (tt->caching != ttm_cached) + set_pages_array_wb(tt->pages, tt->num_pages); +#endif + + if (tt->dma_address || flags->purge) { + for (i = 0; i < tt->num_pages; i += num_pages) { + unsigned int order; + + page = tt->pages[i]; + if (unlikely(!page)) { + num_pages = 1; + continue; + } + + order = ttm_pool_page_order(pool, page); + num_pages = 1UL << order; + if (tt->dma_address) + ttm_pool_unmap(pool, tt->dma_address[i], + num_pages); + if (flags->purge) { + shrunken += num_pages; + page->private = 0; + __free_pages(page, order); + memset(tt->pages + i, 0, + num_pages * sizeof(*tt->pages)); + } + } + } + + if (flags->purge) + return shrunken; + + if (pool->use_dma32) + gfp = GFP_DMA32; + else + gfp = GFP_HIGHUSER; + + alloc_gfp = GFP_KERNEL | __GFP_HIGH | __GFP_NOWARN | __GFP_RETRY_MAYFAIL; + + num_pages = tt->num_pages; + + /* Pretend doing fault injection by shrinking only half of the pages. */ + if (IS_ENABLED(CONFIG_FAULT_INJECTION) && should_fail(&backup_fault_inject, 1)) + num_pages = DIV_ROUND_UP(num_pages, 2); + + for (i = 0; i < num_pages; ++i) { + s64 shandle; + + page = tt->pages[i]; + if (unlikely(!page)) + continue; + + ttm_pool_split_for_swap(pool, page); + + shandle = ttm_backup_backup_page(backup, page, flags->writeback, i, + gfp, alloc_gfp); + if (shandle < 0) { + /* We allow partially shrunken tts */ + ret = shandle; + break; + } + handle = shandle; + tt->pages[i] = ttm_backup_handle_to_page_ptr(handle); + put_page(page); + shrunken++; + } + + return shrunken ? shrunken : ret; +} + +/** * ttm_pool_init - Initialize a pool * * @pool: the pool to initialize @@ -630,7 +1133,9 @@ void ttm_pool_fini(struct ttm_pool *pool) } EXPORT_SYMBOL(ttm_pool_fini); -/* As long as pages are available make sure to release at least one */ +/* Free average pool number of pages. */ +#define TTM_SHRINKER_BATCH ((1 << (MAX_PAGE_ORDER / 2)) * NR_PAGE_ORDERS) + static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) { @@ -638,9 +1143,12 @@ static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, do num_freed += ttm_pool_shrink(); - while (!num_freed && atomic_long_read(&allocated_pages)); + while (num_freed < sc->nr_to_scan && + atomic_long_read(&allocated_pages)); + + sc->nr_scanned = num_freed; - return num_freed; + return num_freed ?: SHRINK_STOP; } /* Return the number of pages available or SHRINK_EMPTY if we have none */ @@ -731,7 +1239,7 @@ int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) { unsigned int i; - if (!pool->use_dma_alloc) { + if (!pool->use_dma_alloc && pool->nid == NUMA_NO_NODE) { seq_puts(m, "unused\n"); return 0; } @@ -740,7 +1248,12 @@ int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) spin_lock(&shrinker_lock); for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { - seq_puts(m, "DMA "); + if (!ttm_pool_select_type(pool, i, 0)) + continue; + if (pool->use_dma_alloc) + seq_puts(m, "DMA "); + else + seq_printf(m, "N%d ", pool->nid); switch (i) { case ttm_cached: seq_puts(m, "\t:"); @@ -764,10 +1277,15 @@ EXPORT_SYMBOL(ttm_pool_debugfs); /* Test the shrinker functions and dump the result */ static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) { - struct shrink_control sc = { .gfp_mask = GFP_NOFS }; + struct shrink_control sc = { + .gfp_mask = GFP_NOFS, + .nr_to_scan = TTM_SHRINKER_BATCH, + }; + unsigned long count; fs_reclaim_acquire(GFP_KERNEL); - seq_printf(m, "%lu/%lu\n", ttm_pool_shrinker_count(mm_shrinker, &sc), + count = ttm_pool_shrinker_count(mm_shrinker, &sc); + seq_printf(m, "%lu/%lu\n", count, ttm_pool_shrinker_scan(mm_shrinker, &sc)); fs_reclaim_release(GFP_KERNEL); @@ -810,6 +1328,10 @@ int ttm_pool_mgr_init(unsigned long num_pages) &ttm_pool_debugfs_globals_fops); debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL, &ttm_pool_debugfs_shrink_fops); +#ifdef CONFIG_FAULT_INJECTION + fault_create_debugfs_attr("backup_fault_inject", ttm_debugfs_root, + &backup_fault_inject); +#endif #endif mm_shrinker = shrinker_alloc(0, "drm-ttm_pool"); @@ -818,6 +1340,7 @@ int ttm_pool_mgr_init(unsigned long num_pages) mm_shrinker->count_objects = ttm_pool_shrinker_count; mm_shrinker->scan_objects = ttm_pool_shrinker_scan; + mm_shrinker->batch = TTM_SHRINKER_BATCH; mm_shrinker->seeks = 1; shrinker_register(mm_shrinker); diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c index ae11d07eb63a..db854b581d83 100644 --- a/drivers/gpu/drm/ttm/ttm_range_manager.c +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -34,6 +34,8 @@ #include <drm/ttm/ttm_range_manager.h> #include <drm/ttm/ttm_bo.h> #include <drm/drm_mm.h> + +#include <linux/export.h> #include <linux/slab.h> #include <linux/spinlock.h> diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index cc29bbf3eabb..e2c82ad07eb4 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -23,6 +23,7 @@ */ #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/io-mapping.h> #include <linux/iosys-map.h> #include <linux/scatterlist.h> @@ -83,6 +84,23 @@ static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk) } /** + * ttm_resource_cursor_init() - Initialize a struct ttm_resource_cursor + * @cursor: The cursor to initialize. + * @man: The resource manager. + * + * Initialize the cursor before using it for iteration. + */ +void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor, + struct ttm_resource_manager *man) +{ + cursor->priority = 0; + cursor->man = man; + ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH); + INIT_LIST_HEAD(&cursor->bulk_link); + INIT_LIST_HEAD(&cursor->hitch.link); +} + +/** * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage * @cursor: The struct ttm_resource_cursor to finalize. * @@ -252,11 +270,16 @@ static bool ttm_resource_is_swapped(struct ttm_resource *res, struct ttm_buffer_ return ttm_tt_is_swapped(bo->ttm); } +static bool ttm_resource_unevictable(struct ttm_resource *res, struct ttm_buffer_object *bo) +{ + return bo->pin_count || ttm_resource_is_swapped(res, bo); +} + /* Add the resource to a bulk move if the BO is configured for it */ void ttm_resource_add_bulk_move(struct ttm_resource *res, struct ttm_buffer_object *bo) { - if (bo->bulk_move && !bo->pin_count && !ttm_resource_is_swapped(res, bo)) + if (bo->bulk_move && !ttm_resource_unevictable(res, bo)) ttm_lru_bulk_move_add(bo->bulk_move, res); } @@ -264,7 +287,7 @@ void ttm_resource_add_bulk_move(struct ttm_resource *res, void ttm_resource_del_bulk_move(struct ttm_resource *res, struct ttm_buffer_object *bo) { - if (bo->bulk_move && !bo->pin_count && !ttm_resource_is_swapped(res, bo)) + if (bo->bulk_move && !ttm_resource_unevictable(res, bo)) ttm_lru_bulk_move_del(bo->bulk_move, res); } @@ -276,10 +299,10 @@ void ttm_resource_move_to_lru_tail(struct ttm_resource *res) lockdep_assert_held(&bo->bdev->lru_lock); - if (bo->pin_count || ttm_resource_is_swapped(res, bo)) { + if (ttm_resource_unevictable(res, bo)) { list_move_tail(&res->lru.link, &bdev->unevictable); - } else if (bo->bulk_move) { + } else if (bo->bulk_move) { struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bo->bulk_move, res); @@ -318,7 +341,7 @@ void ttm_resource_init(struct ttm_buffer_object *bo, man = ttm_manager_type(bo->bdev, place->mem_type); spin_lock(&bo->bdev->lru_lock); - if (bo->pin_count || ttm_resource_is_swapped(res, bo)) + if (ttm_resource_unevictable(res, bo)) list_add_tail(&res->lru.link, &bo->bdev->unevictable); else list_add_tail(&res->lru.link, &man->lru[bo->priority]); @@ -526,7 +549,6 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, struct ttm_operation_ctx ctx = { .interruptible = false, .no_wait_gpu = false, - .force_alloc = true }; struct dma_fence *fence; int ret; @@ -536,6 +558,9 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, cond_resched(); } while (!ret); + if (ret && ret != -ENOENT) + return ret; + spin_lock(&man->move_lock); fence = dma_fence_get(man->move); spin_unlock(&man->move_lock); @@ -612,7 +637,6 @@ ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor, /** * ttm_resource_manager_first() - Start iterating over the resources * of a resource manager - * @man: resource manager to iterate over * @cursor: cursor to record the position * * Initializes the cursor and starts iterating. When done iterating, @@ -621,17 +645,16 @@ ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor, * Return: The first resource from the resource manager. */ struct ttm_resource * -ttm_resource_manager_first(struct ttm_resource_manager *man, - struct ttm_resource_cursor *cursor) +ttm_resource_manager_first(struct ttm_resource_cursor *cursor) { - lockdep_assert_held(&man->bdev->lru_lock); + struct ttm_resource_manager *man = cursor->man; - cursor->priority = 0; - cursor->man = man; - ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH); - INIT_LIST_HEAD(&cursor->bulk_link); - list_add(&cursor->hitch.link, &man->lru[cursor->priority]); + if (WARN_ON_ONCE(!man)) + return NULL; + lockdep_assert_held(&man->bdev->lru_lock); + + list_move(&cursor->hitch.link, &man->lru[cursor->priority]); return ttm_resource_manager_next(cursor); } @@ -667,8 +690,6 @@ ttm_resource_manager_next(struct ttm_resource_cursor *cursor) ttm_resource_cursor_clear_bulk(cursor); } - ttm_resource_cursor_fini(cursor); - return NULL; } diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 3baf215eca23..506e257dfba8 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -33,6 +33,7 @@ #include <linux/cc_platform.h> #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/file.h> #include <linux/module.h> #include <linux/sched.h> @@ -40,6 +41,7 @@ #include <drm/drm_cache.h> #include <drm/drm_device.h> #include <drm/drm_util.h> +#include <drm/ttm/ttm_backup.h> #include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_tt.h> @@ -158,6 +160,8 @@ static void ttm_tt_init_fields(struct ttm_tt *ttm, ttm->swap_storage = NULL; ttm->sg = bo->sg; ttm->caching = caching; + ttm->restore = NULL; + ttm->backup = NULL; } int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, @@ -182,6 +186,13 @@ void ttm_tt_fini(struct ttm_tt *ttm) fput(ttm->swap_storage); ttm->swap_storage = NULL; + if (ttm_tt_is_backed_up(ttm)) + ttm_pool_drop_backed_up(ttm); + if (ttm->backup) { + ttm_backup_fini(ttm->backup); + ttm->backup = NULL; + } + if (ttm->pages) kvfree(ttm->pages); else @@ -254,6 +265,49 @@ out_err: EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_swapin); /** + * ttm_tt_backup() - Helper to back up a struct ttm_tt. + * @bdev: The TTM device. + * @tt: The struct ttm_tt. + * @flags: Flags that govern the backup behaviour. + * + * Update the page accounting and call ttm_pool_shrink_tt to free pages + * or back them up. + * + * Return: Number of pages freed or swapped out, or negative error code on + * error. + */ +long ttm_tt_backup(struct ttm_device *bdev, struct ttm_tt *tt, + const struct ttm_backup_flags flags) +{ + long ret; + + if (WARN_ON(IS_ERR_OR_NULL(tt->backup))) + return 0; + + ret = ttm_pool_backup(&bdev->pool, tt, &flags); + if (ret > 0) { + tt->page_flags &= ~TTM_TT_FLAG_PRIV_POPULATED; + tt->page_flags |= TTM_TT_FLAG_BACKED_UP; + } + + return ret; +} + +int ttm_tt_restore(struct ttm_device *bdev, struct ttm_tt *tt, + const struct ttm_operation_ctx *ctx) +{ + int ret = ttm_pool_restore_and_alloc(&bdev->pool, tt, ctx); + + if (ret) + return ret; + + tt->page_flags &= ~TTM_TT_FLAG_BACKED_UP; + + return 0; +} +EXPORT_SYMBOL(ttm_tt_restore); + +/** * ttm_tt_swapout - swap out tt object * * @bdev: TTM device structure. @@ -348,6 +402,7 @@ int ttm_tt_populate(struct ttm_device *bdev, goto error; ttm->page_flags |= TTM_TT_FLAG_PRIV_POPULATED; + ttm->page_flags &= ~TTM_TT_FLAG_BACKED_UP; if (unlikely(ttm->page_flags & TTM_TT_FLAG_SWAPPED)) { ret = ttm_tt_swapin(ttm); if (unlikely(ret != 0)) { @@ -477,3 +532,32 @@ unsigned long ttm_tt_pages_limit(void) return ttm_pages_limit; } EXPORT_SYMBOL(ttm_tt_pages_limit); + +/** + * ttm_tt_setup_backup() - Allocate and assign a backup structure for a ttm_tt + * @tt: The ttm_tt for wich to allocate and assign a backup structure. + * + * Assign a backup structure to be used for tt backup. This should + * typically be done at bo creation, to avoid allocations at shrinking + * time. + * + * Return: 0 on success, negative error code on failure. + */ +int ttm_tt_setup_backup(struct ttm_tt *tt) +{ + struct file *backup = + ttm_backup_shmem_create(((loff_t)tt->num_pages) << PAGE_SHIFT); + + if (WARN_ON_ONCE(!(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))) + return -EINVAL; + + if (IS_ERR(backup)) + return PTR_ERR(backup); + + if (tt->backup) + ttm_backup_fini(tt->backup); + + tt->backup = backup; + return 0; +} +EXPORT_SYMBOL(ttm_tt_setup_backup); |