diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_bo_test.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_mock_manager.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_backup.c | 54 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 5 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_internal.h | 58 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_device.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_pool.c | 37 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_range_manager.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_resource.c | 5 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 3 |
15 files changed, 134 insertions, 52 deletions
diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c index e08e5a138420..6c77550c51af 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c @@ -175,7 +175,7 @@ struct signal_timer { static void signal_for_ttm_bo_reserve(struct timer_list *t) { - struct signal_timer *s_timer = from_timer(s_timer, t, timer); + struct signal_timer *s_timer = timer_container_of(s_timer, t, timer); struct task_struct *task = s_timer->ctx->task; do_send_sig_info(SIGTERM, SEND_SIG_PRIV, task, PIDTYPE_PID); @@ -201,7 +201,7 @@ static int threaded_ttm_bo_reserve(void *arg) err = ttm_bo_reserve(bo, interruptible, no_wait, &ctx); timer_delete_sync(&s_timer.timer); - destroy_timer_on_stack(&s_timer.timer); + timer_destroy_on_stack(&s_timer.timer); ww_acquire_fini(&ctx); diff --git a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c index b91c13f46225..7aaf0d1395ff 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c +++ b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c @@ -2,6 +2,9 @@ /* * Copyright © 2023 Intel Corporation */ + +#include <linux/export.h> + #include <drm/ttm/ttm_tt.h> #include "ttm_kunit_helpers.h" diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c index f6d1c8a2845d..d7eb6471f2ed 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c @@ -2,6 +2,9 @@ /* * Copyright © 2023 Intel Corporation */ + +#include <linux/export.h> + #include <drm/ttm/ttm_resource.h> #include <drm/ttm/ttm_device.h> #include <drm/ttm/ttm_placement.h> diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index d27691f2e451..fca0a1a3c6fd 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -36,6 +36,7 @@ #include <drm/ttm/ttm_tt.h> #include <drm/ttm/ttm_resource.h> #include <linux/agp_backend.h> +#include <linux/export.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/io.h> diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c index 93c007f18855..cb1b8e5dadf5 100644 --- a/drivers/gpu/drm/ttm/ttm_backup.c +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -4,24 +4,12 @@ */ #include <drm/ttm/ttm_backup.h> + +#include <linux/export.h> #include <linux/page-flags.h> #include <linux/swap.h> /* - * Casting from randomized struct file * to struct ttm_backup * is fine since - * struct ttm_backup is never defined nor dereferenced. - */ -static struct file *ttm_backup_to_file(struct ttm_backup *backup) -{ - return (void *)backup; -} - -static struct ttm_backup *ttm_file_to_backup(struct file *file) -{ - return (void *)file; -} - -/* * Need to map shmem indices to handle since a handle value * of 0 means error, following the swp_entry_t convention. */ @@ -40,12 +28,12 @@ static pgoff_t ttm_backup_handle_to_shmem_idx(pgoff_t handle) * @backup: The struct backup pointer used to obtain the handle * @handle: The handle obtained from the @backup_page function. */ -void ttm_backup_drop(struct ttm_backup *backup, pgoff_t handle) +void ttm_backup_drop(struct file *backup, pgoff_t handle) { loff_t start = ttm_backup_handle_to_shmem_idx(handle); start <<= PAGE_SHIFT; - shmem_truncate_range(file_inode(ttm_backup_to_file(backup)), start, + shmem_truncate_range(file_inode(backup), start, start + PAGE_SIZE - 1); } @@ -55,16 +43,15 @@ void ttm_backup_drop(struct ttm_backup *backup, pgoff_t handle) * @backup: The struct backup pointer used to back up the page. * @dst: The struct page to copy into. * @handle: The handle returned when the page was backed up. - * @intr: Try to perform waits interruptable or at least killable. + * @intr: Try to perform waits interruptible or at least killable. * * Return: 0 on success, Negative error code on failure, notably * -EINTR if @intr was set to true and a signal is pending. */ -int ttm_backup_copy_page(struct ttm_backup *backup, struct page *dst, +int ttm_backup_copy_page(struct file *backup, struct page *dst, pgoff_t handle, bool intr) { - struct file *filp = ttm_backup_to_file(backup); - struct address_space *mapping = filp->f_mapping; + struct address_space *mapping = backup->f_mapping; struct folio *from_folio; pgoff_t idx = ttm_backup_handle_to_shmem_idx(handle); @@ -106,12 +93,11 @@ int ttm_backup_copy_page(struct ttm_backup *backup, struct page *dst, * the folio size- and usage. */ s64 -ttm_backup_backup_page(struct ttm_backup *backup, struct page *page, +ttm_backup_backup_page(struct file *backup, struct page *page, bool writeback, pgoff_t idx, gfp_t page_gfp, gfp_t alloc_gfp) { - struct file *filp = ttm_backup_to_file(backup); - struct address_space *mapping = filp->f_mapping; + struct address_space *mapping = backup->f_mapping; unsigned long handle = 0; struct folio *to_folio; int ret; @@ -136,13 +122,13 @@ ttm_backup_backup_page(struct ttm_backup *backup, struct page *page, .for_reclaim = 1, }; folio_set_reclaim(to_folio); - ret = mapping->a_ops->writepage(folio_file_page(to_folio, idx), &wbc); + ret = shmem_writeout(to_folio, &wbc); if (!folio_test_writeback(to_folio)) folio_clear_reclaim(to_folio); /* - * If writepage succeeds, it unlocks the folio. - * writepage() errors are otherwise dropped, since writepage() - * is only best effort here. + * If writeout succeeds, it unlocks the folio. errors + * are otherwise dropped, since writeout is only best + * effort here. */ if (ret) folio_unlock(to_folio); @@ -161,9 +147,9 @@ ttm_backup_backup_page(struct ttm_backup *backup, struct page *page, * * After a call to this function, it's illegal to use the @backup pointer. */ -void ttm_backup_fini(struct ttm_backup *backup) +void ttm_backup_fini(struct file *backup) { - fput(ttm_backup_to_file(backup)); + fput(backup); } /** @@ -194,14 +180,10 @@ EXPORT_SYMBOL_GPL(ttm_backup_bytes_avail); * * Create a backup utilizing shmem objects. * - * Return: A pointer to a struct ttm_backup on success, + * Return: A pointer to a struct file on success, * an error pointer on error. */ -struct ttm_backup *ttm_backup_shmem_create(loff_t size) +struct file *ttm_backup_shmem_create(loff_t size) { - struct file *filp; - - filp = shmem_file_setup("ttm shmem backup", size, 0); - - return ttm_file_to_backup(filp); + return shmem_file_setup("ttm shmem backup", size, 0); } diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index e218a7ce490e..bb9c5c8e16b5 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -35,6 +35,7 @@ #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_tt.h> +#include <linux/export.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> @@ -46,6 +47,7 @@ #include <linux/dma-resv.h> #include "ttm_module.h" +#include "ttm_bo_internal.h" static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_placement *placement) @@ -1093,7 +1095,8 @@ struct ttm_bo_swapout_walk { struct ttm_lru_walk walk; /** @gfp_flags: The gfp flags to use for ttm_tt_swapout() */ gfp_t gfp_flags; - + /** @hit_low: Whether we should attempt to swap BO's with low watermark threshold */ + /** @evict_low: If we cannot swap a bo when @try_low is false (first pass) */ bool hit_low, evict_low; }; diff --git a/drivers/gpu/drm/ttm/ttm_bo_internal.h b/drivers/gpu/drm/ttm/ttm_bo_internal.h new file mode 100644 index 000000000000..9d8b747a34db --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_internal.h @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ + +#ifndef _TTM_BO_INTERNAL_H_ +#define _TTM_BO_INTERNAL_H_ + +#include <drm/ttm/ttm_bo.h> + +/** + * ttm_bo_get - reference a struct ttm_buffer_object + * + * @bo: The buffer object. + */ +static inline void ttm_bo_get(struct ttm_buffer_object *bo) +{ + kref_get(&bo->kref); +} + +/** + * ttm_bo_get_unless_zero - reference a struct ttm_buffer_object unless + * its refcount has already reached zero. + * @bo: The buffer object. + * + * Used to reference a TTM buffer object in lookups where the object is removed + * from the lookup structure during the destructor and for RCU lookups. + * + * Returns: @bo if the referencing was successful, NULL otherwise. + */ +static inline __must_check struct ttm_buffer_object * +ttm_bo_get_unless_zero(struct ttm_buffer_object *bo) +{ + if (!kref_get_unless_zero(&bo->kref)) + return NULL; + return bo; +} + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 15cab9bda17f..b9a772b26fa1 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -28,6 +28,8 @@ /* * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ + +#include <linux/export.h> #include <linux/swap.h> #include <linux/vmalloc.h> @@ -37,6 +39,8 @@ #include <drm/drm_cache.h> +#include "ttm_bo_internal.h" + struct ttm_transfer_obj { struct ttm_buffer_object base; struct ttm_buffer_object *bo; diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index a194db83421d..b47020fca199 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -31,6 +31,8 @@ #define pr_fmt(fmt) "[TTM] " fmt +#include <linux/export.h> + #include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_tt.h> @@ -220,7 +222,6 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, struct ttm_operation_ctx ctx = { .interruptible = true, .no_wait_gpu = false, - .force_alloc = true }; ttm = bo->ttm; diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c index 02e797fd1891..816e2cba6016 100644 --- a/drivers/gpu/drm/ttm/ttm_device.c +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -28,6 +28,7 @@ #define pr_fmt(fmt) "[TTM DEVICE] " fmt #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/mm.h> #include <drm/ttm/ttm_bo.h> @@ -36,6 +37,7 @@ #include <drm/ttm/ttm_placement.h> #include "ttm_module.h" +#include "ttm_bo_internal.h" /* * ttm_global_mutex - protecting the global state diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index f1c60fa80c2d..bc7a83a9fe44 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -26,6 +26,8 @@ * **************************************************************************/ +#include <linux/export.h> + #include <drm/ttm/ttm_execbuf_util.h> #include <drm/ttm/ttm_bo.h> diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index 83b10706ba89..baf27c70a419 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -31,6 +31,7 @@ * cause they are rather slow compared to alloc_pages+map. */ +#include <linux/export.h> #include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/debugfs.h> @@ -506,7 +507,7 @@ static void ttm_pool_allocated_page_commit(struct page *allocated, * if successful, populate the page-table and dma-address arrays. */ static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore, - struct ttm_backup *backup, + struct file *backup, const struct ttm_operation_ctx *ctx, struct ttm_pool_alloc_state *alloc) @@ -655,7 +656,7 @@ static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt, pgoff_t start_page, pgoff_t end_page) { struct page **pages = &tt->pages[start_page]; - struct ttm_backup *backup = tt->backup; + struct file *backup = tt->backup; pgoff_t i, nr; for (i = start_page; i < end_page; i += nr, pages += nr) { @@ -963,7 +964,7 @@ void ttm_pool_drop_backed_up(struct ttm_tt *tt) long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, const struct ttm_backup_flags *flags) { - struct ttm_backup *backup = tt->backup; + struct file *backup = tt->backup; struct page *page; unsigned long handle; gfp_t alloc_gfp; @@ -1132,7 +1133,9 @@ void ttm_pool_fini(struct ttm_pool *pool) } EXPORT_SYMBOL(ttm_pool_fini); -/* As long as pages are available make sure to release at least one */ +/* Free average pool number of pages. */ +#define TTM_SHRINKER_BATCH ((1 << (MAX_PAGE_ORDER / 2)) * NR_PAGE_ORDERS) + static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) { @@ -1140,9 +1143,12 @@ static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, do num_freed += ttm_pool_shrink(); - while (!num_freed && atomic_long_read(&allocated_pages)); + while (num_freed < sc->nr_to_scan && + atomic_long_read(&allocated_pages)); + + sc->nr_scanned = num_freed; - return num_freed; + return num_freed ?: SHRINK_STOP; } /* Return the number of pages available or SHRINK_EMPTY if we have none */ @@ -1233,7 +1239,7 @@ int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) { unsigned int i; - if (!pool->use_dma_alloc) { + if (!pool->use_dma_alloc && pool->nid == NUMA_NO_NODE) { seq_puts(m, "unused\n"); return 0; } @@ -1242,7 +1248,12 @@ int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) spin_lock(&shrinker_lock); for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { - seq_puts(m, "DMA "); + if (!ttm_pool_select_type(pool, i, 0)) + continue; + if (pool->use_dma_alloc) + seq_puts(m, "DMA "); + else + seq_printf(m, "N%d ", pool->nid); switch (i) { case ttm_cached: seq_puts(m, "\t:"); @@ -1266,10 +1277,15 @@ EXPORT_SYMBOL(ttm_pool_debugfs); /* Test the shrinker functions and dump the result */ static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) { - struct shrink_control sc = { .gfp_mask = GFP_NOFS }; + struct shrink_control sc = { + .gfp_mask = GFP_NOFS, + .nr_to_scan = TTM_SHRINKER_BATCH, + }; + unsigned long count; fs_reclaim_acquire(GFP_KERNEL); - seq_printf(m, "%lu/%lu\n", ttm_pool_shrinker_count(mm_shrinker, &sc), + count = ttm_pool_shrinker_count(mm_shrinker, &sc); + seq_printf(m, "%lu/%lu\n", count, ttm_pool_shrinker_scan(mm_shrinker, &sc)); fs_reclaim_release(GFP_KERNEL); @@ -1324,6 +1340,7 @@ int ttm_pool_mgr_init(unsigned long num_pages) mm_shrinker->count_objects = ttm_pool_shrinker_count; mm_shrinker->scan_objects = ttm_pool_shrinker_scan; + mm_shrinker->batch = TTM_SHRINKER_BATCH; mm_shrinker->seeks = 1; shrinker_register(mm_shrinker); diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c index ae11d07eb63a..db854b581d83 100644 --- a/drivers/gpu/drm/ttm/ttm_range_manager.c +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -34,6 +34,8 @@ #include <drm/ttm/ttm_range_manager.h> #include <drm/ttm/ttm_bo.h> #include <drm/drm_mm.h> + +#include <linux/export.h> #include <linux/slab.h> #include <linux/spinlock.h> diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 7e5a60c55813..e2c82ad07eb4 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -23,6 +23,7 @@ */ #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/io-mapping.h> #include <linux/iosys-map.h> #include <linux/scatterlist.h> @@ -548,7 +549,6 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, struct ttm_operation_ctx ctx = { .interruptible = false, .no_wait_gpu = false, - .force_alloc = true }; struct dma_fence *fence; int ret; @@ -558,6 +558,9 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, cond_resched(); } while (!ret); + if (ret && ret != -ENOENT) + return ret; + spin_lock(&man->move_lock); fence = dma_fence_get(man->move); spin_unlock(&man->move_lock); diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index df0aa6c4b8b8..506e257dfba8 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -33,6 +33,7 @@ #include <linux/cc_platform.h> #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/file.h> #include <linux/module.h> #include <linux/sched.h> @@ -544,7 +545,7 @@ EXPORT_SYMBOL(ttm_tt_pages_limit); */ int ttm_tt_setup_backup(struct ttm_tt *tt) { - struct ttm_backup *backup = + struct file *backup = ttm_backup_shmem_create(((loff_t)tt->num_pages) << PAGE_SHIFT); if (WARN_ON_ONCE(!(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))) |