diff options
Diffstat (limited to 'drivers/gpu/drm/img/img-rogue/services/server/env/linux/physmem_osmem_linux.c')
-rw-r--r-- | drivers/gpu/drm/img/img-rogue/services/server/env/linux/physmem_osmem_linux.c | 3960 |
1 files changed, 3960 insertions, 0 deletions
diff --git a/drivers/gpu/drm/img/img-rogue/services/server/env/linux/physmem_osmem_linux.c b/drivers/gpu/drm/img/img-rogue/services/server/env/linux/physmem_osmem_linux.c new file mode 100644 index 000000000000..310044fbc9b1 --- /dev/null +++ b/drivers/gpu/drm/img/img-rogue/services/server/env/linux/physmem_osmem_linux.c @@ -0,0 +1,3960 @@ +/*************************************************************************/ /*! +@File +@Title Implementation of PMR functions for OS managed memory +@Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved +@Description Part of the memory management. This module is responsible for + implementing the function callbacks for physical memory borrowed + from that normally managed by the operating system. +@License Dual MIT/GPLv2 + +The contents of this file are subject to the MIT license as set out below. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +Alternatively, the contents of this file may be used under the terms of +the GNU General Public License Version 2 ("GPL") in which case the provisions +of GPL are applicable instead of those above. + +If you wish to allow use of your version of this file only under the terms of +GPL, and not to allow others to use your version of this file under the terms +of the MIT license, indicate your decision by deleting the provisions above +and replace them with the notice and other provisions required by GPL as set +out in the file called "GPL-COPYING" included in this distribution. If you do +not delete the provisions above, a recipient may use your version of this file +under the terms of either the MIT license or GPL. + +This License is also included in this distribution in the file called +"MIT-COPYING". + +EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS +PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ /**************************************************************************/ +#include <linux/version.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/highmem.h> +#include <linux/mm_types.h> +#include <linux/vmalloc.h> +#include <linux/gfp.h> +#include <linux/sched.h> +#include <linux/atomic.h> + +#if defined(CONFIG_X86) +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)) +#include <asm/set_memory.h> +#else +#include <asm/cacheflush.h> +#endif +#endif + +/* include/ */ +#include "rgx_heaps.h" +#include "img_types.h" +#include "img_defs.h" +#include "pvr_debug.h" +#include "pvrsrv_error.h" +#include "pvrsrv_memallocflags.h" +#include "rgx_pdump_panics.h" +/* services/server/include/ */ +#include "allocmem.h" +#include "osfunc.h" +#include "pdump_km.h" +#include "pmr.h" +#include "pmr_impl.h" +#include "cache_km.h" +#include "devicemem_server_utils.h" +#include "pvr_vmap.h" +#include "physheap.h" + +/* ourselves */ +#include "physmem_osmem.h" +#include "physmem_osmem_linux.h" + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) +#include "process_stats.h" +#if !defined(PVRSRV_ENABLE_MEMORY_STATS) +#include "hash.h" +#endif +#endif + +#include "kernel_compatibility.h" + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)) +static IMG_UINT32 g_uiMaxOrder = PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM; +#else +/* split_page not available on older kernels */ +#undef PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM +#define PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM 0 +static IMG_UINT32 g_uiMaxOrder; +#endif + +/* + These corresponds to the MMU min/max page sizes and associated PTE + alignment that can be used on the device for an allocation. It is + 4KB (min) and 2MB (max) respectively. +*/ +#define PVR_MIN_PHYSMEM_CONTIG_ALLOC_LOG2PGSZ RGX_HEAP_4KB_PAGE_SHIFT +#define PVR_MAX_PHYSMEM_CONTIG_ALLOC_LOG2PGSZ RGX_HEAP_2MB_PAGE_SHIFT + +/* Defines how many pages should be mapped at once to the kernel */ +#define PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES 1024 /* 4 MB */ + +/* + These are used to get/set/mask lower-order bits in a dma_addr_t + to provide side-band information associated with that address. + These includes whether the address was obtained via alloc_page + or dma_alloc and if address came allocated pre-aligned or an + adjustment was made manually to aligned it. +*/ +#define DMA_SET_ADJUSTED_ADDR(x) ((x) | ((dma_addr_t)0x02)) +#define DMA_IS_ADDR_ADJUSTED(x) ((x) & ((dma_addr_t)0x02)) +#define DMA_SET_ALLOCPG_ADDR(x) ((x) | ((dma_addr_t)0x01)) +#define DMA_IS_ALLOCPG_ADDR(x) ((x) & ((dma_addr_t)0x01)) +#define DMA_GET_ALIGN_ADJUSTMENT(x) ((x>>2) & ((dma_addr_t)0x3ff)) +#define DMA_SET_ALIGN_ADJUSTMENT(x,y) ((x) | (((dma_addr_t)y)<<0x02)) +#define DMA_GET_ADDR(x) (((dma_addr_t)x) & ((dma_addr_t)~0xfff)) +#define DMA_VADDR_NOT_IN_USE 0xCAFEF00DDEADBEEFULL + +#define PVRSRV_ZERO_VALUE 0 + +typedef struct _PMR_OSPAGEARRAY_DATA_ { + /* Device for which this allocation has been made */ + PVRSRV_DEVICE_NODE *psDevNode; + /* The pid that made this allocation */ + IMG_PID uiPid; + + /* + * iNumOSPagesAllocated: + * Number of pages allocated in this PMR so far. + * This allows for up to (2^31 - 1) pages. With 4KB pages, that's 8TB of memory for each PMR. + */ + IMG_INT32 iNumOSPagesAllocated; + + /* + * uiTotalNumOSPages: + * Total number of pages supported by this PMR. (Fixed as of now due the fixed Page table array size) + * number of "pages" (a.k.a. macro pages, compound pages, higher order pages, etc...) + */ + IMG_UINT32 uiTotalNumOSPages; + + /* + uiLog2AllocPageSize; + + size of each "page" -- this would normally be the same as + PAGE_SHIFT, but we support the idea that we may allocate pages + in larger chunks for better contiguity, using order>0 in the + call to alloc_pages() + */ + IMG_UINT32 uiLog2AllocPageSize; + + /* + ui64DmaMask; + */ + IMG_UINT64 ui64DmaMask; + + /* + For non DMA/CMA allocation, pagearray references the pages + thus allocated; one entry per compound page when compound + pages are used. In addition, for DMA/CMA allocations, we + track the returned cpu virtual and device bus address. + */ + struct page **pagearray; + dma_addr_t *dmaphysarray; + void **dmavirtarray; + + +#define FLAG_ZERO (0U) +#define FLAG_POISON_ON_FREE (1U) +#define FLAG_POISON_ON_ALLOC (2U) +#define FLAG_ONDEMAND (3U) +#define FLAG_UNPINNED (4U) +#define FLAG_IS_CMA (5U) +#define FLAG_UNSET_MEMORY_TYPE (6U) + + /* + * Allocation flags related to the pages: + * Zero - Should we Zero memory on alloc + * Poison on free - Should we Poison the memory on free. + * Poison on alloc - Should we Poison the memory on alloc. + * On demand - Is the allocation on Demand i.e Do we defer allocation to time of use. + * Unpinned - Should be protected by page pool lock + * CMA - Is CMA memory allocated via DMA framework + * Unset Memory Type - Upon free do we need to revert the cache type before return to OS + * */ + IMG_UINT32 ui32AllocFlags; + + /* + The cache mode of the PMR. Additionally carrying the CPU-Cache-Clean + flag, advising us to do cache maintenance on behalf of the caller. + Boolean used to track if we need to revert the cache attributes + of the pages used in this allocation. Depends on OS/architecture. + */ + IMG_UINT32 ui32CPUCacheFlags; + /* + * In CMA allocation path, algorithm can allocate double the size of + * requested allocation size to satisfy the alignment. In this case + * the additional pages allocated are tracked through this additional + * variable and are accounted for in the memory statistics */ + IMG_UINT32 ui32CMAAdjustedPageCount; +} PMR_OSPAGEARRAY_DATA; + +/*********************************** + * Page pooling for uncached pages * + ***********************************/ + +static INLINE void +_FreeOSPage_CMA(struct device *dev, + size_t alloc_size, + IMG_UINT32 uiOrder, + void *virt_addr, + dma_addr_t dev_addr, + struct page *psPage); + +static void +_FreeOSPage(IMG_UINT32 uiOrder, + IMG_BOOL bUnsetMemoryType, + struct page *psPage); + +static PVRSRV_ERROR +_FreeOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData, + IMG_UINT32 *pai32FreeIndices, + IMG_UINT32 ui32FreePageCount); + +static PVRSRV_ERROR +_FreePagesFromPoolUnlocked(IMG_UINT32 uiMaxPagesToFree, + IMG_UINT32 *puiPagesFreed); + +/* A struct for our page pool holding an array of zeroed (!) pages. + * We always put units of page arrays to the pool but are + * able to take individual pages */ +typedef struct +{ + /* Linkage for page pool LRU list */ + struct list_head sPagePoolItem; + + /* How many items are still in the page array */ + IMG_UINT32 uiItemsRemaining; + /* Array of the actual pages */ + struct page **ppsPageArray; + +} LinuxPagePoolEntry; + +/* CleanupThread structure to put allocation in page pool */ +typedef struct +{ + PVRSRV_CLEANUP_THREAD_WORK sCleanupWork; + IMG_UINT32 ui32CPUCacheMode; + LinuxPagePoolEntry *psPoolEntry; +} LinuxCleanupData; + +/* A struct for the unpinned items */ +typedef struct +{ + struct list_head sUnpinPoolItem; + PMR_OSPAGEARRAY_DATA *psPageArrayDataPtr; +} LinuxUnpinEntry; + + +/* Caches to hold page pool and page array structures */ +static struct kmem_cache *g_psLinuxPagePoolCache; +static struct kmem_cache *g_psLinuxPageArray; + +/* Track what is live, all protected by pool lock. + * x86 needs two page pools because we have to change the memory attributes + * of the pages which is expensive due to an implicit flush. + * See set_pages_array_uc/wc/wb. */ +static IMG_UINT32 g_ui32UnpinPageCount; +static IMG_UINT32 g_ui32PagePoolUCCount; +#if defined(CONFIG_X86) +static IMG_UINT32 g_ui32PagePoolWCCount; +#endif +/* Tracks asynchronous tasks currently accessing the page pool. + * It is incremented if a defer free task + * is created. Both will decrement the value when they finished the work. + * The atomic prevents piling up of deferred work in case the deferred thread + * cannot keep up with the application.*/ +static ATOMIC_T g_iPoolCleanTasks; +/* We don't want too many asynchronous threads trying to access the page pool + * at the same time */ +#define PVR_LINUX_PHYSMEM_MAX_ASYNC_CLEAN_TASKS 128 + +/* Defines how many pages the page cache should hold. */ +#if defined(PVR_LINUX_PHYSMEM_MAX_POOL_PAGES) +static const IMG_UINT32 g_ui32PagePoolMaxEntries = PVR_LINUX_PHYSMEM_MAX_POOL_PAGES; +#else +static const IMG_UINT32 g_ui32PagePoolMaxEntries; +#endif + +/* We double check if we would exceed this limit if we are below MAX_POOL_PAGES + and want to add an allocation to the pool. + This prevents big allocations being given back to the OS just because they + exceed the MAX_POOL_PAGES limit even though the pool is currently empty. */ +#if defined(PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES) +static const IMG_UINT32 g_ui32PagePoolMaxExcessEntries = PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES; +#else +static const IMG_UINT32 g_ui32PagePoolMaxExcessEntries; +#endif + +#if defined(CONFIG_X86) +#define PHYSMEM_OSMEM_NUM_OF_POOLS 2 +static const IMG_UINT32 g_aui32CPUCacheFlags[PHYSMEM_OSMEM_NUM_OF_POOLS] = { + PVRSRV_MEMALLOCFLAG_CPU_UNCACHED, + PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC +}; +#else +#define PHYSMEM_OSMEM_NUM_OF_POOLS 1 +static const IMG_UINT32 g_aui32CPUCacheFlags[PHYSMEM_OSMEM_NUM_OF_POOLS] = { + PVRSRV_MEMALLOCFLAG_CPU_UNCACHED +}; +#endif + +/* Global structures we use to manage the page pool */ +static DEFINE_MUTEX(g_sPagePoolMutex); + +/* List holding the page array pointers: */ +static LIST_HEAD(g_sPagePoolList_WC); +static LIST_HEAD(g_sPagePoolList_UC); +static LIST_HEAD(g_sUnpinList); + +#if defined(DEBUG) && defined(SUPPORT_VALIDATION) +/* Global structure to manage GPU memory leak */ +static DEFINE_MUTEX(g_sUMALeakMutex); +static IMG_UINT32 g_ui32UMALeakCounter = 0; +#endif + +static inline IMG_UINT32 +_PagesInPoolUnlocked(void) +{ + IMG_UINT32 uiCnt = g_ui32PagePoolUCCount; +#if defined(CONFIG_X86) + uiCnt += g_ui32PagePoolWCCount; +#endif + return uiCnt; +} + +static inline void +_PagePoolLock(void) +{ + mutex_lock(&g_sPagePoolMutex); +} + +static inline int +_PagePoolTrylock(void) +{ + return mutex_trylock(&g_sPagePoolMutex); +} + +static inline void +_PagePoolUnlock(void) +{ + mutex_unlock(&g_sPagePoolMutex); +} + +static PVRSRV_ERROR +_AddUnpinListEntryUnlocked(PMR_OSPAGEARRAY_DATA *psOSPageArrayData) +{ + LinuxUnpinEntry *psUnpinEntry; + + psUnpinEntry = OSAllocMem(sizeof(*psUnpinEntry)); + if (!psUnpinEntry) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: OSAllocMem failed. Cannot add entry to unpin list.", + __func__)); + return PVRSRV_ERROR_OUT_OF_MEMORY; + } + + psUnpinEntry->psPageArrayDataPtr = psOSPageArrayData; + + /* Add into pool that the shrinker can access easily*/ + list_add_tail(&psUnpinEntry->sUnpinPoolItem, &g_sUnpinList); + + g_ui32UnpinPageCount += psOSPageArrayData->iNumOSPagesAllocated; + + return PVRSRV_OK; +} + +static void +_RemoveUnpinListEntryUnlocked(PMR_OSPAGEARRAY_DATA *psOSPageArrayData) +{ + LinuxUnpinEntry *psUnpinEntry, *psTempUnpinEntry; + + /* Remove from pool */ + list_for_each_entry_safe(psUnpinEntry, + psTempUnpinEntry, + &g_sUnpinList, + sUnpinPoolItem) + { + if (psUnpinEntry->psPageArrayDataPtr == psOSPageArrayData) + { + list_del(&psUnpinEntry->sUnpinPoolItem); + break; + } + } + + OSFreeMem(psUnpinEntry); + + g_ui32UnpinPageCount -= psOSPageArrayData->iNumOSPagesAllocated; +} + +static inline IMG_BOOL +_GetPoolListHead(IMG_UINT32 ui32CPUCacheFlags, + struct list_head **ppsPoolHead, + IMG_UINT32 **ppuiCounter) +{ + switch (PVRSRV_CPU_CACHE_MODE(ui32CPUCacheFlags)) + { + case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC: +#if defined(CONFIG_X86) + /* + For x86 we need to keep different lists for uncached + and write-combined as we must always honour the PAT + setting which cares about this difference. + */ + + *ppsPoolHead = &g_sPagePoolList_WC; + *ppuiCounter = &g_ui32PagePoolWCCount; + break; +#endif + + case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED: + *ppsPoolHead = &g_sPagePoolList_UC; + *ppuiCounter = &g_ui32PagePoolUCCount; + break; + + default: + PVR_DPF((PVR_DBG_ERROR, + "%s: Unknown CPU caching mode. " + "Using default UC pool.", + __func__)); + *ppsPoolHead = &g_sPagePoolList_UC; + *ppuiCounter = &g_ui32PagePoolUCCount; + PVR_ASSERT(0); + return IMG_FALSE; + } + return IMG_TRUE; +} + +static struct shrinker g_sShrinker; + +/* Returning the number of pages that still reside in the page pool. */ +static unsigned long +_GetNumberOfPagesInPoolUnlocked(void) +{ + return _PagesInPoolUnlocked() + g_ui32UnpinPageCount; +} + +/* Linux shrinker function that informs the OS about how many pages we are caching and + * it is able to reclaim. */ +static unsigned long +_CountObjectsInPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl) +{ + int remain; + + PVR_ASSERT(psShrinker == &g_sShrinker); + (void)psShrinker; + (void)psShrinkControl; + + /* In order to avoid possible deadlock use mutex_trylock in place of mutex_lock */ + if (_PagePoolTrylock() == 0) + return 0; + remain = _GetNumberOfPagesInPoolUnlocked(); + _PagePoolUnlock(); + + return remain; +} + +/* Linux shrinker function to reclaim the pages from our page pool */ +static unsigned long +_ScanObjectsInPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl) +{ + unsigned long uNumToScan = psShrinkControl->nr_to_scan; + unsigned long uSurplus = 0; + LinuxUnpinEntry *psUnpinEntry, *psTempUnpinEntry; + IMG_UINT32 uiPagesFreed; + + PVR_ASSERT(psShrinker == &g_sShrinker); + (void)psShrinker; + + /* In order to avoid possible deadlock use mutex_trylock in place of mutex_lock */ + if (_PagePoolTrylock() == 0) + return SHRINK_STOP; + + _FreePagesFromPoolUnlocked(uNumToScan, + &uiPagesFreed); + uNumToScan -= uiPagesFreed; + + if (uNumToScan == 0) + { + goto e_exit; + } + + /* Free unpinned memory, starting with LRU entries */ + list_for_each_entry_safe(psUnpinEntry, + psTempUnpinEntry, + &g_sUnpinList, + sUnpinPoolItem) + { + PMR_OSPAGEARRAY_DATA *psPageArrayDataPtr = psUnpinEntry->psPageArrayDataPtr; + IMG_UINT32 uiNumPages = (psPageArrayDataPtr->uiTotalNumOSPages > psPageArrayDataPtr->iNumOSPagesAllocated)? + psPageArrayDataPtr->iNumOSPagesAllocated:psPageArrayDataPtr->uiTotalNumOSPages; + PVRSRV_ERROR eError; + + /* Free associated pages */ + eError = _FreeOSPages(psPageArrayDataPtr, + NULL, + 0); + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Shrinker is unable to free unpinned pages. Error: %s (%d)", + __func__, + PVRSRVGetErrorString(eError), + eError)); + goto e_exit; + } + + /* Remove item from pool */ + list_del(&psUnpinEntry->sUnpinPoolItem); + + g_ui32UnpinPageCount -= uiNumPages; + + /* Check if there is more to free or if we already surpassed the limit */ + if (uiNumPages < uNumToScan) + { + uNumToScan -= uiNumPages; + + } + else if (uiNumPages > uNumToScan) + { + uSurplus += uiNumPages - uNumToScan; + uNumToScan = 0; + goto e_exit; + } + else + { + uNumToScan -= uiNumPages; + goto e_exit; + } + } + +e_exit: + if (list_empty(&g_sUnpinList)) + { + PVR_ASSERT(g_ui32UnpinPageCount == 0); + } + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,12,0)) + { + int remain; + remain = _GetNumberOfPagesInPoolUnlocked(); + _PagePoolUnlock(); + return remain; + } +#else + /* Returning the number of pages freed during the scan */ + _PagePoolUnlock(); + return psShrinkControl->nr_to_scan - uNumToScan + uSurplus; +#endif +} + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,12,0)) +static int +_ShrinkPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl) +{ + if (psShrinkControl->nr_to_scan != 0) + { + return _ScanObjectsInPagePool(psShrinker, psShrinkControl); + } + else + { + /* No pages are being reclaimed so just return the page count */ + return _CountObjectsInPagePool(psShrinker, psShrinkControl); + } +} + +static struct shrinker g_sShrinker = +{ + .shrink = _ShrinkPagePool, + .seeks = DEFAULT_SEEKS +}; +#else +static struct shrinker g_sShrinker = +{ + .count_objects = _CountObjectsInPagePool, + .scan_objects = _ScanObjectsInPagePool, + .seeks = DEFAULT_SEEKS +}; +#endif + +/* Register the shrinker so Linux can reclaim cached pages */ +void LinuxInitPhysmem(void) +{ + g_psLinuxPageArray = kmem_cache_create("pvr-pa", sizeof(PMR_OSPAGEARRAY_DATA), 0, 0, NULL); + + g_psLinuxPagePoolCache = kmem_cache_create("pvr-pp", sizeof(LinuxPagePoolEntry), 0, 0, NULL); + if (g_psLinuxPagePoolCache) + { + /* Only create the shrinker if we created the cache OK */ + register_shrinker(&g_sShrinker,"pvr-pp"); + } + + OSAtomicWrite(&g_iPoolCleanTasks, 0); +} + +/* Unregister the shrinker and remove all pages from the pool that are still left */ +void LinuxDeinitPhysmem(void) +{ + IMG_UINT32 uiPagesFreed; + + if (OSAtomicRead(&g_iPoolCleanTasks) > 0) + { + PVR_DPF((PVR_DBG_WARNING, "Still deferred cleanup tasks running " + "while deinitialising memory subsystem.")); + } + + _PagePoolLock(); + if (_FreePagesFromPoolUnlocked(IMG_UINT32_MAX, &uiPagesFreed) != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, "Unable to free all pages from page pool when " + "deinitialising memory subsystem.")); + PVR_ASSERT(0); + } + + PVR_ASSERT(_PagesInPoolUnlocked() == 0); + + /* Free the page cache */ + kmem_cache_destroy(g_psLinuxPagePoolCache); + + unregister_shrinker(&g_sShrinker); + _PagePoolUnlock(); + + kmem_cache_destroy(g_psLinuxPageArray); +} + +static void EnableOOMKiller(void) +{ + current->flags &= ~PF_DUMPCORE; +} + +static void DisableOOMKiller(void) +{ + /* PF_DUMPCORE is treated by the VM as if the OOM killer was disabled. + * + * As oom_killer_disable() is an inline, non-exported function, we + * can't use it from a modular driver. Furthermore, the OOM killer + * API doesn't look thread safe, which 'current' is. + */ + WARN_ON(current->flags & PF_DUMPCORE); + current->flags |= PF_DUMPCORE; +} + +/* Prints out the addresses in a page array for debugging purposes + * Define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY locally to activate: */ +/* #define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY 1 */ +static inline void +_DumpPageArray(struct page **pagearray, IMG_UINT32 uiPagesToPrint) +{ +#if defined(PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY) + IMG_UINT32 i; + if (pagearray) + { + printk("Array %p:\n", pagearray); + for (i = 0; i < uiPagesToPrint; i++) + { + printk("%p | ", (pagearray)[i]); + } + printk("\n"); + } + else + { + printk("Array is NULL:\n"); + } +#else + PVR_UNREFERENCED_PARAMETER(pagearray); + PVR_UNREFERENCED_PARAMETER(uiPagesToPrint); +#endif +} + +/* Debugging function that dumps out the number of pages for every + * page array that is currently in the page pool. + * Not defined by default. Define locally to activate feature: */ +/* #define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_POOL 1 */ +static void +_DumpPoolStructure(void) +{ +#if defined(PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_POOL) + LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry; + struct list_head *psPoolHead = NULL; + IMG_UINT32 j; + IMG_UINT32 *puiCounter; + + printk("\n"); + /* Empty all pools */ + for (j = 0; j < PHYSMEM_OSMEM_NUM_OF_POOLS; j++) + { + + printk("pool = %u\n", j); + + /* Get the correct list for this caching mode */ + if (!_GetPoolListHead(g_aui32CPUCacheFlags[j], &psPoolHead, &puiCounter)) + { + break; + } + + list_for_each_entry_safe(psPagePoolEntry, + psTempPoolEntry, + psPoolHead, + sPagePoolItem) + { + printk("%u | ", psPagePoolEntry->uiItemsRemaining); + } + printk("\n"); + } +#endif +} + +/* Free a certain number of pages from the page pool. + * Mainly used in error paths or at deinitialisation to + * empty the whole pool. */ +static PVRSRV_ERROR +_FreePagesFromPoolUnlocked(IMG_UINT32 uiMaxPagesToFree, + IMG_UINT32 *puiPagesFreed) +{ + PVRSRV_ERROR eError = PVRSRV_OK; + LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry; + struct list_head *psPoolHead = NULL; + IMG_UINT32 i, j; + IMG_UINT32 *puiCounter; + + *puiPagesFreed = uiMaxPagesToFree; + + /* Empty all pools */ + for (j = 0; j < PHYSMEM_OSMEM_NUM_OF_POOLS; j++) + { + + /* Get the correct list for this caching mode */ + if (!_GetPoolListHead(g_aui32CPUCacheFlags[j], &psPoolHead, &puiCounter)) + { + break; + } + + /* Free the pages and remove page arrays from the pool if they are exhausted */ + list_for_each_entry_safe(psPagePoolEntry, + psTempPoolEntry, + psPoolHead, + sPagePoolItem) + { + IMG_UINT32 uiItemsToFree; + struct page **ppsPageArray; + + /* Check if we are going to free the whole page array or just parts */ + if (psPagePoolEntry->uiItemsRemaining <= uiMaxPagesToFree) + { + uiItemsToFree = psPagePoolEntry->uiItemsRemaining; + ppsPageArray = psPagePoolEntry->ppsPageArray; + } + else + { + uiItemsToFree = uiMaxPagesToFree; + ppsPageArray = &(psPagePoolEntry->ppsPageArray[psPagePoolEntry->uiItemsRemaining - uiItemsToFree]); + } + +#if defined(CONFIG_X86) + /* Set the correct page caching attributes on x86 */ + if (!PVRSRV_CHECK_CPU_CACHED(g_aui32CPUCacheFlags[j])) + { + int ret; + ret = set_pages_array_wb(ppsPageArray, uiItemsToFree); + if (ret) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Failed to reset page attributes", + __func__)); + eError = PVRSRV_ERROR_FAILED_TO_FREE_PAGES; + goto e_exit; + } + } +#endif + + /* Free the actual pages */ + for (i = 0; i < uiItemsToFree; i++) + { + __free_pages(ppsPageArray[i], 0); + ppsPageArray[i] = NULL; + } + + /* Reduce counters */ + uiMaxPagesToFree -= uiItemsToFree; + *puiCounter -= uiItemsToFree; + psPagePoolEntry->uiItemsRemaining -= uiItemsToFree; + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) + /* + * MemStats usually relies on having the bridge lock held, however + * the page pool code may call PVRSRVStatsIncrMemAllocPoolStat and + * PVRSRVStatsDecrMemAllocPoolStat without the bridge lock held, so + * the page pool lock is used to ensure these calls are mutually + * exclusive + */ + PVRSRVStatsDecrMemAllocPoolStat(PAGE_SIZE * uiItemsToFree); +#endif + + /* Is this pool entry exhausted, delete it */ + if (psPagePoolEntry->uiItemsRemaining == 0) + { + OSFreeMemNoStats(psPagePoolEntry->ppsPageArray); + list_del(&psPagePoolEntry->sPagePoolItem); + kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry); + } + + /* Return if we have all our pages */ + if (uiMaxPagesToFree == 0) + { + goto e_exit; + } + } + } + +e_exit: + *puiPagesFreed -= uiMaxPagesToFree; + _DumpPoolStructure(); + return eError; +} + +/* Get a certain number of pages from the page pool and + * copy them directly into a given page array. */ +static void +_GetPagesFromPoolUnlocked(IMG_UINT32 ui32CPUCacheFlags, + IMG_UINT32 uiMaxNumPages, + struct page **ppsPageArray, + IMG_UINT32 *puiNumReceivedPages) +{ + LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry; + struct list_head *psPoolHead = NULL; + IMG_UINT32 i; + IMG_UINT32 *puiCounter; + + *puiNumReceivedPages = 0; + + /* Get the correct list for this caching mode */ + if (!_GetPoolListHead(ui32CPUCacheFlags, &psPoolHead, &puiCounter)) + { + return; + } + + /* Check if there are actually items in the list */ + if (list_empty(psPoolHead)) + { + return; + } + + PVR_ASSERT(*puiCounter > 0); + + /* Receive pages from the pool */ + list_for_each_entry_safe(psPagePoolEntry, + psTempPoolEntry, + psPoolHead, + sPagePoolItem) + { + /* Get the pages from this pool entry */ + for (i = psPagePoolEntry->uiItemsRemaining; i != 0 && *puiNumReceivedPages < uiMaxNumPages; i--) + { + ppsPageArray[*puiNumReceivedPages] = psPagePoolEntry->ppsPageArray[i-1]; + (*puiNumReceivedPages)++; + psPagePoolEntry->uiItemsRemaining--; + } + + /* Is this pool entry exhausted, delete it */ + if (psPagePoolEntry->uiItemsRemaining == 0) + { + OSFreeMemNoStats(psPagePoolEntry->ppsPageArray); + list_del(&psPagePoolEntry->sPagePoolItem); + kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry); + } + + /* Return if we have all our pages */ + if (*puiNumReceivedPages == uiMaxNumPages) + { + goto exit_ok; + } + } + +exit_ok: + + /* Update counters */ + *puiCounter -= *puiNumReceivedPages; + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) + /* MemStats usually relies on having the bridge lock held, however + * the page pool code may call PVRSRVStatsIncrMemAllocPoolStat and + * PVRSRVStatsDecrMemAllocPoolStat without the bridge lock held, so + * the page pool lock is used to ensure these calls are mutually + * exclusive + */ + PVRSRVStatsDecrMemAllocPoolStat(PAGE_SIZE * (*puiNumReceivedPages)); +#endif + + _DumpPoolStructure(); + return; +} + +/* Same as _GetPagesFromPoolUnlocked but handles locking and + * checks first whether pages from the pool are a valid option. */ +static inline void +_GetPagesFromPoolLocked(PVRSRV_DEVICE_NODE *psDevNode, + IMG_UINT32 ui32CPUCacheFlags, + IMG_UINT32 uiPagesToAlloc, + IMG_UINT32 uiOrder, + IMG_BOOL bZero, + struct page **ppsPageArray, + IMG_UINT32 *puiPagesFromPool) +{ +#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) + PVR_UNREFERENCED_PARAMETER(bZero); +#else + /* Don't get pages from pool if it doesn't provide zeroed pages */ + if (bZero) + { + return; + } +#endif + + /* The page pool stores only order 0 pages. If we need zeroed memory we + * directly allocate from the OS because it is faster than + * doing it within the driver. */ + if (uiOrder == 0 && + !PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags)) + { + + _PagePoolLock(); + _GetPagesFromPoolUnlocked(ui32CPUCacheFlags, + uiPagesToAlloc, + ppsPageArray, + puiPagesFromPool); + _PagePoolUnlock(); + } + + return; +} + +/* Takes a page array and maps it into the kernel to write zeros */ +static PVRSRV_ERROR +_MemsetPageArray(IMG_UINT32 uiNumToClean, + struct page **ppsCleanArray, + pgprot_t pgprot, + IMG_UINT8 ui8Pattern, int rv_cache) +{ + IMG_CPU_VIRTADDR pvAddr; + IMG_UINT32 uiMaxPagesToMap = MIN(PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES, + uiNumToClean); + + /* Map and fill the pages with zeros. + * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE + * at a time. */ + while (uiNumToClean != 0) + { + IMG_UINT32 uiToClean = MIN(uiNumToClean, uiMaxPagesToMap); + + if (rv_cache) { + pvAddr = pvr_vmap_cached(ppsCleanArray, uiToClean, VM_WRITE, pgprot); + } else { + pvAddr = pvr_vmap(ppsCleanArray, uiToClean, VM_WRITE, pgprot); + } + if (!pvAddr) + { + if (uiMaxPagesToMap <= 1) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Out of vmalloc memory, unable to map pages for %s.", + __func__, + ui8Pattern == PVRSRV_ZERO_VALUE ? "zeroing" : "poisoning")); + return PVRSRV_ERROR_OUT_OF_MEMORY; + } + else + { + /* Halve the pages to map at once and try again. */ + uiMaxPagesToMap = uiMaxPagesToMap >> 1; + continue; + } + } + + if (pgprot_val(pgprot) == pgprot_val(pgprot_noncached(PAGE_KERNEL))) + { + /* this is most likely unnecessary as all pages must be 8-bytes + * aligned so there unaligned access is impossible */ + OSDeviceMemSet(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean); + } + else if (pgprot_val(pgprot) == pgprot_val(pgprot_writecombine(PAGE_KERNEL))) + { + OSCachedMemSetWMB(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean); + } + else + { + OSCachedMemSet(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean); + } + pvr_vunmap(pvAddr, uiToClean, pgprot); + + ppsCleanArray = &(ppsCleanArray[uiToClean]); + uiNumToClean -= uiToClean; + } + + return PVRSRV_OK; +} + +static PVRSRV_ERROR +_CleanupThread_CleanPages(void *pvData) +{ + LinuxCleanupData *psCleanupData = (LinuxCleanupData*) pvData; + LinuxPagePoolEntry *psPagePoolEntry = psCleanupData->psPoolEntry; + struct list_head *psPoolHead = NULL; + IMG_UINT32 *puiCounter = NULL; +#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) + PVRSRV_ERROR eError; + pgprot_t pgprot; + IMG_UINT32 i; +#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */ + + /* Get the correct pool for this caching mode. */ + _GetPoolListHead(psCleanupData->ui32CPUCacheMode , &psPoolHead, &puiCounter); + +#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) + switch (PVRSRV_CPU_CACHE_MODE(psCleanupData->ui32CPUCacheMode)) + { + case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED: +#if defined(CONFIG_X86) + /* For x86 we can only map with the same attributes + * as in the PAT settings*/ + pgprot = pgprot_noncached(PAGE_KERNEL); + break; +#endif + + case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC: + pgprot = pgprot_writecombine(PAGE_KERNEL); + break; + + default: + PVR_DPF((PVR_DBG_ERROR, + "%s: Unknown caching mode to set page protection flags.", + __func__)); + eError = PVRSRV_ERROR_INVALID_PARAMS; + goto eExit; + } + + /* Map and fill the pages with zeros. + * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE + * at a time. */ + eError = _MemsetPageArray(psPagePoolEntry->uiItemsRemaining, + psPagePoolEntry->ppsPageArray, + pgprot, PVRSRV_ZERO_VALUE, 0); + if (eError != PVRSRV_OK) + { + goto eExit; + } +#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */ + + /* Lock down pool and add item */ + _PagePoolLock(); + + /* Pool counters were already updated so don't do it here again*/ + + /* The pages are all zeroed so return them to the pool. */ + list_add_tail(&psPagePoolEntry->sPagePoolItem, psPoolHead); + + _DumpPoolStructure(); + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) + /* Calling PVRSRVStatsIncrMemAllocPoolStat and PVRSRVStatsDecrMemAllocPoolStat + * inside page pool lock ensures that the stat reflects the state of the pool. */ + PVRSRVStatsIncrMemAllocPoolStat(PAGE_SIZE * psPagePoolEntry->uiItemsRemaining); +#endif + + _PagePoolUnlock(); + + OSFreeMem(pvData); + OSAtomicDecrement(&g_iPoolCleanTasks); + + return PVRSRV_OK; + +#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) +eExit: + /* we failed to zero the pages so return the error so we can + * retry during the next spin */ + if ((psCleanupData->sCleanupWork.ui32RetryCount - 1) > 0) + { + return eError; + } + + /* this was the last retry, give up and free pages to OS */ + PVR_DPF((PVR_DBG_ERROR, + "%s: Deferred task error, freeing pages to OS.", + __func__)); + _PagePoolLock(); + + *puiCounter -= psPagePoolEntry->uiItemsRemaining; + + _PagePoolUnlock(); + + for (i = 0; i < psCleanupData->psPoolEntry->uiItemsRemaining; i++) + { + _FreeOSPage(0, IMG_TRUE, psPagePoolEntry->ppsPageArray[i]); + } + OSFreeMemNoStats(psPagePoolEntry->ppsPageArray); + kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry); + OSFreeMem(psCleanupData); + + OSAtomicDecrement(&g_iPoolCleanTasks); + + return PVRSRV_OK; +#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */ +} + + +/* Put page array to the page pool. + * Handles locking and checks whether the pages are + * suitable to be stored in the pool. */ +static inline IMG_BOOL +_PutPagesToPoolLocked(IMG_UINT32 ui32CPUCacheFlags, + struct page **ppsPageArray, + IMG_BOOL bUnpinned, + IMG_UINT32 uiOrder, + IMG_UINT32 uiNumPages) +{ + LinuxCleanupData *psCleanupData; + PVRSRV_CLEANUP_THREAD_WORK *psCleanupThreadFn; +#if defined(SUPPORT_PHYSMEM_TEST) + PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData(); +#endif + + if (uiOrder == 0 && + !bUnpinned && + !PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags)) + { + IMG_UINT32 uiEntries; + IMG_UINT32 *puiCounter; + struct list_head *psPoolHead; + + + _PagePoolLock(); + + uiEntries = _PagesInPoolUnlocked(); + + /* Check for number of current page pool entries and whether + * we have other asynchronous tasks in-flight */ + if ( (uiEntries < g_ui32PagePoolMaxEntries) && + ((uiEntries + uiNumPages) < + (g_ui32PagePoolMaxEntries + g_ui32PagePoolMaxExcessEntries) )) + { + if (OSAtomicIncrement(&g_iPoolCleanTasks) <= + PVR_LINUX_PHYSMEM_MAX_ASYNC_CLEAN_TASKS) + { +#if defined(SUPPORT_PHYSMEM_TEST) + if (!psPVRSRVData->hCleanupThread) + { + goto eDecrement; + } +#endif + + psCleanupData = OSAllocMem(sizeof(*psCleanupData)); + + if (!psCleanupData) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Failed to get memory for deferred page pool cleanup. " + "Trying to free pages immediately", + __func__)); + goto eDecrement; + } + + psCleanupThreadFn = &psCleanupData->sCleanupWork; + psCleanupData->ui32CPUCacheMode = ui32CPUCacheFlags; + psCleanupData->psPoolEntry = kmem_cache_alloc(g_psLinuxPagePoolCache, GFP_KERNEL); + + if (!psCleanupData->psPoolEntry) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Failed to get memory for deferred page pool cleanup. " + "Trying to free pages immediately", + __func__)); + goto eFreeCleanupData; + } + + if (!_GetPoolListHead(ui32CPUCacheFlags, &psPoolHead, &puiCounter)) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Failed to get correct page pool", + __func__)); + goto eFreePoolEntry; + } + + /* Increase counter here to avoid deferred cleanup tasks piling up */ + *puiCounter = *puiCounter + uiNumPages; + + psCleanupData->psPoolEntry->ppsPageArray = ppsPageArray; + psCleanupData->psPoolEntry->uiItemsRemaining = uiNumPages; + + psCleanupThreadFn->pfnFree = _CleanupThread_CleanPages; + psCleanupThreadFn->pvData = psCleanupData; + psCleanupThreadFn->bDependsOnHW = IMG_FALSE; + CLEANUP_THREAD_SET_RETRY_COUNT(psCleanupThreadFn, + CLEANUP_THREAD_RETRY_COUNT_DEFAULT); + + /* We must not hold the pool lock when calling AddWork because it might call us back to + * free pooled pages directly when unloading the driver */ + _PagePoolUnlock(); + + PVRSRVCleanupThreadAddWork(psCleanupThreadFn); + + + } + else + { + goto eDecrement; + } + + } + else + { + goto eUnlock; + } + } + else + { + goto eExitFalse; + } + + return IMG_TRUE; + +eFreePoolEntry: + OSFreeMem(psCleanupData->psPoolEntry); +eFreeCleanupData: + OSFreeMem(psCleanupData); +eDecrement: + OSAtomicDecrement(&g_iPoolCleanTasks); +eUnlock: + _PagePoolUnlock(); +eExitFalse: + return IMG_FALSE; +} + +/* Get the GFP flags that we pass to the page allocator */ +static inline gfp_t +_GetGFPFlags(IMG_BOOL bZero, + PVRSRV_DEVICE_NODE *psDevNode) +{ + struct device *psDev = psDevNode->psDevConfig->pvOSDevice; + gfp_t gfp_flags = GFP_USER | __GFP_NOWARN | __GFP_NOMEMALLOC; + +#if defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY) + /* Force use of HIGHMEM */ + gfp_flags |= __GFP_HIGHMEM; + + PVR_UNREFERENCED_PARAMETER(psDev); +#else + if (psDev) + { +#if defined(CONFIG_64BIT) || defined(CONFIG_ARM_LPAE) || defined(CONFIG_X86_PAE) + if (*psDev->dma_mask > DMA_BIT_MASK(32)) + { + /* If our system is able to handle large addresses use highmem */ + gfp_flags |= __GFP_HIGHMEM; + } + else if (*psDev->dma_mask == DMA_BIT_MASK(32)) + { + /* Limit to 32 bit. + * Achieved by setting __GFP_DMA32 for 64 bit systems */ + gfp_flags |= __GFP_DMA32; + } + else + { + /* Limit to size of DMA zone. */ + gfp_flags |= __GFP_DMA; + } +#else + if (*psDev->dma_mask < DMA_BIT_MASK(32)) + { + gfp_flags |= __GFP_DMA; + } + else + { + gfp_flags |= __GFP_HIGHMEM; + } +#endif /* if defined(CONFIG_64BIT) || defined(CONFIG_ARM_LPAE) || defined(CONFIG_X86_PAE) */ + } + +#endif /* if defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY) */ + + if (bZero) + { + gfp_flags |= __GFP_ZERO; + } + + return gfp_flags; +} + +/* + * @Function _PoisonDevicePage + * + * @Description Poisons a device page. In normal case the device page has the + * same size as the OS page and so the ui32DevPageOrder will be + * equal to 0 and page argument will point to one OS page + * structure. In case of Non4K pages the order will be greater + * than 0 and page argument will point to an array of OS + * allocated pages. + * + * @Input psDevNode pointer to the device object + * @Input page array of the pages allocated by from the OS + * @Input ui32DevPageOrder order of the page (same as the one used to allocate + * the page array by alloc_pages()) + * @Input ui32CPUCacheFlags CPU cache flags applied to the page + * @Input ui8PoisonValue value used to poison the page + */ +static void +_PoisonDevicePage(PVRSRV_DEVICE_NODE *psDevNode, + struct page *page, + IMG_UINT32 ui32DevPageOrder, + IMG_UINT32 ui32CPUCacheFlags, + IMG_BYTE ui8PoisonValue) +{ + IMG_UINT32 ui32OsPageIdx; + + for (ui32OsPageIdx = 0; + ui32OsPageIdx < (1U << ui32DevPageOrder); + ui32OsPageIdx++) + { + struct page *current_page = page + ui32OsPageIdx; + IMG_CPU_PHYADDR sCPUPhysAddrStart = {page_to_phys(current_page)}; + IMG_CPU_PHYADDR sCPUPhysAddrEnd = {sCPUPhysAddrStart.uiAddr + PAGE_SIZE}; + + void *kvaddr = kmap_atomic(current_page); + + /* kmap_atomic maps pages as cached so it's safe to use OSCachedMemSet + * here (also pages are always 8 bytes aligned anyway) */ + OSCachedMemSet(kvaddr, ui8PoisonValue, PAGE_SIZE); + + OSCPUCacheFlushRangeKM(psDevNode, kvaddr, kvaddr + PAGE_SIZE, + sCPUPhysAddrStart, sCPUPhysAddrEnd); + + kunmap_atomic(kvaddr); + } +} + +/* Allocate and initialise the structure to hold the metadata of the allocation */ +static PVRSRV_ERROR +_AllocOSPageArray(PVRSRV_DEVICE_NODE *psDevNode, + PMR_SIZE_T uiChunkSize, + IMG_UINT32 ui32NumPhysChunks, + IMG_UINT32 ui32NumVirtChunks, + IMG_UINT32 uiLog2AllocPageSize, + IMG_UINT32 ui32AllocFlags, + IMG_UINT32 ui32CPUCacheFlags, + IMG_PID uiPid, + PMR_OSPAGEARRAY_DATA **ppsPageArrayDataPtr) +{ + PVRSRV_ERROR eError; + PMR_SIZE_T uiSize = uiChunkSize * ui32NumVirtChunks; + IMG_UINT32 uiNumOSPageSizeVirtPages; + IMG_UINT32 uiNumDevPageSizeVirtPages; + PMR_OSPAGEARRAY_DATA *psPageArrayData; + IMG_UINT64 ui64DmaMask = 0; + PVR_UNREFERENCED_PARAMETER(ui32NumPhysChunks); + + /* Use of cast below is justified by the assertion that follows to + * prove that no significant bits have been truncated */ + uiNumOSPageSizeVirtPages = (IMG_UINT32) (((uiSize - 1) >> PAGE_SHIFT) + 1); + PVR_ASSERT(((PMR_SIZE_T) uiNumOSPageSizeVirtPages << PAGE_SHIFT) == uiSize); + + uiNumDevPageSizeVirtPages = uiNumOSPageSizeVirtPages >> (uiLog2AllocPageSize - PAGE_SHIFT); + + /* Allocate the struct to hold the metadata */ + psPageArrayData = kmem_cache_alloc(g_psLinuxPageArray, GFP_KERNEL); + if (psPageArrayData == NULL) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: OS refused the memory allocation for the private data.", + __func__)); + eError = PVRSRV_ERROR_OUT_OF_MEMORY; + goto e_freed_none; + } + + /* + * Allocate the page array + * + * We avoid tracking this memory because this structure might go into the page pool. + * The OS can drain the pool asynchronously and when doing that we have to avoid + * any potential deadlocks. + * + * In one scenario the process stats vmalloc hash table lock is held and then + * the oom-killer softirq is trying to call _ScanObjectsInPagePool(), it must not + * try to acquire the vmalloc hash table lock again. + */ + psPageArrayData->pagearray = OSAllocZMemNoStats(sizeof(struct page *) * uiNumDevPageSizeVirtPages); + if (psPageArrayData->pagearray == NULL) + { + eError = PVRSRV_ERROR_OUT_OF_MEMORY; + goto e_free_kmem_cache; + } + else + { + if (BIT_ISSET(ui32AllocFlags, FLAG_IS_CMA)) + { + /* Allocate additional DMA/CMA cpu kernel virtual address & device bus address array state */ + psPageArrayData->dmavirtarray = OSAllocZMemNoStats(sizeof(void*) * uiNumDevPageSizeVirtPages); + if (psPageArrayData->dmavirtarray == NULL) + { + eError = PVRSRV_ERROR_OUT_OF_MEMORY; + goto e_free_pagearray; + } + + psPageArrayData->dmaphysarray = OSAllocZMemNoStats(sizeof(dma_addr_t) * uiNumDevPageSizeVirtPages); + if (psPageArrayData->dmaphysarray == NULL) + { + eError = PVRSRV_ERROR_OUT_OF_MEMORY; + goto e_free_cpuvirtaddrarray; + } + } + } + + if (psDevNode->psDevConfig && psDevNode->psDevConfig->pvOSDevice) + { + struct device *psDev = psDevNode->psDevConfig->pvOSDevice; + ui64DmaMask = *psDev->dma_mask; + } + + /* Init metadata */ + psPageArrayData->psDevNode = psDevNode; + psPageArrayData->uiPid = uiPid; + psPageArrayData->iNumOSPagesAllocated = 0; + psPageArrayData->uiTotalNumOSPages = uiNumOSPageSizeVirtPages; + psPageArrayData->uiLog2AllocPageSize = uiLog2AllocPageSize; + psPageArrayData->ui64DmaMask = ui64DmaMask; + psPageArrayData->ui32AllocFlags = ui32AllocFlags; + psPageArrayData->ui32CPUCacheFlags = ui32CPUCacheFlags; + psPageArrayData->ui32CMAAdjustedPageCount = 0; + + *ppsPageArrayDataPtr = psPageArrayData; + return PVRSRV_OK; + +/* Error path */ +e_free_cpuvirtaddrarray: + OSFreeMemNoStats(psPageArrayData->dmavirtarray); + +e_free_pagearray: + OSFreeMemNoStats(psPageArrayData->pagearray); + +e_free_kmem_cache: + kmem_cache_free(g_psLinuxPageArray, psPageArrayData); + PVR_DPF((PVR_DBG_ERROR, + "%s: OS refused the memory allocation for the page pointer table. " + "Did you ask for too much?", + __func__)); + +e_freed_none: + PVR_ASSERT(eError != PVRSRV_OK); + return eError; +} + +static inline void +_ApplyCacheMaintenance(PVRSRV_DEVICE_NODE *psDevNode, + struct page **ppsPage, + IMG_UINT32 uiNumPages) +{ + void * pvAddr; + + if (OSCPUCacheOpAddressType() == OS_CACHE_OP_ADDR_TYPE_VIRTUAL) + { + pgprot_t pgprot = PAGE_KERNEL; + + IMG_UINT32 uiNumToClean = uiNumPages; + struct page **ppsCleanArray = ppsPage; + + /* Map and flush page. + * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE + * at a time. */ + while (uiNumToClean != 0) + { + IMG_UINT32 uiToClean = MIN(PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES, + uiNumToClean); + IMG_CPU_PHYADDR sUnused = + { IMG_CAST_TO_CPUPHYADDR_UINT(0xCAFEF00DDEADBEEFULL) }; + + pvAddr = pvr_vmap(ppsCleanArray, uiToClean, -1, pgprot); + if (!pvAddr) + { + PVR_DPF((PVR_DBG_ERROR, + "Unable to flush page cache for new allocation, skipping flush.")); + return; + } + + CacheOpExec(psDevNode, + pvAddr, + pvAddr + PAGE_SIZE, + sUnused, + sUnused, + PVRSRV_CACHE_OP_FLUSH); + + pvr_vunmap(pvAddr, uiToClean, pgprot); + ppsCleanArray = &(ppsCleanArray[uiToClean]); + uiNumToClean -= uiToClean; + } + } + else + { + IMG_UINT32 ui32Idx; + + for (ui32Idx = 0; ui32Idx < uiNumPages; ++ui32Idx) + { + IMG_CPU_PHYADDR sCPUPhysAddrStart, sCPUPhysAddrEnd; + + pvAddr = kmap(ppsPage[ui32Idx]); + sCPUPhysAddrStart.uiAddr = page_to_phys(ppsPage[ui32Idx]); + sCPUPhysAddrEnd.uiAddr = sCPUPhysAddrStart.uiAddr + PAGE_SIZE; + + /* If we're zeroing, we need to make sure the cleared memory is pushed out + * of the cache before the cache lines are invalidated */ + CacheOpExec(psDevNode, + pvAddr, + pvAddr + PAGE_SIZE, + sCPUPhysAddrStart, + sCPUPhysAddrEnd, + PVRSRV_CACHE_OP_FLUSH); + + kunmap(ppsPage[ui32Idx]); + } + } +} + +/* Change the caching attribute of pages on x86 systems and takes care of + * cache maintenance. This function is supposed to be called once for pages that + * came from alloc_pages(). It expects an array of OS page sized pages! + * + * Flush/Invalidate pages in case the allocation is not cached. Necessary to + * remove pages from the cache that might be flushed later and corrupt memory. */ +static inline PVRSRV_ERROR +_ApplyOSPagesAttribute(PVRSRV_DEVICE_NODE *psDevNode, + struct page **ppsPage, + IMG_UINT32 uiNumPages, + IMG_BOOL bFlush, + IMG_UINT32 ui32CPUCacheFlags) +{ + PVRSRV_ERROR eError = PVRSRV_OK; + IMG_BOOL bCPUCached = PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags); + IMG_BOOL bCPUUncached = PVRSRV_CHECK_CPU_UNCACHED(ui32CPUCacheFlags); + IMG_BOOL bCPUWriteCombine = PVRSRV_CHECK_CPU_WRITE_COMBINE(ui32CPUCacheFlags); + + if (ppsPage != NULL && uiNumPages != 0) + { +#if defined(CONFIG_X86) + /* On x86 we have to set page cache attributes for non-cached pages. + * The call is implicitly taking care of all flushing/invalidating + * and therefore we can skip the usual cache maintenance after this. */ + if (bCPUUncached || bCPUWriteCombine) + { + /* On x86 if we already have a mapping (e.g. low memory) we need to change the mode of + current mapping before we map it ourselves */ + int ret = IMG_FALSE; + + switch (PVRSRV_CPU_CACHE_MODE(ui32CPUCacheFlags)) + { + case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED: + ret = set_pages_array_uc(ppsPage, uiNumPages); + if (ret) + { + eError = PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE; + PVR_DPF((PVR_DBG_ERROR, "Setting Linux page caching mode to UC failed, returned %d", ret)); + } + break; + + case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC: + ret = set_pages_array_wc(ppsPage, uiNumPages); + if (ret) + { + eError = PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE; + PVR_DPF((PVR_DBG_ERROR, "Setting Linux page caching mode to WC failed, returned %d", ret)); + } + break; + + case PVRSRV_MEMALLOCFLAG_CPU_CACHED: + break; + + default: + break; + } + } + else +#endif + { + if ( bFlush || + bCPUUncached || bCPUWriteCombine || + (bCPUCached && PVRSRV_CHECK_CPU_CACHE_CLEAN(ui32CPUCacheFlags)) ) + { + /* We can be given pages which still remain in the cache. + In order to make sure that the data we write through our mappings + doesn't get overwritten by later cache evictions we invalidate the + pages that are given to us. + + Note: + This still seems to be true if we request cold pages, it's just less + likely to be in the cache. */ + _ApplyCacheMaintenance(psDevNode, + ppsPage, + uiNumPages); + } + } + } + + return eError; +} + +/* Same as _AllocOSPage except it uses DMA framework to perform allocation. + * uiPageIndex is expected to be the pagearray index where to store the higher order page. */ +static PVRSRV_ERROR +_AllocOSPage_CMA(PMR_OSPAGEARRAY_DATA *psPageArrayData, + gfp_t gfp_flags, + IMG_UINT32 ui32AllocOrder, + IMG_UINT32 ui32MinOrder, + IMG_UINT32 uiPageIndex) +{ + void *virt_addr; + struct page *page; + dma_addr_t bus_addr; + IMG_UINT32 uiAllocIsMisaligned; + size_t alloc_size = PAGE_SIZE << ui32AllocOrder; + struct device *dev = psPageArrayData->psDevNode->psDevConfig->pvOSDevice; + PVR_ASSERT(ui32AllocOrder == ui32MinOrder); + + do + { + DisableOOMKiller(); +#if defined(PVR_LINUX_PHYSMEM_SUPPRESS_DMA_AC) + virt_addr = NULL; +#else + virt_addr = dma_alloc_coherent(dev, alloc_size, &bus_addr, gfp_flags); +#endif + if (virt_addr == NULL) + { + /* The idea here is primarily to support some older kernels with + broken or non-functioning DMA/CMA implementations (< Linux-3.4) + and to also handle DMA/CMA allocation failures by attempting a + normal page allocation though we expect dma_alloc_coherent() + already attempts this internally also before failing but + nonetheless it does no harm to retry the allocation ourselves */ + page = alloc_pages(gfp_flags, ui32AllocOrder); + if (page) + { + /* Taint bus_addr as alloc_page, needed when freeing; + also acquire the low memory page address only, this + prevents mapping possible high memory pages into + kernel virtual address space which might exhaust + the VMALLOC address space */ + bus_addr = DMA_SET_ALLOCPG_ADDR(page_to_phys(page)); + virt_addr = (void*)(uintptr_t) DMA_VADDR_NOT_IN_USE; + } + else + { + EnableOOMKiller(); + return PVRSRV_ERROR_OUT_OF_MEMORY; + } + } + else + { +#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64) + page = pfn_to_page(bus_addr >> PAGE_SHIFT); +#else + /* Assumes bus address space is identical to physical address space */ + page = phys_to_page(bus_addr); +#endif + } + EnableOOMKiller(); + + /* Physical allocation alignment works/hidden behind the scene transparently, + we do this here if the allocated buffer address does not meet its alignment + requirement by over-allocating using the next power-2 order and reporting + aligned-adjusted values back to meet the requested alignment constraint. + Evidently we waste memory by doing this so should only do so if we do not + initially meet the alignment constraint. */ + uiAllocIsMisaligned = DMA_GET_ADDR(bus_addr) & ((PAGE_SIZE<<ui32MinOrder)-1); + if (uiAllocIsMisaligned || ui32AllocOrder > ui32MinOrder) + { + IMG_BOOL bUsedAllocPages = DMA_IS_ALLOCPG_ADDR(bus_addr); + if (ui32AllocOrder == ui32MinOrder) + { + if (bUsedAllocPages) + { + __free_pages(page, ui32AllocOrder); + } + else + { + dma_free_coherent(dev, alloc_size, virt_addr, bus_addr); + } + + ui32AllocOrder = ui32AllocOrder + 1; + alloc_size = PAGE_SIZE << ui32AllocOrder; + + PVR_ASSERT(uiAllocIsMisaligned != 0); + } + else + { + size_t align_adjust = PAGE_SIZE << ui32MinOrder; + + /* Adjust virtual/bus addresses to meet alignment */ + bus_addr = bUsedAllocPages ? page_to_phys(page) : bus_addr; + align_adjust = PVR_ALIGN((size_t)bus_addr, align_adjust); + align_adjust -= (size_t)bus_addr; + + if (align_adjust) + { + if (bUsedAllocPages) + { + page += align_adjust >> PAGE_SHIFT; + bus_addr = DMA_SET_ALLOCPG_ADDR(page_to_phys(page)); + virt_addr = (void*)(uintptr_t) DMA_VADDR_NOT_IN_USE; + } + else + { + bus_addr += align_adjust; + virt_addr += align_adjust; +#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64) + page = pfn_to_page(bus_addr >> PAGE_SHIFT); +#else + /* Assumes bus address space is identical to physical address space */ + page = phys_to_page(bus_addr); +#endif + } + + /* Store adjustments in PAGE_SIZE counts */ + align_adjust = align_adjust >> PAGE_SHIFT; + bus_addr = DMA_SET_ALIGN_ADJUSTMENT(bus_addr, align_adjust); + } + + /* Taint bus_addr due to over-allocation, allows us to free + * memory correctly */ + bus_addr = DMA_SET_ADJUSTED_ADDR(bus_addr); + uiAllocIsMisaligned = 0; + } + } + } while (uiAllocIsMisaligned); + + /* Convert OSPageSize-based index into DevicePageSize-based index */ + psPageArrayData->ui32CMAAdjustedPageCount += (alloc_size - (PAGE_SIZE << ui32AllocOrder )); + + psPageArrayData->dmavirtarray[uiPageIndex] = virt_addr; + psPageArrayData->dmaphysarray[uiPageIndex] = bus_addr; + psPageArrayData->pagearray[uiPageIndex] = page; + + return PVRSRV_OK; +} + +/* Allocate a page of order uiAllocOrder and stores it in the page array ppsPage at + * position uiPageIndex. + * + * If the order is higher than 0, it splits the page into multiples and + * stores them at position uiPageIndex to uiPageIndex+(1<<uiAllocOrder). + * + * This function is supposed to be used for uiMinOrder == 0 only! */ +static PVRSRV_ERROR +_AllocOSPage(PMR_OSPAGEARRAY_DATA *psPageArrayData, + gfp_t gfp_flags, + IMG_UINT32 uiAllocOrder, + IMG_UINT32 uiMinOrder, + IMG_UINT32 uiPageIndex) +{ + struct page *psPage; + IMG_UINT32 ui32Count; + + /* Parameter check. If it fails we write into the wrong places in the array. */ + PVR_ASSERT(uiMinOrder == 0); + + /* Allocate the page */ + DisableOOMKiller(); + psPage = alloc_pages(gfp_flags, uiAllocOrder); + EnableOOMKiller(); + + if (psPage == NULL) + { + return PVRSRV_ERROR_OUT_OF_MEMORY; + } + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)) + /* In case we need to, split the higher order page; + this should only be used for order-0 allocations + as higher order allocations should use DMA/CMA */ + if (uiAllocOrder != 0) + { + split_page(psPage, uiAllocOrder); + } +#endif + + /* Store the page (or multiple split pages) in the page array */ + for (ui32Count = 0; ui32Count < (1 << uiAllocOrder); ui32Count++) + { + psPageArrayData->pagearray[uiPageIndex + ui32Count] = &(psPage[ui32Count]); + } + + return PVRSRV_OK; +} + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) +#if defined(PVRSRV_ENABLE_MEMORY_STATS) + +static inline void _AddMemAllocRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData, + struct page *psPage) +{ + IMG_CPU_PHYADDR sCPUPhysAddr = { page_to_phys(psPage) }; + PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES, + NULL, sCPUPhysAddr, + 1 << psPageArrayData->uiLog2AllocPageSize, + NULL, psPageArrayData->uiPid + DEBUG_MEMSTATS_VALUES); +} + +static inline void _RemoveMemAllocRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData, + struct page *psPage) +{ + PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES, + (IMG_UINT64) page_to_phys(psPage), + psPageArrayData->uiPid); +} + +#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */ + +static inline void _IncrMemAllocStat_UmaPages(size_t uiSize, IMG_PID uiPid) +{ + PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES, + uiSize, uiPid); +} + +static inline void _DecrMemAllocStat_UmaPages(size_t uiSize, IMG_PID uiPid) +{ + PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES, + uiSize, uiPid); +} + +#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */ +#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */ + +/* Allocation of OS pages: We may allocate 2^N order pages at a time for two reasons. + * + * Firstly to support device pages which are larger than OS. By asking the OS for 2^N + * order OS pages at a time we guarantee the device page is contiguous. + * + * Secondly for performance where we may ask for 2^N order pages to reduce the number + * of calls to alloc_pages, and thus reduce time for huge allocations. + * + * Regardless of page order requested, we need to break them down to track _OS pages. + * The maximum order requested is increased if all max order allocations were successful. + * If any request fails we reduce the max order. + */ +static PVRSRV_ERROR +_AllocOSPages_Fast(PMR_OSPAGEARRAY_DATA *psPageArrayData) +{ + PVRSRV_ERROR eError; + IMG_UINT32 uiArrayIndex = 0; + IMG_UINT32 ui32Order; + IMG_UINT32 ui32MinOrder = psPageArrayData->uiLog2AllocPageSize - PAGE_SHIFT; + IMG_BOOL bIncreaseMaxOrder = IMG_TRUE; + + IMG_UINT32 ui32NumPageReq; + IMG_UINT32 uiOSPagesToAlloc; + IMG_UINT32 uiDevPagesFromPool = 0; + + gfp_t gfp_flags = _GetGFPFlags(ui32MinOrder ? BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_ZERO) : IMG_FALSE, /* Zero all pages later as batch */ + psPageArrayData->psDevNode); + gfp_t ui32GfpFlags; + gfp_t ui32HighOrderGfpFlags = ((gfp_flags & ~__GFP_RECLAIM) | __GFP_NORETRY); + + struct page **ppsPageArray = psPageArrayData->pagearray; + struct page **ppsPageAttributeArray = NULL; + + uiOSPagesToAlloc = psPageArrayData->uiTotalNumOSPages; + + /* Try to get pages from the pool since it is faster; + the page pool currently only supports zero-order pages + thus currently excludes all DMA/CMA allocated memory */ + _GetPagesFromPoolLocked(psPageArrayData->psDevNode, + psPageArrayData->ui32CPUCacheFlags, + uiOSPagesToAlloc, + ui32MinOrder, + BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_ZERO), + ppsPageArray, + &uiDevPagesFromPool); + + uiArrayIndex = uiDevPagesFromPool; + + if ((uiOSPagesToAlloc - uiDevPagesFromPool) < PVR_LINUX_HIGHORDER_ALLOCATION_THRESHOLD) + { /* Small allocations: ask for one device page at a time */ + ui32Order = ui32MinOrder; + bIncreaseMaxOrder = IMG_FALSE; + } + else + { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)) + /* Large zero-order or none zero-order allocations, ask for + MAX(max-order, min-order) order pages at a time; alloc + failures throttles this down to ZeroOrder allocations */ + ui32Order = MAX(g_uiMaxOrder, ui32MinOrder); +#else + /* Because split_page() is not available on older kernels + we cannot mix-and-match any-order pages in the PMR; + only same-order pages must be present in page array. + So we unconditionally force it to use ui32MinOrder on + these older kernels */ + ui32Order = ui32MinOrder; +#if defined(DEBUG) + if (! BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + /* Check that this is zero */ + PVR_ASSERT(! ui32Order); + } +#endif +#endif + } + + /* Only if asking for more contiguity than we actually need, let it fail */ + ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags; + ui32NumPageReq = (1 << ui32Order); + + while (uiArrayIndex < uiOSPagesToAlloc) + { + IMG_UINT32 ui32PageRemain = uiOSPagesToAlloc - uiArrayIndex; + + while (ui32NumPageReq > ui32PageRemain) + { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)) + /* Pages to request is larger than that remaining + so ask for less so never over allocate */ + ui32Order = MAX(ui32Order >> 1, ui32MinOrder); +#else + /* Pages to request is larger than that remaining so + do nothing thus over allocate as we do not support + mix/match of any-order pages in PMR page-array in + older kernels (simplifies page free logic) */ + PVR_ASSERT(ui32Order == ui32MinOrder); +#endif + ui32NumPageReq = (1 << ui32Order); + ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags; + } + + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + /* As the DMA/CMA framework rounds-up request to the + next power-of-two, we request multiple uiMinOrder + pages to satisfy allocation request in order to + minimise wasting memory */ + eError = _AllocOSPage_CMA(psPageArrayData, + ui32GfpFlags, + ui32Order, + ui32MinOrder, + uiArrayIndex >> ui32MinOrder); + } + else + { + /* Allocate uiOrder pages at uiArrayIndex */ + eError = _AllocOSPage(psPageArrayData, + ui32GfpFlags, + ui32Order, + ui32MinOrder, + uiArrayIndex); + } + + if (eError == PVRSRV_OK) + { + /* Successful request. Move onto next. */ + uiArrayIndex += ui32NumPageReq; + } + else + { + if (ui32Order > ui32MinOrder) + { + /* Last request failed. Let's ask for less next time */ + ui32Order = MAX(ui32Order >> 1, ui32MinOrder); + bIncreaseMaxOrder = IMG_FALSE; + ui32NumPageReq = (1 << ui32Order); + ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags; + g_uiMaxOrder = ui32Order; +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0)) + /* We should not trigger this code path in older kernels, + this is enforced by ensuring ui32Order == ui32MinOrder */ + PVR_ASSERT(ui32Order == ui32MinOrder); +#endif + } + else + { + /* Failed to alloc pages at required contiguity. Failed allocation */ + PVR_DPF((PVR_DBG_ERROR, "%s: %s failed to honour request at %u of %u, flags = %x, order = %u (%s)", + __func__, + BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA) ? "dma_alloc_coherent" : "alloc_pages", + uiArrayIndex, + uiOSPagesToAlloc, + ui32GfpFlags, + ui32Order, + PVRSRVGetErrorString(eError))); + eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES; + goto e_free_pages; + } + } + } + + if (bIncreaseMaxOrder && (g_uiMaxOrder < PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM)) + { /* All successful allocations on max order. Let's ask for more next time */ + g_uiMaxOrder++; + } + + /* Construct table of page pointers to apply attributes */ + ppsPageAttributeArray = &ppsPageArray[uiDevPagesFromPool]; + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + IMG_UINT32 uiIdx, uiIdy, uiIdz; + + ppsPageAttributeArray = OSAllocMem(sizeof(struct page *) * uiOSPagesToAlloc); + PVR_LOG_GOTO_IF_NOMEM(ppsPageAttributeArray, eError, e_free_pages); + + for (uiIdx = 0; uiIdx < uiOSPagesToAlloc; uiIdx += ui32NumPageReq) + { + uiIdy = uiIdx >> ui32Order; + for (uiIdz = 0; uiIdz < ui32NumPageReq; uiIdz++) + { + ppsPageAttributeArray[uiIdx+uiIdz] = ppsPageArray[uiIdy]; + ppsPageAttributeArray[uiIdx+uiIdz] += uiIdz; + } + } + } + + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_ZERO) && ui32MinOrder == 0) + { + eError = _MemsetPageArray(uiOSPagesToAlloc - uiDevPagesFromPool, + ppsPageAttributeArray, PAGE_KERNEL, + PVRSRV_ZERO_VALUE, 0); + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, "Failed to zero pages (fast)")); + goto e_free_pages; + } + } + else if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_ALLOC)) + { + /* need to call twice because ppsPageArray and ppsPageAttributeArray + * can point to different allocations: first for pages obtained from + * the pool and then the remaining pages */ + eError = _MemsetPageArray(uiDevPagesFromPool, ppsPageArray, PAGE_KERNEL, + PVRSRV_POISON_ON_ALLOC_VALUE, 0); + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, "Failed to poison pages (fast)")); + } + eError = _MemsetPageArray(uiOSPagesToAlloc - uiDevPagesFromPool, + ppsPageAttributeArray, PAGE_KERNEL, + PVRSRV_POISON_ON_ALLOC_VALUE, 0); + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, "Failed to poison pages (fast)")); + } + + /* for poisoning need to also flush the pool pages as the 0s have + * been overwritten */ + _ApplyCacheMaintenance(psPageArrayData->psDevNode, ppsPageArray, + uiDevPagesFromPool); + } + + /* Do the cache management as required */ + eError = _ApplyOSPagesAttribute(psPageArrayData->psDevNode, + ppsPageAttributeArray, + uiOSPagesToAlloc - uiDevPagesFromPool, + BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_ZERO) || + BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_ALLOC), + psPageArrayData->ui32CPUCacheFlags); + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, "Failed to set page attributes")); + goto e_free_pages; + } + else + { + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + OSFreeMem(ppsPageAttributeArray); + } + } + + /* Update metadata */ + psPageArrayData->iNumOSPagesAllocated = psPageArrayData->uiTotalNumOSPages; + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) + { +#if defined(PVRSRV_ENABLE_MEMORY_STATS) + IMG_UINT32 ui32NumPages = + psPageArrayData->iNumOSPagesAllocated >> ui32MinOrder; + IMG_UINT32 i; + + for (i = 0; i < ui32NumPages; i++) + { + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + _AddMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i]); + } + else + { + _AddMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i << ui32MinOrder]); + } + } +#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */ + _IncrMemAllocStat_UmaPages(((uiOSPagesToAlloc * PAGE_SIZE)+(psPageArrayData->ui32CMAAdjustedPageCount)), + psPageArrayData->uiPid); +#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */ + } +#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */ + + return PVRSRV_OK; + +/* Error path */ +e_free_pages: + { + IMG_UINT32 ui32PageToFree; + + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + IMG_UINT32 uiDevArrayIndex = uiArrayIndex >> ui32Order; + IMG_UINT32 uiDevPageSize = PAGE_SIZE << ui32Order; + PVR_ASSERT(ui32Order == ui32MinOrder); + + if (ppsPageAttributeArray) + { + OSFreeMem(ppsPageAttributeArray); + } + + for (ui32PageToFree = 0; ui32PageToFree < uiDevArrayIndex; ui32PageToFree++) + { + _FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice, + uiDevPageSize, + ui32MinOrder, + psPageArrayData->dmavirtarray[ui32PageToFree], + psPageArrayData->dmaphysarray[ui32PageToFree], + ppsPageArray[ui32PageToFree]); + psPageArrayData->dmaphysarray[ui32PageToFree]= (dma_addr_t)0; + psPageArrayData->dmavirtarray[ui32PageToFree] = NULL; + ppsPageArray[ui32PageToFree] = NULL; + } + } + else + { + /* Free the pages we got from the pool */ + for (ui32PageToFree = 0; ui32PageToFree < uiDevPagesFromPool; ui32PageToFree++) + { + _FreeOSPage(ui32MinOrder, + BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE), + ppsPageArray[ui32PageToFree]); + ppsPageArray[ui32PageToFree] = NULL; + } + + for (ui32PageToFree = uiDevPagesFromPool; ui32PageToFree < uiArrayIndex; ui32PageToFree++) + { + _FreeOSPage(ui32MinOrder, IMG_FALSE, ppsPageArray[ui32PageToFree]); + ppsPageArray[ui32PageToFree] = NULL; + } + } + + return eError; + } +} + +static INLINE PVRSRV_ERROR +_CheckIfIndexInRange(IMG_UINT32 ui32Index, IMG_UINT32 *pui32Indices, IMG_UINT32 ui32Limit) +{ + if (pui32Indices[ui32Index] >= ui32Limit) + { + PVR_DPF((PVR_DBG_ERROR, "%s: Given alloc index %u at %u is larger than page array %u.", + __func__, pui32Indices[ui32Index], ui32Index, ui32Limit)); + return PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE; + } + + return PVRSRV_OK; +} + +static INLINE PVRSRV_ERROR +_CheckIfPageNotAllocated(IMG_UINT32 ui32Index, IMG_UINT32 *pui32Indices, struct page **ppsPageArray) +{ + if (ppsPageArray[pui32Indices[ui32Index]] != NULL) + { + PVR_DPF((PVR_DBG_ERROR, "%s: Mapping number %u at page array index %u already exists. " + "Page struct %p", __func__, pui32Indices[ui32Index], ui32Index, + ppsPageArray[pui32Indices[ui32Index]])); + return PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS; + } + + return PVRSRV_OK; +} + +/* Allocation of OS pages: This function is used for sparse allocations. + * + * Sparse allocations provide only a proportion of sparse physical backing within the total + * virtual range. */ +static PVRSRV_ERROR +_AllocOSPages_Sparse(PMR_OSPAGEARRAY_DATA *psPageArrayData, + IMG_UINT32 *puiAllocIndices, + IMG_UINT32 uiDevPagesToAlloc) +{ + PVRSRV_ERROR eError; + IMG_UINT32 i; + struct page **ppsPageArray = psPageArrayData->pagearray; + IMG_UINT32 uiOrder = psPageArrayData->uiLog2AllocPageSize - PAGE_SHIFT; + IMG_UINT32 uiDevPagesFromPool = 0; + IMG_UINT32 uiOSPagesToAlloc = uiDevPagesToAlloc * (1 << uiOrder); + IMG_UINT32 uiDevPagesAllocated = psPageArrayData->uiTotalNumOSPages >> uiOrder; + const IMG_UINT32 ui32AllocFlags = psPageArrayData->ui32AllocFlags; + gfp_t ui32GfpFlags = _GetGFPFlags(uiOrder ? BIT_ISSET(ui32AllocFlags, FLAG_ZERO): + IMG_FALSE, /* Zero pages later as batch */ + psPageArrayData->psDevNode); + + /* We use this page array to receive pages from the pool and then reuse it afterwards to + * store pages that need their cache attribute changed on x86 */ + struct page **ppsTempPageArray; + IMG_UINT32 uiTempPageArrayIndex = 0; + + /* Allocate the temporary page array that we need here to receive pages + * from the pool and to store pages that need their caching attributes changed. + * Allocate number of OS pages to be able to use the attribute function later. */ + ppsTempPageArray = OSAllocMem(sizeof(struct page*) * uiOSPagesToAlloc); + PVR_LOG_GOTO_IF_NOMEM(ppsTempPageArray, eError, e_exit); + + /* Check the requested number of pages if they fit in the page array */ + if (uiDevPagesAllocated < + ((psPageArrayData->iNumOSPagesAllocated >> uiOrder) + uiDevPagesToAlloc)) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Trying to allocate more pages (Order %u) than this buffer can handle, " + "Request + Allocated < Max! Request %u, Allocated %u, Max %u.", + __func__, + uiOrder, + uiDevPagesToAlloc, + psPageArrayData->iNumOSPagesAllocated >> uiOrder, + uiDevPagesAllocated)); + eError = PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE; + goto e_free_temp_array; + } + + /* Try to get pages from the pool since it is faster. The pages from pool are going to be + * allocated only if: + * - PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES == 1 && uiOrder == 0 + * - PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES == 0 && uiOrder == 0 && + * !BIT_ISSET(ui32AllocFlags, FLAG_ZERO) */ + _GetPagesFromPoolLocked(psPageArrayData->psDevNode, + psPageArrayData->ui32CPUCacheFlags, + uiDevPagesToAlloc, + uiOrder, + BIT_ISSET(ui32AllocFlags, FLAG_ZERO), + ppsTempPageArray, + &uiDevPagesFromPool); + + /* In general device pages can have higher order than 0 but page pool always provides only 0 + * order pages so they can be assigned to the OS pages values (in other words if we're + * allocating non-4k pages uiDevPagesFromPool will always be 0) */ + uiTempPageArrayIndex = uiDevPagesFromPool; + + /* Move pages we got from the pool to the array. */ + for (i = 0; i < uiDevPagesFromPool; i++) + { + eError = _CheckIfIndexInRange(i, puiAllocIndices, uiDevPagesAllocated); + PVR_GOTO_IF_ERROR(eError, e_free_pool_pages); + eError = _CheckIfPageNotAllocated(i, puiAllocIndices, ppsPageArray); + PVR_GOTO_IF_ERROR(eError, e_free_pool_pages); + + ppsPageArray[puiAllocIndices[i]] = ppsTempPageArray[i]; + } + + /* Allocate pages from the OS */ + for (i = uiDevPagesFromPool; i < uiDevPagesToAlloc; i++) + { + eError = _CheckIfIndexInRange(i, puiAllocIndices, uiDevPagesAllocated); + PVR_GOTO_IF_ERROR(eError, e_free_pages); + eError = _CheckIfPageNotAllocated(i, puiAllocIndices, ppsPageArray); + PVR_GOTO_IF_ERROR(eError, e_free_pages); + + /* Allocated pages and assign them the array. */ + if (BIT_ISSET(ui32AllocFlags, FLAG_IS_CMA)) + { + /* As the DMA/CMA framework rounds-up request to the + next power-of-two, we request multiple uiMinOrder + pages to satisfy allocation request in order to + minimise wasting memory */ + eError = _AllocOSPage_CMA(psPageArrayData, + ui32GfpFlags, + uiOrder, + uiOrder, + puiAllocIndices[i]); + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, "Failed to alloc CMA pages")); + goto e_free_pages; + } + } + else + { + DisableOOMKiller(); + ppsPageArray[puiAllocIndices[i]] = alloc_pages(ui32GfpFlags, uiOrder); + EnableOOMKiller(); + } + + if (ppsPageArray[puiAllocIndices[i]] != NULL) + { + /* Append pages to the temporary array so it's easier to process + * them later on. */ + + if (BIT_ISSET(ui32AllocFlags, FLAG_IS_CMA)) + { + IMG_UINT32 idx; + struct page *psPageAddr; + + psPageAddr = ppsPageArray[puiAllocIndices[i]]; + + /* "divide" CMA pages into OS pages if they have higher order */ + for (idx = 0; idx < (1 << uiOrder); idx++) + { + ppsTempPageArray[uiTempPageArrayIndex + idx] = psPageAddr; + psPageAddr++; + } + uiTempPageArrayIndex += (1 << uiOrder); + } + else + { + ppsTempPageArray[uiTempPageArrayIndex] = ppsPageArray[puiAllocIndices[i]]; + uiTempPageArrayIndex++; + } + } + else + { + /* Failed to alloc pages at required contiguity. Failed allocation */ + PVR_DPF((PVR_DBG_ERROR, + "%s: alloc_pages failed to honour request at %u of %u, flags = %x, order = %u", + __func__, i, uiDevPagesToAlloc, ui32GfpFlags, uiOrder)); + eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES; + goto e_free_pages; + } + } + + if (BIT_ISSET(ui32AllocFlags, FLAG_ZERO) && uiOrder == 0) + { + /* At this point this array contains pages allocated from the page pool at its start + * and pages allocated from the OS after that. + * If there are pages from the pool here they must be zeroed already hence we don't have + * to do it again. This is because if PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES is enabled pool pages + * are zeroed in the cleanup thread. If it's disabled they aren't, and in that case we never + * allocate pages with FLAG_ZERO from the pool. This is why those pages need to be zeroed + * here. + * All of the above is true for the 0 order pages. For higher order we never allocated from + * the pool and those pages are allocated already zeroed from the OS. + * Long story short we can always skip pages allocated from the pool because they are either + * zeroed or we didn't allocate any of them. */ + eError = _MemsetPageArray(uiTempPageArrayIndex - uiDevPagesFromPool, + &ppsTempPageArray[uiDevPagesFromPool], + PAGE_KERNEL, PVRSRV_ZERO_VALUE, 0); + PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to zero pages (sparse)", e_free_pages); + } + else if (BIT_ISSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC)) + { + /* Here we need to poison all of the pages regardless if they were + * allocated from the pool or from the system. */ + eError = _MemsetPageArray(uiTempPageArrayIndex, ppsTempPageArray, + PAGE_KERNEL, PVRSRV_POISON_ON_ALLOC_VALUE, 0); + PVR_LOG_IF_FALSE(eError == PVRSRV_OK, "failed to poison pages (sparse)"); + + /* We need to flush the cache for the poisoned pool pages here. The flush for the pages + * allocated from the system is done below because we also need to add appropriate cache + * attributes to them. Pages allocated from the pool already come with correct caching + * mode. */ + _ApplyCacheMaintenance(psPageArrayData->psDevNode, ppsTempPageArray, uiDevPagesFromPool); + } + + /* Do the cache management as required */ + eError = _ApplyOSPagesAttribute(psPageArrayData->psDevNode, + &ppsTempPageArray[uiDevPagesFromPool], + uiTempPageArrayIndex - uiDevPagesFromPool, + BIT_ISSET(ui32AllocFlags, FLAG_ZERO) || + BIT_ISSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC), + psPageArrayData->ui32CPUCacheFlags); + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, "Failed to set page attributes")); + goto e_free_pages; + } + + /* Update metadata */ + psPageArrayData->iNumOSPagesAllocated += uiOSPagesToAlloc; + + /* Free temporary page array */ + OSFreeMem(ppsTempPageArray); + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) +#if defined(PVRSRV_ENABLE_MEMORY_STATS) + for (i = 0; i < uiDevPagesToAlloc; i++) + { + _AddMemAllocRecord_UmaPages(psPageArrayData, + ppsPageArray[puiAllocIndices[i]]); + } +#else + _IncrMemAllocStat_UmaPages(((uiOSPagesToAlloc * PAGE_SIZE)+(psPageArrayData->ui32CMAAdjustedPageCount)), + psPageArrayData->uiPid); +#endif +#endif + + return PVRSRV_OK; + +e_free_pages: + if (BIT_ISSET(ui32AllocFlags, FLAG_IS_CMA)) + { + IMG_UINT32 uiDevPageSize = PAGE_SIZE << uiOrder; + + /* Free the pages we just allocated from the CMA */ + for (; i > uiDevPagesFromPool; i--) + { + _FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice, + uiDevPageSize, + uiOrder, + psPageArrayData->dmavirtarray[puiAllocIndices[i-1]], + psPageArrayData->dmaphysarray[puiAllocIndices[i-1]], + ppsPageArray[puiAllocIndices[i-1]]); + psPageArrayData->dmaphysarray[puiAllocIndices[i-1]]= (dma_addr_t) 0; + psPageArrayData->dmavirtarray[puiAllocIndices[i-1]] = NULL; + ppsPageArray[puiAllocIndices[i-1]] = NULL; + } + } + else + { + /* Free the pages we just allocated from the OS */ + for (; i > uiDevPagesFromPool; i--) + { + _FreeOSPage(0, IMG_FALSE, ppsPageArray[puiAllocIndices[i-1]]); + ppsPageArray[puiAllocIndices[i-1]] = NULL; + } + } + +e_free_pool_pages: + /* And now free all of the pages we allocated from the pool. */ + for (i = 0; i < uiDevPagesFromPool; i++) + { + _FreeOSPage(0, BIT_ISSET(ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE), + ppsTempPageArray[i]); + + /* not using _CheckIfIndexInRange() to not print error message */ + if (puiAllocIndices[i] < uiDevPagesAllocated) + { + ppsPageArray[puiAllocIndices[i]] = NULL; + } + } + +e_free_temp_array: + OSFreeMem(ppsTempPageArray); + +e_exit: + return eError; +} + +/* Allocate pages for a given page array. + * + * The executed allocation path depends whether an array with allocation + * indices has been passed or not */ +static PVRSRV_ERROR +_AllocOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData, + IMG_UINT32 *puiAllocIndices, + IMG_UINT32 uiPagesToAlloc) +{ + PVRSRV_ERROR eError; + struct page **ppsPageArray; + + /* Parameter checks */ + PVR_ASSERT(NULL != psPageArrayData); + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + PVR_ASSERT(psPageArrayData->dmaphysarray != NULL); + PVR_ASSERT(psPageArrayData->dmavirtarray != NULL); + } + PVR_ASSERT(psPageArrayData->pagearray != NULL); + PVR_ASSERT(0 <= psPageArrayData->iNumOSPagesAllocated); + + ppsPageArray = psPageArrayData->pagearray; + + /* Go the sparse alloc path if we have an array with alloc indices.*/ + if (puiAllocIndices != NULL) + { + eError = _AllocOSPages_Sparse(psPageArrayData, + puiAllocIndices, + uiPagesToAlloc); + } + else + { + eError = _AllocOSPages_Fast(psPageArrayData); + } + + if (eError != PVRSRV_OK) + { + goto e_exit; + } + + _DumpPageArray(ppsPageArray, + psPageArrayData->uiTotalNumOSPages >> + (psPageArrayData->uiLog2AllocPageSize - PAGE_SHIFT) ); + + PVR_DPF((PVR_DBG_MESSAGE, "physmem_osmem_linux.c: allocated OS memory for PMR @0x%p", psPageArrayData)); + return PVRSRV_OK; + +e_exit: + return eError; +} + +/* Same as _FreeOSPage except free memory using DMA framework */ +static INLINE void +_FreeOSPage_CMA(struct device *dev, + size_t alloc_size, + IMG_UINT32 uiOrder, + void *virt_addr, + dma_addr_t dev_addr, + struct page *psPage) +{ + if (DMA_IS_ALLOCPG_ADDR(dev_addr)) + { +#if defined(CONFIG_X86) + void *pvPageVAddr = page_address(psPage); + if (pvPageVAddr) + { + int ret = set_memory_wb((unsigned long)pvPageVAddr, 1); + if (ret) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Failed to reset page attribute", + __func__)); + } + } +#endif + + if (DMA_IS_ADDR_ADJUSTED(dev_addr)) + { + psPage -= DMA_GET_ALIGN_ADJUSTMENT(dev_addr); + uiOrder += 1; + } + + __free_pages(psPage, uiOrder); + } + else + { + if (DMA_IS_ADDR_ADJUSTED(dev_addr)) + { + size_t align_adjust; + + align_adjust = DMA_GET_ALIGN_ADJUSTMENT(dev_addr); + alloc_size = alloc_size << 1; + + dev_addr = DMA_GET_ADDR(dev_addr); + dev_addr -= align_adjust << PAGE_SHIFT; + virt_addr -= align_adjust << PAGE_SHIFT; + } + + dma_free_coherent(dev, alloc_size, virt_addr, DMA_GET_ADDR(dev_addr)); + } +} + +/* Free a single page back to the OS. + * Make sure the cache type is set back to the default value. + * + * Note: + * We must _only_ check bUnsetMemoryType in the case where we need to free + * the page back to the OS since we may have to revert the cache properties + * of the page to the default as given by the OS when it was allocated. */ +static void +_FreeOSPage(IMG_UINT32 uiOrder, + IMG_BOOL bUnsetMemoryType, + struct page *psPage) +{ + +#if defined(CONFIG_X86) + void *pvPageVAddr; + pvPageVAddr = page_address(psPage); + + if (pvPageVAddr && bUnsetMemoryType) + { + int ret; + + ret = set_memory_wb((unsigned long)pvPageVAddr, 1); + if (ret) + { + PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute", + __func__)); + } + } +#else + PVR_UNREFERENCED_PARAMETER(bUnsetMemoryType); +#endif + __free_pages(psPage, uiOrder); +} + +/* Free the struct holding the metadata */ +static PVRSRV_ERROR +_FreeOSPagesArray(PMR_OSPAGEARRAY_DATA *psPageArrayData) +{ + PVR_DPF((PVR_DBG_MESSAGE, "physmem_osmem_linux.c: freed OS memory for PMR @0x%p", psPageArrayData)); + + /* Check if the page array actually still exists. + * It might be the case that has been moved to the page pool */ + if (psPageArrayData->pagearray != NULL) + { + OSFreeMemNoStats(psPageArrayData->pagearray); + } + + kmem_cache_free(g_psLinuxPageArray, psPageArrayData); + + return PVRSRV_OK; +} + +/* Free all or some pages from a sparse page array */ +static PVRSRV_ERROR +_FreeOSPages_Sparse(PMR_OSPAGEARRAY_DATA *psPageArrayData, + IMG_UINT32 *pai32FreeIndices, + IMG_UINT32 ui32FreePageCount) +{ + IMG_BOOL bSuccess; + IMG_UINT32 uiOrder = psPageArrayData->uiLog2AllocPageSize - PAGE_SHIFT; + IMG_UINT32 uiPageIndex, i, j, uiTempIdx = 0; + struct page **ppsPageArray = psPageArrayData->pagearray; + IMG_UINT32 uiNumPages; + + struct page **ppsTempPageArray; + IMG_UINT32 uiTempArraySize; + + /* We really should have something to free before we call this */ + PVR_ASSERT(psPageArrayData->iNumOSPagesAllocated != 0); + + if (pai32FreeIndices == NULL) + { + uiNumPages = psPageArrayData->uiTotalNumOSPages >> uiOrder; + uiTempArraySize = psPageArrayData->iNumOSPagesAllocated; + } + else + { + uiNumPages = ui32FreePageCount; + uiTempArraySize = ui32FreePageCount << uiOrder; + } + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS) + for (i = 0; i < uiNumPages; i++) + { + IMG_UINT32 idx = pai32FreeIndices ? pai32FreeIndices[i] : i; + + if (NULL != ppsPageArray[idx]) + { + _RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[idx]); + } + } +#endif + + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_FREE)) + { + for (i = 0; i < uiNumPages; i++) + { + IMG_UINT32 idx = pai32FreeIndices ? pai32FreeIndices[i] : i; + + if (NULL != ppsPageArray[idx]) + { + _PoisonDevicePage(psPageArrayData->psDevNode, + ppsPageArray[idx], + uiOrder, + psPageArrayData->ui32CPUCacheFlags, + PVRSRV_POISON_ON_FREE_VALUE); + } + } + } + + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + IMG_UINT32 uiDevNumPages = uiNumPages; + IMG_UINT32 uiDevPageSize = 1<<psPageArrayData->uiLog2AllocPageSize; + + for (i = 0; i < uiDevNumPages; i++) + { + IMG_UINT32 idx = pai32FreeIndices ? pai32FreeIndices[i] : i; + if (NULL != ppsPageArray[idx]) + { + _FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice, + uiDevPageSize, + uiOrder, + psPageArrayData->dmavirtarray[idx], + psPageArrayData->dmaphysarray[idx], + ppsPageArray[idx]); + psPageArrayData->dmaphysarray[idx] = (dma_addr_t)0; + psPageArrayData->dmavirtarray[idx] = NULL; + ppsPageArray[idx] = NULL; + uiTempIdx++; + } + } + uiTempIdx <<= uiOrder; + } + else + { + + /* OSAllocMemNoStats required because this code may be run without the bridge lock held */ + ppsTempPageArray = OSAllocMemNoStats(sizeof(struct page*) * uiTempArraySize); + if (ppsTempPageArray == NULL) + { + PVR_DPF((PVR_DBG_ERROR, "%s: Failed free_pages metadata allocation", __func__)); + return PVRSRV_ERROR_OUT_OF_MEMORY; + } + + /* Put pages in a contiguous array so further processing is easier */ + for (i = 0; i < uiNumPages; i++) + { + uiPageIndex = pai32FreeIndices ? pai32FreeIndices[i] : i; + if (NULL != ppsPageArray[uiPageIndex]) + { + struct page *psPage = ppsPageArray[uiPageIndex]; + + for (j = 0; j < (1<<uiOrder); j++) + { + ppsTempPageArray[uiTempIdx] = psPage; + uiTempIdx++; + psPage++; + } + + ppsPageArray[uiPageIndex] = NULL; + } + } + + /* Try to move the temp page array to the pool */ + bSuccess = _PutPagesToPoolLocked(psPageArrayData->ui32CPUCacheFlags, + ppsTempPageArray, + BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNPINNED), + 0, + uiTempIdx); + if (bSuccess) + { + goto exit_ok; + } + + /* Free pages and reset page caching attributes on x86 */ +#if defined(CONFIG_X86) + if (uiTempIdx != 0 && BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE)) + { + int iError; + iError = set_pages_array_wb(ppsTempPageArray, uiTempIdx); + + if (iError) + { + PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attributes", __func__)); + } + } +#endif + + /* Free the pages */ + for (i = 0; i < uiTempIdx; i++) + { + __free_pages(ppsTempPageArray[i], 0); + } + + /* Free the temp page array here if it did not move to the pool */ + OSFreeMemNoStats(ppsTempPageArray); + } + +exit_ok: + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS) + _DecrMemAllocStat_UmaPages(((uiTempIdx * PAGE_SIZE)-(psPageArrayData->ui32CMAAdjustedPageCount)), + psPageArrayData->uiPid); +#endif + + if (pai32FreeIndices && ((uiTempIdx >> uiOrder) != ui32FreePageCount)) + { + PVR_DPF((PVR_DBG_ERROR, "%s: Probable sparse duplicate indices: ReqFreeCount: %d " + "ActualFreedCount: %d", __func__, ui32FreePageCount, (uiTempIdx >> uiOrder))); + } + /* Update metadata */ + psPageArrayData->iNumOSPagesAllocated -= uiTempIdx; + PVR_ASSERT(0 <= psPageArrayData->iNumOSPagesAllocated); + return PVRSRV_OK; +} + +/* Free all the pages in a page array */ +static PVRSRV_ERROR +_FreeOSPages_Fast(PMR_OSPAGEARRAY_DATA *psPageArrayData) +{ + IMG_BOOL bSuccess; + IMG_UINT32 i; + IMG_UINT32 uiNumPages = psPageArrayData->uiTotalNumOSPages; + IMG_UINT32 uiOrder = psPageArrayData->uiLog2AllocPageSize - PAGE_SHIFT; + IMG_UINT32 uiDevNumPages = uiNumPages >> uiOrder; + IMG_UINT32 uiDevPageSize = PAGE_SIZE << uiOrder; + struct page **ppsPageArray = psPageArrayData->pagearray; + + /* We really should have something to free before we call this */ + PVR_ASSERT(psPageArrayData->iNumOSPagesAllocated != 0); + +#if defined(PVRSRV_ENABLE_PROCESS_STATS) +#if defined(PVRSRV_ENABLE_MEMORY_STATS) + for (i = 0; i < uiDevNumPages; i++) + { + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + _RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i]); + }else + { + _RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i << uiOrder]); + } + } +#else + _DecrMemAllocStat_UmaPages(((uiNumPages * PAGE_SIZE)-(psPageArrayData->ui32CMAAdjustedPageCount)), + psPageArrayData->uiPid); +#endif +#endif + + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_FREE)) + { + for (i = 0; i < uiDevNumPages; i++) + { + _PoisonDevicePage(psPageArrayData->psDevNode, + ppsPageArray[i], + uiOrder, + psPageArrayData->ui32CPUCacheFlags, + PVRSRV_POISON_ON_FREE_VALUE); + } + } + + /* Try to move the page array to the pool */ + bSuccess = _PutPagesToPoolLocked(psPageArrayData->ui32CPUCacheFlags, + ppsPageArray, + BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNPINNED), + uiOrder, + uiNumPages); + if (bSuccess) + { + psPageArrayData->pagearray = NULL; + goto exit_ok; + } + + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_CMA)) + { + for (i = 0; i < uiDevNumPages; i++) + { + _FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice, + uiDevPageSize, + uiOrder, + psPageArrayData->dmavirtarray[i], + psPageArrayData->dmaphysarray[i], + ppsPageArray[i]); + psPageArrayData->dmaphysarray[i] = (dma_addr_t)0; + psPageArrayData->dmavirtarray[i] = NULL; + ppsPageArray[i] = NULL; + } + } + else + { +#if defined(CONFIG_X86) + if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE)) + { + int ret; + + ret = set_pages_array_wb(ppsPageArray, uiNumPages); + if (ret) + { + PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attributes", + __func__)); + } + } +#endif + + for (i = 0; i < uiNumPages; i++) + { + _FreeOSPage(uiOrder, IMG_FALSE, ppsPageArray[i]); + ppsPageArray[i] = NULL; + } + } + +exit_ok: + /* Update metadata */ + psPageArrayData->iNumOSPagesAllocated = 0; + return PVRSRV_OK; +} + +/* Free pages from a page array. + * Takes care of mem stats and chooses correct free path depending on parameters. */ +static PVRSRV_ERROR +_FreeOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData, + IMG_UINT32 *pai32FreeIndices, + IMG_UINT32 ui32FreePageCount) +{ + PVRSRV_ERROR eError; + + /* Go the sparse or non-sparse path */ + if (psPageArrayData->iNumOSPagesAllocated != psPageArrayData->uiTotalNumOSPages + || pai32FreeIndices != NULL) + { + eError = _FreeOSPages_Sparse(psPageArrayData, + pai32FreeIndices, + ui32FreePageCount); + } + else + { + eError = _FreeOSPages_Fast(psPageArrayData); + } + + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, "_FreeOSPages_FreePages failed")); + } + + _DumpPageArray(psPageArrayData->pagearray, + psPageArrayData->uiTotalNumOSPages >> + (psPageArrayData->uiLog2AllocPageSize - PAGE_SHIFT) ); + + return eError; +} + +/* + * + * Implementation of callback functions + * + */ + +/* Destruction function is called after last reference disappears, + * but before PMR itself is freed. + */ +static PVRSRV_ERROR +PMRFinalizeOSMem(PMR_IMPL_PRIVDATA pvPriv) +{ + PVRSRV_ERROR eError; + PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv; + + /* We can't free pages until now. */ + if (psOSPageArrayData->iNumOSPagesAllocated != 0) + { +#if defined(DEBUG) && defined(SUPPORT_VALIDATION) + PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData(); + IMG_UINT32 ui32UMALeakMax = psPVRSRVData->sMemLeakIntervals.ui32GPU; + + mutex_lock(&g_sUMALeakMutex); + + g_ui32UMALeakCounter++; + if (ui32UMALeakMax && g_ui32UMALeakCounter >= ui32UMALeakMax) + { + g_ui32UMALeakCounter = 0; + mutex_unlock(&g_sUMALeakMutex); + + PVR_DPF((PVR_DBG_WARNING, "%s: Skipped freeing of PMR 0x%p to trigger memory leak.", __func__, pvPriv)); + return PVRSRV_OK; + } + + mutex_unlock(&g_sUMALeakMutex); +#endif + _PagePoolLock(); + if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_UNPINNED)) + { + _RemoveUnpinListEntryUnlocked(psOSPageArrayData); + } + _PagePoolUnlock(); + + eError = _FreeOSPages(psOSPageArrayData, + NULL, + 0); + PVR_ASSERT(eError == PVRSRV_OK); /* can we do better? */ + } + + eError = _FreeOSPagesArray(psOSPageArrayData); + PVR_ASSERT(eError == PVRSRV_OK); /* can we do better? */ + return PVRSRV_OK; +} + +/* Callback function for locking the system physical page addresses. + * This function must be called before the lookup address func. */ +static PVRSRV_ERROR +PMRLockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv) +{ + PVRSRV_ERROR eError; + PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv; + + if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND)) + { + /* Allocate Memory for deferred allocation */ + eError = _AllocOSPages(psOSPageArrayData, NULL, psOSPageArrayData->uiTotalNumOSPages); + if (eError != PVRSRV_OK) + { + return eError; + } + } + + eError = PVRSRV_OK; + return eError; +} + +static PVRSRV_ERROR +PMRUnlockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv) +{ + /* Just drops the refcount. */ + PVRSRV_ERROR eError = PVRSRV_OK; + PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv; + + if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND)) + { + /* Free Memory for deferred allocation */ + eError = _FreeOSPages(psOSPageArrayData, + NULL, + 0); + if (eError != PVRSRV_OK) + { + return eError; + } + } + + PVR_ASSERT(eError == PVRSRV_OK); + return eError; +} + +static INLINE IMG_BOOL IsOffsetValid(const PMR_OSPAGEARRAY_DATA *psOSPageArrayData, + IMG_UINT32 ui32Offset) +{ + return (ui32Offset >> psOSPageArrayData->uiLog2AllocPageSize) < + psOSPageArrayData->uiTotalNumOSPages; +} + +/* Determine PA for specified offset into page array. */ +static IMG_DEV_PHYADDR GetOffsetPA(const PMR_OSPAGEARRAY_DATA *psOSPageArrayData, + IMG_UINT32 ui32Offset) +{ + IMG_UINT32 ui32Log2AllocPageSize = psOSPageArrayData->uiLog2AllocPageSize; + IMG_UINT32 ui32PageIndex = ui32Offset >> ui32Log2AllocPageSize; + IMG_UINT32 ui32InPageOffset = ui32Offset - (ui32PageIndex << ui32Log2AllocPageSize); + IMG_DEV_PHYADDR sPA; + + PVR_ASSERT(ui32InPageOffset < (1U << ui32Log2AllocPageSize)); + + sPA.uiAddr = page_to_phys(psOSPageArrayData->pagearray[ui32PageIndex]); + sPA.uiAddr += ui32InPageOffset; + + return sPA; +} + +/* N.B. It is assumed that PMRLockSysPhysAddressesOSMem() is called _before_ this function! */ +static PVRSRV_ERROR +PMRSysPhysAddrOSMem(PMR_IMPL_PRIVDATA pvPriv, + IMG_UINT32 ui32Log2PageSize, + IMG_UINT32 ui32NumOfPages, + IMG_DEVMEM_OFFSET_T *puiOffset, + IMG_BOOL *pbValid, + IMG_DEV_PHYADDR *psDevPAddr) +{ + const PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv; + IMG_UINT32 uiIdx; + + if (psOSPageArrayData->uiLog2AllocPageSize < ui32Log2PageSize) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Requested physical addresses from PMR " + "for incompatible contiguity %u!", + __func__, + ui32Log2PageSize)); + return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY; + } + + for (uiIdx=0; uiIdx < ui32NumOfPages; uiIdx++) + { + if (pbValid[uiIdx]) + { + PVR_LOG_RETURN_IF_FALSE(IsOffsetValid(psOSPageArrayData, puiOffset[uiIdx]), + "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE); + + psDevPAddr[uiIdx] = GetOffsetPA(psOSPageArrayData, puiOffset[uiIdx]); + +#if !defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY) + /* this is just a precaution, normally this should be always + * available */ + if (psOSPageArrayData->ui64DmaMask) + { + if (psDevPAddr[uiIdx].uiAddr > psOSPageArrayData->ui64DmaMask) + { + PVR_DPF((PVR_DBG_ERROR, "%s: physical address" + " (%" IMG_UINT64_FMTSPECX ") out of allowable range" + " [0; %" IMG_UINT64_FMTSPECX "]", __func__, + psDevPAddr[uiIdx].uiAddr, + psOSPageArrayData->ui64DmaMask)); + BUG(); + } + } +#endif + } + } + + return PVRSRV_OK; +} + +typedef struct _PMR_OSPAGEARRAY_KERNMAP_DATA_ { + void *pvBase; + IMG_UINT32 ui32PageCount; + pgprot_t PageProps; +} PMR_OSPAGEARRAY_KERNMAP_DATA; + +static PVRSRV_ERROR +PMRAcquireKernelMappingDataOSMem(PMR_IMPL_PRIVDATA pvPriv, + size_t uiOffset, + size_t uiSize, + void **ppvKernelAddressOut, + IMG_HANDLE *phHandleOut, + PMR_FLAGS_T ulFlags) +{ + PVRSRV_ERROR eError; + PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv; + void *pvAddress; + pgprot_t prot = PAGE_KERNEL; + IMG_UINT32 ui32PageOffset=0; + size_t uiMapOffset=0; + IMG_UINT32 ui32PageCount = 0; + IMG_UINT32 uiLog2AllocPageSize = psOSPageArrayData->uiLog2AllocPageSize; + IMG_UINT32 uiOSPageShift = OSGetPageShift(); + IMG_UINT32 uiPageSizeDiff = 0; + struct page **pagearray; + PMR_OSPAGEARRAY_KERNMAP_DATA *psData; + + int riscv_cache = 0; + + /* For cases device page size greater than the OS page size, + * multiple physically contiguous OS pages constitute one device page. + * However only the first page address of such an ensemble is stored + * as part of the mapping table in the driver. Hence when mapping the PMR + * in part/full, all OS pages that constitute the device page + * must also be mapped to kernel. + * + * For the case where device page size less than OS page size, + * treat it the same way as the page sizes are equal */ + if (uiLog2AllocPageSize > uiOSPageShift) + { + uiPageSizeDiff = uiLog2AllocPageSize - uiOSPageShift; + } + + /* + Zero offset and size as a special meaning which means map in the + whole of the PMR, this is due to fact that the places that call + this callback might not have access to be able to determine the + physical size + */ + if ((uiOffset == 0) && (uiSize == 0)) + { + ui32PageOffset = 0; + uiMapOffset = 0; + /* Page count = amount of OS pages */ + ui32PageCount = psOSPageArrayData->iNumOSPagesAllocated; + } + else + { + size_t uiEndoffset; + + ui32PageOffset = uiOffset >> uiLog2AllocPageSize; + uiMapOffset = uiOffset - (ui32PageOffset << uiLog2AllocPageSize); + uiEndoffset = uiOffset + uiSize - 1; + /* Add one as we want the count, not the offset */ + /* Page count = amount of device pages (note uiLog2AllocPageSize being used) */ + ui32PageCount = (uiEndoffset >> uiLog2AllocPageSize) + 1; + ui32PageCount -= ui32PageOffset; + + /* The OS page count to be mapped might be different if the + * OS page size is lesser than the device page size */ + ui32PageCount <<= uiPageSizeDiff; + } + + switch (PVRSRV_CPU_CACHE_MODE(psOSPageArrayData->ui32CPUCacheFlags)) + { + case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED: + prot = pgprot_noncached(prot); + break; + + case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC: + prot = pgprot_writecombine(prot); + break; + + case PVRSRV_MEMALLOCFLAG_CPU_CACHED: + riscv_cache = 1; + break; + + default: + eError = PVRSRV_ERROR_INVALID_PARAMS; + goto e0; + } + + if (uiPageSizeDiff) + { + /* Each device page can be broken down into ui32SubPageCount OS pages */ + IMG_UINT32 ui32SubPageCount = 1 << uiPageSizeDiff; + IMG_UINT32 i; + struct page **psPage = &psOSPageArrayData->pagearray[ui32PageOffset]; + + /* Allocate enough memory for the OS page pointers for this mapping */ + pagearray = OSAllocMem(ui32PageCount * sizeof(pagearray[0])); + + if (pagearray == NULL) + { + eError = PVRSRV_ERROR_OUT_OF_MEMORY; + goto e0; + } + + /* construct array that holds the page pointers that constitute the requested + * mapping */ + for (i = 0; i < ui32PageCount; i++) + { + IMG_UINT32 ui32OSPageArrayIndex = i / ui32SubPageCount; + IMG_UINT32 ui32OSPageArrayOffset = i % ui32SubPageCount; + + /* + * The driver only stores OS page pointers for the first OS page + * within each device page (psPage[ui32OSPageArrayIndex]). + * Get the next OS page structure at device page granularity, + * then calculate OS page pointers for all the other pages. + */ + pagearray[i] = psPage[ui32OSPageArrayIndex] + ui32OSPageArrayOffset; + } + } + else + { + pagearray = &psOSPageArrayData->pagearray[ui32PageOffset]; + } + + psData = OSAllocMem(sizeof(*psData)); + if (psData == NULL) + { + eError = PVRSRV_ERROR_OUT_OF_MEMORY; + goto e1; + } + + if (riscv_cache) { + pvAddress = pvr_vmap_cached(pagearray, ui32PageCount, VM_READ | VM_WRITE, prot); + } else { + pvAddress = pvr_vmap(pagearray, ui32PageCount, VM_READ | VM_WRITE, prot); + } + if (pvAddress == NULL) + { + eError = PVRSRV_ERROR_OUT_OF_MEMORY; + goto e2; + } + + *ppvKernelAddressOut = pvAddress + uiMapOffset; + psData->pvBase = pvAddress; + psData->ui32PageCount = ui32PageCount; + psData->PageProps = prot; + *phHandleOut = psData; + + if (uiPageSizeDiff) + { + OSFreeMem(pagearray); + } + + return PVRSRV_OK; + + /* + error exit paths follow + */ +e2: + OSFreeMem(psData); +e1: + if (uiPageSizeDiff) + { + OSFreeMem(pagearray); + } +e0: + PVR_ASSERT(eError != PVRSRV_OK); + return eError; +} + +static void PMRReleaseKernelMappingDataOSMem(PMR_IMPL_PRIVDATA pvPriv, + IMG_HANDLE hHandle) +{ + PMR_OSPAGEARRAY_KERNMAP_DATA *psData = hHandle; + PVR_UNREFERENCED_PARAMETER(pvPriv); + + pvr_vunmap(psData->pvBase, psData->ui32PageCount, psData->PageProps); + OSFreeMem(psData); +} + +static +PVRSRV_ERROR PMRUnpinOSMem(PMR_IMPL_PRIVDATA pPriv) +{ + PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pPriv; + PVRSRV_ERROR eError = PVRSRV_OK; + + /* Lock down the pool and add the array to the unpin list */ + _PagePoolLock(); + + /* Check current state */ + PVR_ASSERT(BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_UNPINNED) == IMG_FALSE); + PVR_ASSERT(BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND) == IMG_FALSE); + + eError = _AddUnpinListEntryUnlocked(psOSPageArrayData); + + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Unable to add allocation to unpinned list (%d).", + __func__, + eError)); + + goto e_exit; + } + + /* Set the Unpinned bit */ + BIT_SET(psOSPageArrayData->ui32AllocFlags, FLAG_UNPINNED); + +e_exit: + _PagePoolUnlock(); + return eError; +} + +static +PVRSRV_ERROR PMRPinOSMem(PMR_IMPL_PRIVDATA pPriv, + PMR_MAPPING_TABLE *psMappingTable) +{ + PVRSRV_ERROR eError; + PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pPriv; + IMG_UINT32 *pui32MapTable = NULL; + IMG_UINT32 i, j = 0, ui32Temp = 0; + + _PagePoolLock(); + + /* Check current state */ + PVR_ASSERT(BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_UNPINNED)); + + /* Clear unpinned bit */ + BIT_UNSET(psOSPageArrayData->ui32AllocFlags, FLAG_UNPINNED); + + /* If there are still pages in the array remove entries from the pool */ + if (psOSPageArrayData->iNumOSPagesAllocated != 0) + { + _RemoveUnpinListEntryUnlocked(psOSPageArrayData); + _PagePoolUnlock(); + + eError = PVRSRV_OK; + goto e_exit_mapalloc_failure; + } + _PagePoolUnlock(); + + /* If pages were reclaimed we allocate new ones and + * return PVRSRV_ERROR_PMR_NEW_MEMORY */ + if (psMappingTable->ui32NumVirtChunks == 1) + { + eError = _AllocOSPages(psOSPageArrayData, NULL, psOSPageArrayData->uiTotalNumOSPages); + } + else + { + pui32MapTable = (IMG_UINT32 *)OSAllocMem(sizeof(*pui32MapTable) * psMappingTable->ui32NumPhysChunks); + if (NULL == pui32MapTable) + { + eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES; + PVR_DPF((PVR_DBG_ERROR, + "%s: Unable to Alloc Map Table.", + __func__)); + goto e_exit_mapalloc_failure; + } + + for (i = 0, j = 0; i < psMappingTable->ui32NumVirtChunks; i++) + { + ui32Temp = psMappingTable->aui32Translation[i]; + if (TRANSLATION_INVALID != ui32Temp) + { + pui32MapTable[j++] = ui32Temp; + } + } + eError = _AllocOSPages(psOSPageArrayData, pui32MapTable, psMappingTable->ui32NumPhysChunks); + } + + if (eError != PVRSRV_OK) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Unable to get new pages for unpinned allocation.", + __func__)); + + eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES; + goto e_exit; + } + + PVR_DPF((PVR_DBG_MESSAGE, + "%s: Allocating new pages for unpinned allocation. " + "Old content is lost!", + __func__)); + + eError = PVRSRV_ERROR_PMR_NEW_MEMORY; + +e_exit: + OSFreeMem(pui32MapTable); +e_exit_mapalloc_failure: + return eError; +} + +/*************************************************************************/ /*! +@Function PMRChangeSparseMemOSMem +@Description This function Changes the sparse mapping by allocating and + freeing of pages. It changes the GPU and CPU maps accordingly. +@Return PVRSRV_ERROR failure code +*/ /**************************************************************************/ +static PVRSRV_ERROR +PMRChangeSparseMemOSMem(PMR_IMPL_PRIVDATA pPriv, + const PMR *psPMR, + IMG_UINT32 ui32AllocPageCount, + IMG_UINT32 *pai32AllocIndices, + IMG_UINT32 ui32FreePageCount, + IMG_UINT32 *pai32FreeIndices, + IMG_UINT32 uiFlags) +{ + PVRSRV_ERROR eError; + + PMR_MAPPING_TABLE *psPMRMapTable = PMR_GetMappingTable(psPMR); + PMR_OSPAGEARRAY_DATA *psPMRPageArrayData = (PMR_OSPAGEARRAY_DATA *)pPriv; + struct page **psPageArray = psPMRPageArrayData->pagearray; + void **psDMAVirtArray = psPMRPageArrayData->dmavirtarray; + dma_addr_t *psDMAPhysArray = psPMRPageArrayData->dmaphysarray; + + struct page *psPage; + dma_addr_t psDMAPAddr; + void *pvDMAVAddr; + + IMG_UINT32 ui32AdtnlAllocPages = 0; /*<! Number of pages to alloc from the OS */ + IMG_UINT32 ui32AdtnlFreePages = 0; /*<! Number of pages to free back to the OS */ + IMG_UINT32 ui32CommonRequestCount = 0; /*<! Number of pages to move position in the page array */ + IMG_UINT32 ui32Loop = 0; + IMG_UINT32 ui32Index = 0; + IMG_UINT32 uiAllocpgidx; + IMG_UINT32 uiFreepgidx; + IMG_UINT32 uiOrder = psPMRPageArrayData->uiLog2AllocPageSize - PAGE_SHIFT; + IMG_BOOL bCMA = BIT_ISSET(psPMRPageArrayData->ui32AllocFlags, FLAG_IS_CMA); + + + /* Check SPARSE flags and calculate pages to allocate and free */ + if (SPARSE_RESIZE_BOTH == (uiFlags & SPARSE_RESIZE_BOTH)) + { + ui32CommonRequestCount = (ui32AllocPageCount > ui32FreePageCount) ? + ui32FreePageCount : ui32AllocPageCount; + + PDUMP_PANIC(PMR_DeviceNode(psPMR), SPARSEMEM_SWAP, "Request to swap alloc & free pages not supported"); + } + + if (SPARSE_RESIZE_ALLOC == (uiFlags & SPARSE_RESIZE_ALLOC)) + { + ui32AdtnlAllocPages = ui32AllocPageCount - ui32CommonRequestCount; + } + else + { + ui32AllocPageCount = 0; + } + + if (SPARSE_RESIZE_FREE == (uiFlags & SPARSE_RESIZE_FREE)) + { + ui32AdtnlFreePages = ui32FreePageCount - ui32CommonRequestCount; + } + else + { + ui32FreePageCount = 0; + } + + if (0 == (ui32CommonRequestCount || ui32AdtnlAllocPages || ui32AdtnlFreePages)) + { + eError = PVRSRV_ERROR_INVALID_PARAMS; + PVR_DPF((PVR_DBG_ERROR, + "%s: Missing parameters for number of pages to alloc/free", + __func__)); + return eError; + } + + /* The incoming request is classified into two operations independent of + * each other: alloc & free pages. + * These operations can be combined with two mapping operations as well + * which are GPU & CPU space mappings. + * + * From the alloc and free page requests, the net amount of pages to be + * allocated or freed is computed. Pages that were requested to be freed + * will be reused to fulfil alloc requests. + * + * The order of operations is: + * 1. Allocate new pages from the OS + * 2. Move the free pages from free request to alloc positions. + * 3. Free the rest of the pages not used for alloc + * + * Alloc parameters are validated at the time of allocation + * and any error will be handled then. */ + + /* Validate the free indices */ + if (ui32FreePageCount) + { + if (NULL != pai32FreeIndices){ + + for (ui32Loop = 0; ui32Loop < ui32FreePageCount; ui32Loop++) + { + uiFreepgidx = pai32FreeIndices[ui32Loop]; + + if (uiFreepgidx > (psPMRPageArrayData->uiTotalNumOSPages >> uiOrder)) + { + eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE; + goto e0; + } + + if (NULL == psPageArray[uiFreepgidx]) + { + eError = PVRSRV_ERROR_INVALID_PARAMS; + PVR_DPF((PVR_DBG_ERROR, + "%s: Trying to free non-allocated page", + __func__)); + goto e0; + } + } + } + else + { + eError = PVRSRV_ERROR_INVALID_PARAMS; + PVR_DPF((PVR_DBG_ERROR, + "%s: Given non-zero free count but missing indices array", + __func__)); + return eError; + } + } + + /* Validate the alloc indices */ + for (ui32Loop = ui32AdtnlAllocPages; ui32Loop < ui32AllocPageCount; ui32Loop++) + { + uiAllocpgidx = pai32AllocIndices[ui32Loop]; + + if (uiAllocpgidx > (psPMRPageArrayData->uiTotalNumOSPages >> uiOrder)) + { + eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE; + goto e0; + } + + if (SPARSE_REMAP_MEM != (uiFlags & SPARSE_REMAP_MEM)) + { + if ((NULL != psPageArray[uiAllocpgidx]) || + (TRANSLATION_INVALID != psPMRMapTable->aui32Translation[uiAllocpgidx])) + { + eError = PVRSRV_ERROR_INVALID_PARAMS; + PVR_DPF((PVR_DBG_ERROR, + "%s: Trying to allocate already allocated page again", + __func__)); + goto e0; + } + } + else + { + if ((NULL == psPageArray[uiAllocpgidx]) || + (TRANSLATION_INVALID == psPMRMapTable->aui32Translation[uiAllocpgidx]) ) + { + eError = PVRSRV_ERROR_INVALID_PARAMS; + PVR_DPF((PVR_DBG_ERROR, + "%s: Unable to remap memory due to missing page", + __func__)); + goto e0; + } + } + } + + ui32Loop = 0; + + /* Allocate new pages from the OS */ + if (0 != ui32AdtnlAllocPages) + { + eError = _AllocOSPages(psPMRPageArrayData, pai32AllocIndices, ui32AdtnlAllocPages); + if (PVRSRV_OK != eError) + { + PVR_DPF((PVR_DBG_MESSAGE, + "%s: New Addtl Allocation of pages failed", + __func__)); + goto e0; + } + + psPMRMapTable->ui32NumPhysChunks += ui32AdtnlAllocPages; + /*Mark the corresponding pages of translation table as valid */ + for (ui32Loop = 0; ui32Loop < ui32AdtnlAllocPages; ui32Loop++) + { + psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = pai32AllocIndices[ui32Loop]; + } + } + + + ui32Index = ui32Loop; + + /* Move the corresponding free pages to alloc request */ + for (ui32Loop = 0; ui32Loop < ui32CommonRequestCount; ui32Loop++, ui32Index++) + { + uiAllocpgidx = pai32AllocIndices[ui32Index]; + uiFreepgidx = pai32FreeIndices[ui32Loop]; + + psPage = psPageArray[uiAllocpgidx]; + psPageArray[uiAllocpgidx] = psPageArray[uiFreepgidx]; + + if (bCMA) + { + pvDMAVAddr = psDMAVirtArray[uiAllocpgidx]; + psDMAPAddr = psDMAPhysArray[uiAllocpgidx]; + psDMAVirtArray[uiAllocpgidx] = psDMAVirtArray[uiFreepgidx]; + psDMAPhysArray[uiAllocpgidx] = psDMAPhysArray[uiFreepgidx]; + } + + /* Is remap mem used in real world scenario? Should it be turned to a + * debug feature? The condition check needs to be out of loop, will be + * done at later point though after some analysis */ + if (SPARSE_REMAP_MEM != (uiFlags & SPARSE_REMAP_MEM)) + { + psPMRMapTable->aui32Translation[uiFreepgidx] = TRANSLATION_INVALID; + psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx; + psPageArray[uiFreepgidx] = NULL; + if (bCMA) + { + psDMAVirtArray[uiFreepgidx] = NULL; + psDMAPhysArray[uiFreepgidx] = (dma_addr_t)0; + } + } + else + { + psPMRMapTable->aui32Translation[uiFreepgidx] = uiFreepgidx; + psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx; + psPageArray[uiFreepgidx] = psPage; + if (bCMA) + { + psDMAVirtArray[uiFreepgidx] = pvDMAVAddr; + psDMAPhysArray[uiFreepgidx] = psDMAPAddr; + } + } + } + + /* Free the additional free pages */ + if (0 != ui32AdtnlFreePages) + { + eError = _FreeOSPages(psPMRPageArrayData, + &pai32FreeIndices[ui32Loop], + ui32AdtnlFreePages); + if (eError != PVRSRV_OK) + { + goto e0; + } + psPMRMapTable->ui32NumPhysChunks -= ui32AdtnlFreePages; + while (ui32Loop < ui32FreePageCount) + { + psPMRMapTable->aui32Translation[pai32FreeIndices[ui32Loop]] = TRANSLATION_INVALID; + ui32Loop++; + } + } + + eError = PVRSRV_OK; + +e0: + return eError; +} + +/*************************************************************************/ /*! +@Function PMRChangeSparseMemCPUMapOSMem +@Description This function Changes CPU maps accordingly +@Return PVRSRV_ERROR failure code +*/ /**************************************************************************/ +static +PVRSRV_ERROR PMRChangeSparseMemCPUMapOSMem(PMR_IMPL_PRIVDATA pPriv, + const PMR *psPMR, + IMG_UINT64 sCpuVAddrBase, + IMG_UINT32 ui32AllocPageCount, + IMG_UINT32 *pai32AllocIndices, + IMG_UINT32 ui32FreePageCount, + IMG_UINT32 *pai32FreeIndices) +{ + struct page **psPageArray; + PMR_OSPAGEARRAY_DATA *psPMRPageArrayData = (PMR_OSPAGEARRAY_DATA *)pPriv; + IMG_CPU_PHYADDR sCPUPAddr; + + sCPUPAddr.uiAddr = 0; + psPageArray = psPMRPageArrayData->pagearray; + + return OSChangeSparseMemCPUAddrMap((void **)psPageArray, + sCpuVAddrBase, + sCPUPAddr, + ui32AllocPageCount, + pai32AllocIndices, + ui32FreePageCount, + pai32FreeIndices, + IMG_FALSE); +} + +static PMR_IMPL_FUNCTAB _sPMROSPFuncTab = { + .pfnLockPhysAddresses = &PMRLockSysPhysAddressesOSMem, + .pfnUnlockPhysAddresses = &PMRUnlockSysPhysAddressesOSMem, + .pfnDevPhysAddr = &PMRSysPhysAddrOSMem, + .pfnAcquireKernelMappingData = &PMRAcquireKernelMappingDataOSMem, + .pfnReleaseKernelMappingData = &PMRReleaseKernelMappingDataOSMem, + .pfnReadBytes = NULL, + .pfnWriteBytes = NULL, + .pfnUnpinMem = &PMRUnpinOSMem, + .pfnPinMem = &PMRPinOSMem, + .pfnChangeSparseMem = &PMRChangeSparseMemOSMem, + .pfnChangeSparseMemCPUMap = &PMRChangeSparseMemCPUMapOSMem, + .pfnFinalize = &PMRFinalizeOSMem, +}; + +/* Wrapper around OS page allocation. */ +static PVRSRV_ERROR +DoPageAlloc(PMR_OSPAGEARRAY_DATA *psPrivData, + IMG_UINT32 *puiAllocIndices, + IMG_UINT32 ui32NumPhysChunks, + IMG_UINT32 ui32NumVirtChunks, + IMG_DEVMEM_SIZE_T uiChunkSize, + IMG_UINT32 ui32Log2AllocPageSize) +{ + PVRSRV_ERROR eError = PVRSRV_OK; + + /* Do we fill the whole page array or just parts (sparse)? */ + if (ui32NumPhysChunks == ui32NumVirtChunks) + { + /* Allocate the physical pages */ + eError = _AllocOSPages(psPrivData, + NULL, + psPrivData->uiTotalNumOSPages >> + (ui32Log2AllocPageSize - PAGE_SHIFT)); + } + else if (ui32NumPhysChunks != 0) + { + /* Calculate the number of pages we want to allocate */ + IMG_UINT32 ui32PagesToAlloc = + (IMG_UINT32)((((ui32NumPhysChunks * uiChunkSize) - 1) >> ui32Log2AllocPageSize) + 1); + + /* Make sure calculation is correct */ + PVR_ASSERT(((PMR_SIZE_T) ui32PagesToAlloc << ui32Log2AllocPageSize) == + (ui32NumPhysChunks * uiChunkSize)); + + /* Allocate the physical pages */ + eError = _AllocOSPages(psPrivData, puiAllocIndices, + ui32PagesToAlloc); + } + + return eError; +} + +static void _EncodeAllocationFlags(IMG_UINT32 uiLog2AllocPageSize, + PVRSRV_MEMALLOCFLAGS_T uiFlags, + IMG_UINT32* ui32AllocFlags) +{ + + /* + * Use CMA framework if order is greater than OS page size; please note + * that OSMMapPMRGeneric() has the same expectation as well. + */ + /* IsCMA? */ + if (uiLog2AllocPageSize > PAGE_SHIFT) + { + BIT_SET(*ui32AllocFlags, FLAG_IS_CMA); + } + + /* OnDemand? */ + if (PVRSRV_CHECK_ON_DEMAND(uiFlags)) + { + BIT_SET(*ui32AllocFlags, FLAG_ONDEMAND); + } + + /* Zero? */ + if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags)) + { + BIT_SET(*ui32AllocFlags, FLAG_ZERO); + } + + /* Poison on alloc? */ + if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags)) + { + BIT_SET(*ui32AllocFlags, FLAG_POISON_ON_ALLOC); + } + +#if defined(DEBUG) + /* Poison on free? */ + if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags)) + { + BIT_SET(*ui32AllocFlags, FLAG_POISON_ON_FREE); + } +#endif + + /* Indicate whether this is an allocation with default caching attribute (i.e cached) or not */ + if (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || + PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags)) + { + BIT_SET(*ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE); + } + +} + +void PhysmemGetOSRamMemStats(PHEAP_IMPL_DATA pvImplData, + IMG_UINT64 *pui64TotalSize, + IMG_UINT64 *pui64FreeSize) +{ + struct sysinfo sMeminfo; + si_meminfo(&sMeminfo); + + PVR_UNREFERENCED_PARAMETER(pvImplData); + + *pui64TotalSize = sMeminfo.totalram * sMeminfo.mem_unit; + *pui64FreeSize = sMeminfo.freeram * sMeminfo.mem_unit; + +} + +PVRSRV_ERROR +PhysmemNewOSRamBackedPMR(PHYS_HEAP *psPhysHeap, + CONNECTION_DATA *psConnection, + IMG_DEVMEM_SIZE_T uiSize, + IMG_DEVMEM_SIZE_T uiChunkSize, + IMG_UINT32 ui32NumPhysChunks, + IMG_UINT32 ui32NumVirtChunks, + IMG_UINT32 *puiAllocIndices, + IMG_UINT32 uiLog2AllocPageSize, + PVRSRV_MEMALLOCFLAGS_T uiFlags, + const IMG_CHAR *pszAnnotation, + IMG_PID uiPid, + PMR **ppsPMRPtr, + IMG_UINT32 ui32PDumpFlags) +{ + PVRSRV_ERROR eError; + PVRSRV_ERROR eError2; + PMR *psPMR; + struct _PMR_OSPAGEARRAY_DATA_ *psPrivData; + PMR_FLAGS_T uiPMRFlags; + IMG_UINT32 ui32CPUCacheFlags; + IMG_UINT32 ui32AllocFlags = 0; + PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap); + + PVR_UNREFERENCED_PARAMETER(psConnection); + + /* + * The host driver (but not guest) can still use this factory for firmware + * allocations + */ + if (PVRSRV_VZ_MODE_IS(GUEST) && PVRSRV_CHECK_FW_MAIN(uiFlags)) + { + PVR_ASSERT(0); + eError = PVRSRV_ERROR_INVALID_PARAMS; + goto errorOnParam; + } + + /* Select correct caching mode */ + eError = DevmemCPUCacheMode(psDevNode, uiFlags, &ui32CPUCacheFlags); + if (eError != PVRSRV_OK) + { + goto errorOnParam; + } + + if (PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags)) + { + ui32CPUCacheFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN; + } + + _EncodeAllocationFlags(uiLog2AllocPageSize, uiFlags, &ui32AllocFlags); + + +#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) + /* Overwrite flags and always zero pages that could go back to UM */ + BIT_SET(ui32AllocFlags, FLAG_ZERO); + BIT_UNSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC); +#endif + + /* Physical allocation alignment is generally not supported except under + very restrictive conditions, also there is a maximum alignment value + which must not exceed the largest device page-size. If these are not + met then fail the aligned-requested allocation */ + if (BIT_ISSET(ui32AllocFlags, FLAG_IS_CMA)) + { + IMG_UINT32 uiAlign = 1 << uiLog2AllocPageSize; + if (uiAlign > uiSize || uiAlign > (1 << PVR_MAX_PHYSMEM_CONTIG_ALLOC_LOG2PGSZ)) + { + PVR_DPF((PVR_DBG_ERROR, + "%s: Invalid PA alignment: size 0x%llx, align 0x%x", + __func__, uiSize, uiAlign)); + eError = PVRSRV_ERROR_INVALID_ALIGNMENT; + goto errorOnParam; + } + PVR_ASSERT(uiLog2AllocPageSize > PVR_MIN_PHYSMEM_CONTIG_ALLOC_LOG2PGSZ); + } + + /* Create Array structure that hold the physical pages */ + eError = _AllocOSPageArray(psDevNode, + uiChunkSize, + ui32NumPhysChunks, + ui32NumVirtChunks, + uiLog2AllocPageSize, + ui32AllocFlags, + ui32CPUCacheFlags, + uiPid, + &psPrivData); + if (eError != PVRSRV_OK) + { + goto errorOnAllocPageArray; + } + + if (!BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND)) + { + eError = DoPageAlloc(psPrivData, puiAllocIndices, ui32NumPhysChunks, + ui32NumVirtChunks, uiChunkSize, uiLog2AllocPageSize); + if (eError != PVRSRV_OK) + { + goto errorOnAllocPages; + } + } + + /* + * In this instance, we simply pass flags straight through. + * + * Generically, uiFlags can include things that control the PMR factory, but + * we don't need any such thing (at the time of writing!), and our caller + * specifies all PMR flags so we don't need to meddle with what was given to + * us. + */ + uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK); + + /* + * Check no significant bits were lost in cast due to different bit widths + * for flags + */ + PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK)); + + if (BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND)) + { + PDUMPCOMMENT(PhysHeapDeviceNode(psPhysHeap), "Deferred Allocation PMR (UMA)"); + } + + eError = PMRCreatePMR(psPhysHeap, + uiSize, + uiChunkSize, + ui32NumPhysChunks, + ui32NumVirtChunks, + puiAllocIndices, + uiLog2AllocPageSize, + uiPMRFlags, + pszAnnotation, + &_sPMROSPFuncTab, + psPrivData, + PMR_TYPE_OSMEM, + &psPMR, + ui32PDumpFlags); + if (eError != PVRSRV_OK) + { + goto errorOnCreate; + } + + *ppsPMRPtr = psPMR; + + return PVRSRV_OK; + +errorOnCreate: + if (!BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND)) + { + eError2 = _FreeOSPages(psPrivData, NULL, 0); + PVR_ASSERT(eError2 == PVRSRV_OK); + } + +errorOnAllocPages: + eError2 = _FreeOSPagesArray(psPrivData); + PVR_ASSERT(eError2 == PVRSRV_OK); + +errorOnAllocPageArray: +errorOnParam: + PVR_ASSERT(eError != PVRSRV_OK); + return eError; +} |