summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/vfio/vfio_iommu_spapr_tce.c13
-rw-r--r--drivers/vfio/vfio_iommu_type1.c150
2 files changed, 77 insertions, 86 deletions
diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
index cf3de91fbfe7..78dca1aa6410 100644
--- a/drivers/vfio/vfio_iommu_spapr_tce.c
+++ b/drivers/vfio/vfio_iommu_spapr_tce.c
@@ -198,6 +198,11 @@ static long tce_iommu_register_pages(struct tce_container *container,
return ret;
tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
+ if (!tcemem) {
+ mm_iommu_put(container->mm, mem);
+ return -ENOMEM;
+ }
+
tcemem->mem = mem;
list_add(&tcemem->next, &container->prereg_list);
@@ -1335,8 +1340,16 @@ static int tce_iommu_attach_group(void *iommu_data,
if (!table_group->ops || !table_group->ops->take_ownership ||
!table_group->ops->release_ownership) {
+ if (container->v2) {
+ ret = -EPERM;
+ goto unlock_exit;
+ }
ret = tce_iommu_take_ownership(container, table_group);
} else {
+ if (!container->v2) {
+ ret = -EPERM;
+ goto unlock_exit;
+ }
ret = tce_iommu_take_ownership_ddw(container, table_group);
if (!tce_groups_attached(container) && !container->tables[0])
container->def_window_pending = true;
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 32d2633092a3..8549cb111627 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -246,69 +246,46 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
return ret;
}
-struct vwork {
- struct mm_struct *mm;
- long npage;
- struct work_struct work;
-};
-
-/* delayed decrement/increment for locked_vm */
-static void vfio_lock_acct_bg(struct work_struct *work)
+static int vfio_lock_acct(struct task_struct *task, long npage, bool *lock_cap)
{
- struct vwork *vwork = container_of(work, struct vwork, work);
- struct mm_struct *mm;
-
- mm = vwork->mm;
- down_write(&mm->mmap_sem);
- mm->locked_vm += vwork->npage;
- up_write(&mm->mmap_sem);
- mmput(mm);
- kfree(vwork);
-}
-
-static void vfio_lock_acct(struct task_struct *task, long npage)
-{
- struct vwork *vwork;
struct mm_struct *mm;
bool is_current;
+ int ret;
if (!npage)
- return;
+ return 0;
is_current = (task->mm == current->mm);
mm = is_current ? task->mm : get_task_mm(task);
if (!mm)
- return; /* process exited */
+ return -ESRCH; /* process exited */
- if (down_write_trylock(&mm->mmap_sem)) {
- mm->locked_vm += npage;
- up_write(&mm->mmap_sem);
- if (!is_current)
- mmput(mm);
- return;
- }
+ ret = down_write_killable(&mm->mmap_sem);
+ if (!ret) {
+ if (npage > 0) {
+ if (lock_cap ? !*lock_cap :
+ !has_capability(task, CAP_IPC_LOCK)) {
+ unsigned long limit;
+
+ limit = task_rlimit(task,
+ RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+
+ if (mm->locked_vm + npage > limit)
+ ret = -ENOMEM;
+ }
+ }
+
+ if (!ret)
+ mm->locked_vm += npage;
- if (is_current) {
- mm = get_task_mm(task);
- if (!mm)
- return;
+ up_write(&mm->mmap_sem);
}
- /*
- * Couldn't get mmap_sem lock, so must setup to update
- * mm->locked_vm later. If locked_vm were atomic, we
- * wouldn't need this silliness
- */
- vwork = kmalloc(sizeof(struct vwork), GFP_KERNEL);
- if (WARN_ON(!vwork)) {
+ if (!is_current)
mmput(mm);
- return;
- }
- INIT_WORK(&vwork->work, vfio_lock_acct_bg);
- vwork->mm = mm;
- vwork->npage = npage;
- schedule_work(&vwork->work);
+
+ return ret;
}
/*
@@ -403,10 +380,10 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
* first page and all consecutive pages with the same locking.
*/
static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
- long npage, unsigned long *pfn_base)
+ long npage, unsigned long *pfn_base,
+ bool lock_cap, unsigned long limit)
{
- unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
- bool lock_cap = capable(CAP_IPC_LOCK);
+ unsigned long pfn = 0;
long ret, pinned = 0, lock_acct = 0;
bool rsvd;
dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
@@ -442,8 +419,6 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
/* Lock all the consecutive pages from pfn_base */
for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; pinned < npage;
pinned++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) {
- unsigned long pfn = 0;
-
ret = vaddr_get_pfn(current->mm, vaddr, dma->prot, &pfn);
if (ret)
break;
@@ -460,14 +435,25 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
put_pfn(pfn, dma->prot);
pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
__func__, limit << PAGE_SHIFT);
- break;
+ ret = -ENOMEM;
+ goto unpin_out;
}
lock_acct++;
}
}
out:
- vfio_lock_acct(current, lock_acct);
+ ret = vfio_lock_acct(current, lock_acct, &lock_cap);
+
+unpin_out:
+ if (ret) {
+ if (!rsvd) {
+ for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
+ put_pfn(pfn, dma->prot);
+ }
+
+ return ret;
+ }
return pinned;
}
@@ -488,7 +474,7 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
}
if (do_accounting)
- vfio_lock_acct(dma->task, locked - unlocked);
+ vfio_lock_acct(dma->task, locked - unlocked, NULL);
return unlocked;
}
@@ -496,37 +482,26 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
unsigned long *pfn_base, bool do_accounting)
{
- unsigned long limit;
- bool lock_cap = has_capability(dma->task, CAP_IPC_LOCK);
struct mm_struct *mm;
int ret;
- bool rsvd;
mm = get_task_mm(dma->task);
if (!mm)
return -ENODEV;
ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
- if (ret)
- goto pin_page_exit;
-
- rsvd = is_invalid_reserved_pfn(*pfn_base);
- limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
- if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
- put_pfn(*pfn_base, dma->prot);
- pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK (%ld) exceeded\n",
- __func__, dma->task->comm, task_pid_nr(dma->task),
- limit << PAGE_SHIFT);
- ret = -ENOMEM;
- goto pin_page_exit;
+ if (!ret && do_accounting && !is_invalid_reserved_pfn(*pfn_base)) {
+ ret = vfio_lock_acct(dma->task, 1, NULL);
+ if (ret) {
+ put_pfn(*pfn_base, dma->prot);
+ if (ret == -ENOMEM)
+ pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK "
+ "(%ld) exceeded\n", __func__,
+ dma->task->comm, task_pid_nr(dma->task),
+ task_rlimit(dma->task, RLIMIT_MEMLOCK));
+ }
}
- if (!rsvd && do_accounting)
- vfio_lock_acct(dma->task, 1);
- ret = 1;
-
-pin_page_exit:
mmput(mm);
return ret;
}
@@ -543,7 +518,7 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova,
unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);
if (do_accounting)
- vfio_lock_acct(dma->task, -unlocked);
+ vfio_lock_acct(dma->task, -unlocked, NULL);
return unlocked;
}
@@ -606,10 +581,8 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
remote_vaddr = dma->vaddr + iova - dma->iova;
ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn[i],
do_accounting);
- if (ret <= 0) {
- WARN_ON(!ret);
+ if (ret)
goto pin_unwind;
- }
ret = vfio_add_to_pfn_list(dma, iova, phys_pfn[i]);
if (ret) {
@@ -740,7 +713,7 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
dma->iommu_mapped = false;
if (do_accounting) {
- vfio_lock_acct(dma->task, -unlocked);
+ vfio_lock_acct(dma->task, -unlocked, NULL);
return 0;
}
return unlocked;
@@ -951,13 +924,15 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
unsigned long vaddr = dma->vaddr;
size_t size = map_size;
long npage;
- unsigned long pfn;
+ unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+ bool lock_cap = capable(CAP_IPC_LOCK);
int ret = 0;
while (size) {
/* Pin a contiguous chunk of memory */
npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
- size >> PAGE_SHIFT, &pfn);
+ size >> PAGE_SHIFT, &pfn,
+ lock_cap, limit);
if (npage <= 0) {
WARN_ON(!npage);
ret = (int)npage;
@@ -1067,6 +1042,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
{
struct vfio_domain *d;
struct rb_node *n;
+ unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+ bool lock_cap = capable(CAP_IPC_LOCK);
int ret;
/* Arbitrarily pick the first domain in the list for lookups */
@@ -1113,7 +1090,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
npage = vfio_pin_pages_remote(dma, vaddr,
n >> PAGE_SHIFT,
- &pfn);
+ &pfn, lock_cap,
+ limit);
if (npage <= 0) {
WARN_ON(!npage);
ret = (int)npage;
@@ -1382,7 +1360,7 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)
if (!is_invalid_reserved_pfn(vpfn->pfn))
locked++;
}
- vfio_lock_acct(dma->task, locked - unlocked);
+ vfio_lock_acct(dma->task, locked - unlocked, NULL);
}
}
generated by cgit v1.2.3 (git 2.39.0) at 2024-11-23 18:18:08 +0300