From dc21752e2c930ae32f11affb2e1d7a6f8a44a16a Mon Sep 17 00:00:00 2001
From: Dennis Dalessandro <dennis.dalessandro@intel.com>
Date: Wed, 6 Jan 2016 09:59:04 -0800
Subject: IB/rdmavt: Add mmap stub

Adds the stub for the mmap verbs call.

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Reviewed-by: Ira Weiny <ira.weiny@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
---
 drivers/infiniband/sw/rdmavt/mmap.c | 60 +++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 drivers/infiniband/sw/rdmavt/mmap.c

(limited to 'drivers/infiniband/sw/rdmavt/mmap.c')

diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
new file mode 100644
index 000000000000..d09f3a05a747
--- /dev/null
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -0,0 +1,60 @@
+/*
+ * Copyright(c) 2015 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <linux/slab.h>
+#include "mmap.h"
+
+/**
+ * rvt_mmap - create a new mmap region
+ * @context: the IB user context of the process making the mmap() call
+ * @vma: the VMA to be initialized
+ * Return zero if the mmap is OK. Otherwise, return an errno.
+ */
+int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
+{
+	return -EOPNOTSUPP;
+}
-- 
cgit v1.2.3


From 822514d75a9647662fff39d728c1f4636b75d904 Mon Sep 17 00:00:00 2001
From: Dennis Dalessandro <dennis.dalessandro@intel.com>
Date: Wed, 6 Jan 2016 10:04:57 -0800
Subject: IB/rdmavt: Add mmap related functions

The mmap data structure was moved in a previous commit. This patch now
pulls in the related functions.

Reviewed-by: Ira Weiny <ira.weiny@intel.com>
Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
---
 drivers/infiniband/sw/rdmavt/mmap.c | 140 +++++++++++++++++++++++++++++++++++-
 drivers/infiniband/sw/rdmavt/mmap.h |   2 +-
 drivers/infiniband/sw/rdmavt/vt.c   |   1 +
 include/rdma/rdma_vt.h              |  15 ++++
 4 files changed, 156 insertions(+), 2 deletions(-)

(limited to 'drivers/infiniband/sw/rdmavt/mmap.c')

diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
index d09f3a05a747..fc30ff7f24ea 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.c
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -46,8 +46,61 @@
  */
 
 #include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <asm/pgtable.h>
 #include "mmap.h"
 
+void rvt_mmap_init(struct rvt_dev_info *rdi)
+{
+	INIT_LIST_HEAD(&rdi->pending_mmaps);
+	spin_lock_init(&rdi->pending_lock);
+	rdi->mmap_offset = PAGE_SIZE;
+	spin_lock_init(&rdi->mmap_offset_lock);
+}
+
+/**
+ * rvt_release_mmap_info - free mmap info structure
+ * @ref: a pointer to the kref within struct rvt_mmap_info
+ */
+void rvt_release_mmap_info(struct kref *ref)
+{
+	struct rvt_mmap_info *ip =
+		container_of(ref, struct rvt_mmap_info, ref);
+	struct rvt_dev_info *rdi = ib_to_rvt(ip->context->device);
+
+	spin_lock_irq(&rdi->pending_lock);
+	list_del(&ip->pending_mmaps);
+	spin_unlock_irq(&rdi->pending_lock);
+
+	vfree(ip->obj);
+	kfree(ip);
+}
+EXPORT_SYMBOL(rvt_release_mmap_info);
+
+/*
+ * open and close keep track of how many times the CQ is mapped,
+ * to avoid releasing it.
+ */
+static void rvt_vma_open(struct vm_area_struct *vma)
+{
+	struct rvt_mmap_info *ip = vma->vm_private_data;
+
+	kref_get(&ip->ref);
+}
+
+static void rvt_vma_close(struct vm_area_struct *vma)
+{
+	struct rvt_mmap_info *ip = vma->vm_private_data;
+
+	kref_put(&ip->ref, rvt_release_mmap_info);
+}
+
+static const struct vm_operations_struct rvt_vm_ops = {
+	.open = rvt_vma_open,
+	.close = rvt_vma_close,
+};
+
 /**
  * rvt_mmap - create a new mmap region
  * @context: the IB user context of the process making the mmap() call
@@ -56,5 +109,90 @@
  */
 int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 {
-	return -EOPNOTSUPP;
+	struct rvt_dev_info *rdi = ib_to_rvt(context->device);
+	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+	unsigned long size = vma->vm_end - vma->vm_start;
+	struct rvt_mmap_info *ip, *pp;
+	int ret = -EINVAL;
+
+	/*
+	 * Search the device's list of objects waiting for a mmap call.
+	 * Normally, this list is very short since a call to create a
+	 * CQ, QP, or SRQ is soon followed by a call to mmap().
+	 */
+	spin_lock_irq(&rdi->pending_lock);
+	list_for_each_entry_safe(ip, pp, &rdi->pending_mmaps,
+				 pending_mmaps) {
+		/* Only the creator is allowed to mmap the object */
+		if (context != ip->context || (__u64)offset != ip->offset)
+			continue;
+		/* Don't allow a mmap larger than the object. */
+		if (size > ip->size)
+			break;
+
+		list_del_init(&ip->pending_mmaps);
+		spin_unlock_irq(&rdi->pending_lock);
+
+		ret = remap_vmalloc_range(vma, ip->obj, 0);
+		if (ret)
+			goto done;
+		vma->vm_ops = &rvt_vm_ops;
+		vma->vm_private_data = ip;
+		rvt_vma_open(vma);
+		goto done;
+	}
+	spin_unlock_irq(&rdi->pending_lock);
+done:
+	return ret;
+}
+EXPORT_SYMBOL(rvt_mmap);
+
+/*
+ * Allocate information for hfi1_mmap
+ */
+struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
+					   u32 size,
+					   struct ib_ucontext *context,
+					   void *obj)
+{
+	struct rvt_mmap_info *ip;
+
+	ip = kmalloc(sizeof(*ip), GFP_KERNEL);
+	if (!ip)
+		return ip;
+
+	size = PAGE_ALIGN(size);
+
+	spin_lock_irq(&rdi->mmap_offset_lock);
+	if (rdi->mmap_offset == 0)
+		rdi->mmap_offset = PAGE_SIZE;
+	ip->offset = rdi->mmap_offset;
+	rdi->mmap_offset += size;
+	spin_unlock_irq(&rdi->mmap_offset_lock);
+
+	INIT_LIST_HEAD(&ip->pending_mmaps);
+	ip->size = size;
+	ip->context = context;
+	ip->obj = obj;
+	kref_init(&ip->ref);
+
+	return ip;
+}
+EXPORT_SYMBOL(rvt_create_mmap_info);
+
+void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
+			  u32 size, void *obj)
+{
+	size = PAGE_ALIGN(size);
+
+	spin_lock_irq(&rdi->mmap_offset_lock);
+	if (rdi->mmap_offset == 0)
+		rdi->mmap_offset = PAGE_SIZE;
+	ip->offset = rdi->mmap_offset;
+	rdi->mmap_offset += size;
+	spin_unlock_irq(&rdi->mmap_offset_lock);
+
+	ip->size = size;
+	ip->obj = obj;
 }
+EXPORT_SYMBOL(rvt_update_mmap_info);
diff --git a/drivers/infiniband/sw/rdmavt/mmap.h b/drivers/infiniband/sw/rdmavt/mmap.h
index 94f6377cef3c..3513e25a8491 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.h
+++ b/drivers/infiniband/sw/rdmavt/mmap.h
@@ -50,6 +50,6 @@
 
 #include <rdma/rdma_vt.h>
 
-int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+void rvt_mmap_init(struct rvt_dev_info *rdi);
 
 #endif          /* DEF_RDMAVTMMAP_H */
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index f2d995d2f62c..ab4105a8d120 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -241,6 +241,7 @@ int rvt_register_device(struct rvt_dev_info *rdi)
 	}
 
 	/* Once we get past here we can use the rvt_pr macros */
+	rvt_mmap_init(rdi);
 
 	/* Dev Ops */
 	CHECK_DRIVER_OVERRIDE(rdi, query_device);
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index 950c2910e3f4..fd25d2309ee3 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -254,6 +254,12 @@ struct rvt_dev_info {
 	struct rvt_ibport **ports;
 
 	struct rvt_qp_ibdev *qp_dev;
+
+	/* memory maps */
+	struct list_head pending_mmaps;
+	spinlock_t mmap_offset_lock; /* protect mmap_offset */
+	u32 mmap_offset;
+	spinlock_t pending_lock; /* protect pending mmap list */
 };
 
 static inline struct rvt_pd *ibpd_to_rvtpd(struct ib_pd *ibpd)
@@ -285,4 +291,13 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
 		u32 len, u64 vaddr, u32 rkey, int acc);
 int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 		struct rvt_sge *isge, struct ib_sge *sge, int acc);
+int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+void rvt_release_mmap_info(struct kref *ref);
+struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
+					   u32 size,
+					   struct ib_ucontext *context,
+					   void *obj);
+void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
+			  u32 size, void *obj);
+
 #endif          /* DEF_RDMA_VT_H */
-- 
cgit v1.2.3


From fe31419501ba133a967da7b7da0d32945ef21840 Mon Sep 17 00:00:00 2001
From: Dennis Dalessandro <dennis.dalessandro@intel.com>
Date: Fri, 22 Jan 2016 13:04:58 -0800
Subject: IB/rdmavt: Fix copyright date

Update all files added by rdmavt which do not yet have 2016 as the
copyright year.

Reviewed-by: Ira Weiny <ira.weiny@intel.com>
Reviewed-by: Harish Chegondi <harish.chegondi@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
---
 drivers/infiniband/sw/rdmavt/ah.c    | 2 +-
 drivers/infiniband/sw/rdmavt/ah.h    | 2 +-
 drivers/infiniband/sw/rdmavt/cq.c    | 2 +-
 drivers/infiniband/sw/rdmavt/cq.h    | 2 +-
 drivers/infiniband/sw/rdmavt/dma.c   | 2 +-
 drivers/infiniband/sw/rdmavt/dma.h   | 2 +-
 drivers/infiniband/sw/rdmavt/mad.c   | 2 +-
 drivers/infiniband/sw/rdmavt/mad.h   | 2 +-
 drivers/infiniband/sw/rdmavt/mcast.c | 2 +-
 drivers/infiniband/sw/rdmavt/mcast.h | 2 +-
 drivers/infiniband/sw/rdmavt/mmap.c  | 2 +-
 drivers/infiniband/sw/rdmavt/mmap.h  | 2 +-
 drivers/infiniband/sw/rdmavt/mr.c    | 2 +-
 drivers/infiniband/sw/rdmavt/mr.h    | 2 +-
 drivers/infiniband/sw/rdmavt/pd.c    | 2 +-
 drivers/infiniband/sw/rdmavt/pd.h    | 2 +-
 drivers/infiniband/sw/rdmavt/qp.c    | 2 +-
 drivers/infiniband/sw/rdmavt/qp.h    | 2 +-
 drivers/infiniband/sw/rdmavt/srq.c   | 2 +-
 drivers/infiniband/sw/rdmavt/srq.h   | 2 +-
 drivers/infiniband/sw/rdmavt/trace.c | 2 +-
 drivers/infiniband/sw/rdmavt/trace.h | 2 +-
 drivers/infiniband/sw/rdmavt/vt.c    | 2 +-
 drivers/infiniband/sw/rdmavt/vt.h    | 2 +-
 include/rdma/rdma_vt.h               | 2 +-
 include/rdma/rdmavt_mr.h             | 2 +-
 include/rdma/rdmavt_qp.h             | 2 +-
 27 files changed, 27 insertions(+), 27 deletions(-)

(limited to 'drivers/infiniband/sw/rdmavt/mmap.c')

diff --git a/drivers/infiniband/sw/rdmavt/ah.c b/drivers/infiniband/sw/rdmavt/ah.c
index c194d9d9bd25..9372c4321858 100644
--- a/drivers/infiniband/sw/rdmavt/ah.c
+++ b/drivers/infiniband/sw/rdmavt/ah.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/ah.h b/drivers/infiniband/sw/rdmavt/ah.h
index 8cd7ea7303e4..e9c36be87d79 100644
--- a/drivers/infiniband/sw/rdmavt/ah.h
+++ b/drivers/infiniband/sw/rdmavt/ah.h
@@ -2,7 +2,7 @@
 #define DEF_RVTAH_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index 7308a274643d..055aa71bed18 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/cq.h b/drivers/infiniband/sw/rdmavt/cq.h
index 3813d90efad2..6182c29eff66 100644
--- a/drivers/infiniband/sw/rdmavt/cq.h
+++ b/drivers/infiniband/sw/rdmavt/cq.h
@@ -2,7 +2,7 @@
 #define DEF_RVTCQ_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/dma.c b/drivers/infiniband/sw/rdmavt/dma.c
index c0701416136a..33076a5eee2f 100644
--- a/drivers/infiniband/sw/rdmavt/dma.c
+++ b/drivers/infiniband/sw/rdmavt/dma.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/dma.h b/drivers/infiniband/sw/rdmavt/dma.h
index a80cc3556915..979f07e09195 100644
--- a/drivers/infiniband/sw/rdmavt/dma.h
+++ b/drivers/infiniband/sw/rdmavt/dma.h
@@ -2,7 +2,7 @@
 #define DEF_RDMAVTDMA_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/mad.c b/drivers/infiniband/sw/rdmavt/mad.c
index e01f3fb47c5c..5c720d35304d 100644
--- a/drivers/infiniband/sw/rdmavt/mad.c
+++ b/drivers/infiniband/sw/rdmavt/mad.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/mad.h b/drivers/infiniband/sw/rdmavt/mad.h
index 5d8a6a9f1f8c..c89faf411d09 100644
--- a/drivers/infiniband/sw/rdmavt/mad.h
+++ b/drivers/infiniband/sw/rdmavt/mad.h
@@ -2,7 +2,7 @@
 #define DEF_RVTMAD_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/mcast.c b/drivers/infiniband/sw/rdmavt/mcast.c
index 528c1ca798a9..e06a8755cbef 100644
--- a/drivers/infiniband/sw/rdmavt/mcast.c
+++ b/drivers/infiniband/sw/rdmavt/mcast.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/mcast.h b/drivers/infiniband/sw/rdmavt/mcast.h
index cd15a981d7bf..29f579267608 100644
--- a/drivers/infiniband/sw/rdmavt/mcast.h
+++ b/drivers/infiniband/sw/rdmavt/mcast.h
@@ -2,7 +2,7 @@
 #define DEF_RVTMCAST_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
index fc30ff7f24ea..d6330d7b4405 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.c
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/mmap.h b/drivers/infiniband/sw/rdmavt/mmap.h
index 3513e25a8491..e8067471c722 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.h
+++ b/drivers/infiniband/sw/rdmavt/mmap.h
@@ -2,7 +2,7 @@
 #define DEF_RDMAVTMMAP_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index f1dcaf4d6c3e..ee36be37c55d 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/mr.h b/drivers/infiniband/sw/rdmavt/mr.h
index c5339aa2d241..69380512c6d1 100644
--- a/drivers/infiniband/sw/rdmavt/mr.h
+++ b/drivers/infiniband/sw/rdmavt/mr.h
@@ -2,7 +2,7 @@
 #define DEF_RVTMR_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/pd.c b/drivers/infiniband/sw/rdmavt/pd.c
index f8dba88880e5..62fee44be3a3 100644
--- a/drivers/infiniband/sw/rdmavt/pd.c
+++ b/drivers/infiniband/sw/rdmavt/pd.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/pd.h b/drivers/infiniband/sw/rdmavt/pd.h
index 56d75e6cb21f..1892ca4a9746 100644
--- a/drivers/infiniband/sw/rdmavt/pd.h
+++ b/drivers/infiniband/sw/rdmavt/pd.h
@@ -2,7 +2,7 @@
 #define DEF_RDMAVTPD_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 615358ec394d..8d3563a4249c 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/qp.h b/drivers/infiniband/sw/rdmavt/qp.h
index f438809e18e2..8409f80d5f25 100644
--- a/drivers/infiniband/sw/rdmavt/qp.h
+++ b/drivers/infiniband/sw/rdmavt/qp.h
@@ -2,7 +2,7 @@
 #define DEF_RVTQP_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c
index bbb623a5f679..c9eb8b33cae3 100644
--- a/drivers/infiniband/sw/rdmavt/srq.c
+++ b/drivers/infiniband/sw/rdmavt/srq.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/srq.h b/drivers/infiniband/sw/rdmavt/srq.h
index 0c3c5a7e64cb..9f07880e9e07 100644
--- a/drivers/infiniband/sw/rdmavt/srq.h
+++ b/drivers/infiniband/sw/rdmavt/srq.h
@@ -2,7 +2,7 @@
 #define DEF_RVTSRQ_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/trace.c b/drivers/infiniband/sw/rdmavt/trace.c
index 19afe3988f67..d593285a349c 100644
--- a/drivers/infiniband/sw/rdmavt/trace.c
+++ b/drivers/infiniband/sw/rdmavt/trace.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/trace.h b/drivers/infiniband/sw/rdmavt/trace.h
index b269291b6dc9..d5b128118b73 100644
--- a/drivers/infiniband/sw/rdmavt/trace.h
+++ b/drivers/infiniband/sw/rdmavt/trace.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 7496d43685ab..571463eca65c 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/drivers/infiniband/sw/rdmavt/vt.h b/drivers/infiniband/sw/rdmavt/vt.h
index d9f78ccf1c35..a5c36d32fa4d 100644
--- a/drivers/infiniband/sw/rdmavt/vt.h
+++ b/drivers/infiniband/sw/rdmavt/vt.h
@@ -2,7 +2,7 @@
 #define DEF_RDMAVT_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index 31f9e5a08da0..f6569b24497f 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -2,7 +2,7 @@
 #define DEF_RDMA_VT_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/include/rdma/rdmavt_mr.h b/include/rdma/rdmavt_mr.h
index 4aa81713b4f3..5edffdca8c53 100644
--- a/include/rdma/rdmavt_mr.h
+++ b/include/rdma/rdmavt_mr.h
@@ -2,7 +2,7 @@
 #define DEF_RDMAVT_INCMR_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h
index f0e24266bdb4..91f20fd91e00 100644
--- a/include/rdma/rdmavt_qp.h
+++ b/include/rdma/rdmavt_qp.h
@@ -2,7 +2,7 @@
 #define DEF_RDMAVT_INCQP_H
 
 /*
- * Copyright(c) 2015 Intel Corporation.
+ * Copyright(c) 2016 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
-- 
cgit v1.2.3


From d1b697b678cd591e12c493a9b91343107816cceb Mon Sep 17 00:00:00 2001
From: Mitko Haralanov <mitko.haralanov@intel.com>
Date: Wed, 3 Feb 2016 14:14:54 -0800
Subject: IB/rdmavt: Add Mem affinity support

Change verbs memory allocations to the device numa node.  This keeps memory
close to the device for optimal performance.

Reviewed-by: Dean Luick <dean.luick@intel.com>
Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Reviewed-by: Ira Weiny <ira.weiny@intel.com>
Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Mitko Haralanov <mitko.haralanov@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
---
 drivers/infiniband/sw/rdmavt/mmap.c |  2 +-
 drivers/infiniband/sw/rdmavt/mr.c   |  2 +-
 drivers/infiniband/sw/rdmavt/qp.c   | 21 ++++++++++++---------
 3 files changed, 14 insertions(+), 11 deletions(-)

(limited to 'drivers/infiniband/sw/rdmavt/mmap.c')

diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
index d6330d7b4405..49180c4eb76e 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.c
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -157,7 +157,7 @@ struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
 {
 	struct rvt_mmap_info *ip;
 
-	ip = kmalloc(sizeof(*ip), GFP_KERNEL);
+	ip = kmalloc_node(sizeof(*ip), GFP_KERNEL, rdi->dparms.node);
 	if (!ip)
 		return ip;
 
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index ee36be37c55d..8bff6bbfece2 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -87,7 +87,7 @@ int rvt_driver_mr_init(struct rvt_dev_info *rdi)
 	}
 	lk_tab_size = rdi->lkey_table.max * sizeof(*rdi->lkey_table.table);
 	rdi->lkey_table.table = (struct rvt_mregion __rcu **)
-			       vmalloc(lk_tab_size);
+			       vmalloc_node(lk_tab_size, rdi->dparms.node);
 	if (!rdi->lkey_table.table)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index e9e3138d1203..471d9c59f765 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -186,7 +186,8 @@ int rvt_driver_qp_init(struct rvt_dev_info *rdi)
 		return -EINVAL;
 
 	/* allocate parent object */
-	rdi->qp_dev = kzalloc(sizeof(*rdi->qp_dev), GFP_KERNEL);
+	rdi->qp_dev = kzalloc_node(sizeof(*rdi->qp_dev), GFP_KERNEL,
+				   rdi->dparms.node);
 	if (!rdi->qp_dev)
 		return -ENOMEM;
 
@@ -194,9 +195,9 @@ int rvt_driver_qp_init(struct rvt_dev_info *rdi)
 	rdi->qp_dev->qp_table_size = rdi->dparms.qp_table_size;
 	rdi->qp_dev->qp_table_bits = ilog2(rdi->dparms.qp_table_size);
 	rdi->qp_dev->qp_table =
-		kmalloc(rdi->qp_dev->qp_table_size *
-			sizeof(*rdi->qp_dev->qp_table),
-			GFP_KERNEL);
+		kmalloc_node(rdi->qp_dev->qp_table_size *
+			     sizeof(*rdi->qp_dev->qp_table),
+			     GFP_KERNEL, rdi->dparms.node);
 	if (!rdi->qp_dev->qp_table)
 		goto no_qp_table;
 
@@ -542,8 +543,9 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 				(init_attr->cap.max_send_wr + 1) * sz,
 				gfp, PAGE_KERNEL);
 		else
-			swq = vmalloc(
-				(init_attr->cap.max_send_wr + 1) * sz);
+			swq = vmalloc_node(
+				(init_attr->cap.max_send_wr + 1) * sz,
+				rdi->dparms.node);
 		if (!swq)
 			return ERR_PTR(-ENOMEM);
 
@@ -558,7 +560,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 		} else if (init_attr->cap.max_recv_sge > 1)
 			sg_list_sz = sizeof(*qp->r_sg_list) *
 				(init_attr->cap.max_recv_sge - 1);
-		qp = kzalloc(sz + sg_list_sz, gfp);
+		qp = kzalloc_node(sz + sg_list_sz, gfp, rdi->dparms.node);
 		if (!qp)
 			goto bail_swq;
 
@@ -592,9 +594,10 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 						qp->r_rq.size * sz,
 						gfp, PAGE_KERNEL);
 			else
-				qp->r_rq.wq = vmalloc(
+				qp->r_rq.wq = vmalloc_node(
 						sizeof(struct rvt_rwq) +
-						qp->r_rq.size * sz);
+						qp->r_rq.size * sz,
+						rdi->dparms.node);
 			if (!qp->r_rq.wq)
 				goto bail_driver_priv;
 		}
-- 
cgit v1.2.3


From 90793f7179478df19ac4b2244cfd9764b28e4b38 Mon Sep 17 00:00:00 2001
From: Dennis Dalessandro <dennis.dalessandro@intel.com>
Date: Sun, 14 Feb 2016 12:10:29 -0800
Subject: IB/rdmavt: Clean up comments and add more documentation

Add, remove, and otherwise clean up existing comments that are leftover
from the initial code postings of rdmavt. Many of the comments were added
to provide an idea on the direction we were thinking of going. Now that the
design is solidified make a pass over and clean everything up. Also add
details where lacking.

Ensure all non static functions have nano comments.

Reviewed-by: Jubin John <jubin.john@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
---
 drivers/infiniband/sw/rdmavt/ah.c    | 27 ++++++++++++++++
 drivers/infiniband/sw/rdmavt/cq.c    | 30 +++++++++++------
 drivers/infiniband/sw/rdmavt/mad.c   | 30 +++++++++++------
 drivers/infiniband/sw/rdmavt/mcast.c | 34 ++++++++++++++++++--
 drivers/infiniband/sw/rdmavt/mmap.c  | 28 ++++++++++++----
 drivers/infiniband/sw/rdmavt/mr.c    | 38 ++++++++++++++--------
 drivers/infiniband/sw/rdmavt/pd.c    | 16 ++++++++++
 drivers/infiniband/sw/rdmavt/qp.c    | 57 +++++++++++++++++++++++++++++----
 drivers/infiniband/sw/rdmavt/srq.c   | 21 +++++++++++-
 drivers/infiniband/sw/rdmavt/vt.c    | 62 ++++++++++++++++++++++++++++--------
 include/rdma/rdma_vt.h               | 15 ---------
 11 files changed, 280 insertions(+), 78 deletions(-)

(limited to 'drivers/infiniband/sw/rdmavt/mmap.c')

diff --git a/drivers/infiniband/sw/rdmavt/ah.c b/drivers/infiniband/sw/rdmavt/ah.c
index 9372c4321858..16c446142c2a 100644
--- a/drivers/infiniband/sw/rdmavt/ah.c
+++ b/drivers/infiniband/sw/rdmavt/ah.c
@@ -53,6 +53,11 @@
  * rvt_check_ah - validate the attributes of AH
  * @ibdev: the ib device
  * @ah_attr: the attributes of the AH
+ *
+ * If driver supports a more detailed check_ah function call back to it
+ * otherwise just check the basics.
+ *
+ * Return: 0 on success
  */
 int rvt_check_ah(struct ib_device *ibdev,
 		 struct ib_ah_attr *ah_attr)
@@ -95,6 +100,8 @@ EXPORT_SYMBOL(rvt_check_ah);
  * @ah_attr: the attributes of the AH
  *
  * This may be called from interrupt context.
+ *
+ * Return: newly allocated ah
  */
 struct ib_ah *rvt_create_ah(struct ib_pd *pd,
 			    struct ib_ah_attr *ah_attr)
@@ -129,6 +136,12 @@ struct ib_ah *rvt_create_ah(struct ib_pd *pd,
 	return &ah->ibah;
 }
 
+/**
+ * rvt_destory_ah - Destory an address handle
+ * @ibah: address handle
+ *
+ * Return: 0 on success
+ */
 int rvt_destroy_ah(struct ib_ah *ibah)
 {
 	struct rvt_dev_info *dev = ib_to_rvt(ibah->device);
@@ -147,6 +160,13 @@ int rvt_destroy_ah(struct ib_ah *ibah)
 	return 0;
 }
 
+/**
+ * rvt_modify_ah - modify an ah with given attrs
+ * @ibah: address handle to modify
+ * @ah_attr: attrs to apply
+ *
+ * Return: 0 on success
+ */
 int rvt_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
 {
 	struct rvt_ah *ah = ibah_to_rvtah(ibah);
@@ -159,6 +179,13 @@ int rvt_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
 	return 0;
 }
 
+/**
+ * rvt_query_ah - return attrs for ah
+ * @ibah: address handle to query
+ * @ah_attr: return info in this
+ *
+ * Return: always 0
+ */
 int rvt_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
 {
 	struct rvt_ah *ah = ibah_to_rvtah(ibah);
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index 0e6dbe5904ff..c69c0709696a 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -173,10 +173,10 @@ static void send_complete(struct kthread_work *work)
  * @context: unused by the QLogic_IB driver
  * @udata: user data for libibverbs.so
  *
- * Returns a pointer to the completion queue or negative errno values
- * for failure.
- *
  * Called by ib_create_cq() in the generic verbs code.
+ *
+ * Return: pointer to the completion queue or negative errno values
+ * for failure.
  */
 struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 			    const struct ib_cq_init_attr *attr,
@@ -286,9 +286,9 @@ done:
  * rvt_destroy_cq - destroy a completion queue
  * @ibcq: the completion queue to destroy.
  *
- * Returns 0 for success.
- *
  * Called by ib_destroy_cq() in the generic verbs code.
+ *
+ * Return: always 0
  */
 int rvt_destroy_cq(struct ib_cq *ibcq)
 {
@@ -313,10 +313,10 @@ int rvt_destroy_cq(struct ib_cq *ibcq)
  * @ibcq: the completion queue
  * @notify_flags: the type of notification to request
  *
- * Returns 0 for success.
- *
  * This may be called from interrupt context.  Also called by
  * ib_req_notify_cq() in the generic verbs code.
+ *
+ * Return: 0 for success.
  */
 int rvt_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags)
 {
@@ -345,7 +345,7 @@ int rvt_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags)
  * rvt_resize_cq - change the size of the CQ
  * @ibcq: the completion queue
  *
- * Returns 0 for success.
+ * Return: 0 for success.
  */
 int rvt_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
 {
@@ -456,10 +456,10 @@ bail_free:
  * @num_entries: the maximum number of entries to return
  * @entry: pointer to array where work completions are placed
  *
- * Returns the number of completion entries polled.
- *
  * This may be called from interrupt context.  Also called by ib_poll_cq()
  * in the generic verbs code.
+ *
+ * Return: the number of completion entries polled.
  */
 int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
 {
@@ -496,6 +496,12 @@ int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
 	return npolled;
 }
 
+/**
+ * rvt_driver_cq_init - Init cq resources on behalf of driver
+ * @rdi: rvt dev structure
+ *
+ * Return: 0 on success
+ */
 int rvt_driver_cq_init(struct rvt_dev_info *rdi)
 {
 	int ret = 0;
@@ -530,6 +536,10 @@ int rvt_driver_cq_init(struct rvt_dev_info *rdi)
 	return ret;
 }
 
+/**
+ * rvt_cq_exit - tear down cq reources
+ * @rdi: rvt dev structure
+ */
 void rvt_cq_exit(struct rvt_dev_info *rdi)
 {
 	struct kthread_worker *worker;
diff --git a/drivers/infiniband/sw/rdmavt/mad.c b/drivers/infiniband/sw/rdmavt/mad.c
index 2feae47492df..f6e99778d7ca 100644
--- a/drivers/infiniband/sw/rdmavt/mad.c
+++ b/drivers/infiniband/sw/rdmavt/mad.c
@@ -59,14 +59,13 @@
  * @in_mad: the incoming MAD
  * @out_mad: any outgoing MAD reply
  *
- * Returns IB_MAD_RESULT_SUCCESS if this is a MAD that we are not
- * interested in processing.
- *
  * Note that the verbs framework has already done the MAD sanity checks,
  * and hop count/pointer updating for IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
  * MADs.
  *
  * This is called by the ib_mad module.
+ *
+ * Return: IB_MAD_RESULT_SUCCESS or error
  */
 int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,
@@ -75,13 +74,10 @@ int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 		    u16 *out_mad_pkey_index)
 {
 	/*
-	 * Drivers will need to provide a number of things. For exmaple counters
-	 * will need to be maintained by the driver but shoud live in the rvt
-	 * structure. More study will be needed to finalize the interface
-	 * between drivers and rvt for mad packets.
-	 *
-	 *VT-DRIVER-API: ????
-	 *
+	 * MAD processing is quite different between hfi1 and qib. Therfore this
+	 * is expected to be provided by the driver. Other drivers in the future
+	 * may chose to implement this but it should not be made into a
+	 * requirement.
 	 */
 	if (ibport_num_to_idx(ibdev, port_num) < 0)
 		return -EINVAL;
@@ -95,6 +91,14 @@ static void rvt_send_mad_handler(struct ib_mad_agent *agent,
 	ib_free_send_mad(mad_send_wc->send_buf);
 }
 
+/**
+ * rvt_create_mad_agents - create mad agents
+ * @rdi: rvt dev struct
+ *
+ * If driver needs to be notified of mad agent creation then call back
+ *
+ * Return 0 on success
+ */
 int rvt_create_mad_agents(struct rvt_dev_info *rdi)
 {
 	struct ib_mad_agent *agent;
@@ -136,6 +140,12 @@ err:
 	return ret;
 }
 
+/**
+ * rvt_free_mad_agents - free up mad agents
+ * @rdi: rvt dev struct
+ *
+ * If driver needs notification of mad agent removal make the call back
+ */
 void rvt_free_mad_agents(struct rvt_dev_info *rdi)
 {
 	struct ib_mad_agent *agent;
diff --git a/drivers/infiniband/sw/rdmavt/mcast.c b/drivers/infiniband/sw/rdmavt/mcast.c
index e06a8755cbef..983d319ac976 100644
--- a/drivers/infiniband/sw/rdmavt/mcast.c
+++ b/drivers/infiniband/sw/rdmavt/mcast.c
@@ -53,6 +53,12 @@
 
 #include "mcast.h"
 
+/**
+ * rvt_driver_mcast - init resources for multicast
+ * @rdi: rvt dev struct
+ *
+ * This is per device that registers with rdmavt
+ */
 void rvt_driver_mcast_init(struct rvt_dev_info *rdi)
 {
 	/*
@@ -130,9 +136,9 @@ static void rvt_mcast_free(struct rvt_mcast *mcast)
  * @ibp: the IB port structure
  * @mgid: the multicast GID to search for
  *
- * Returns NULL if not found.
- *
  * The caller is responsible for decrementing the reference count if found.
+ *
+ * Return: NULL if not found.
  */
 struct rvt_mcast *rvt_mcast_find(struct rvt_ibport *ibp, union ib_gid *mgid)
 {
@@ -170,7 +176,7 @@ EXPORT_SYMBOL(rvt_mcast_find);
  * @mcast: the mcast GID table
  * @mqp: the QP to attach
  *
- * Return zero if both were added.  Return EEXIST if the GID was already in
+ * Return: zero if both were added.  Return EEXIST if the GID was already in
  * the table but the QP was added.  Return ESRCH if the QP was already
  * attached and neither structure was added.
  */
@@ -247,6 +253,14 @@ bail:
 	return ret;
 }
 
+/**
+ * rvt_attach_mcast - attach a qp to a multicast group
+ * @ibqp: Infiniband qp
+ * @igd: multicast guid
+ * @lid: multicast lid
+ *
+ * Return: 0 on success
+ */
 int rvt_attach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 {
 	struct rvt_qp *qp = ibqp_to_rvtqp(ibqp);
@@ -298,6 +312,14 @@ bail_mcast:
 	return ret;
 }
 
+/**
+ * rvt_detach_mcast - remove a qp from a multicast group
+ * @ibqp: Infiniband qp
+ * @igd: multicast guid
+ * @lid: multicast lid
+ *
+ * Return: 0 on success
+ */
 int rvt_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 {
 	struct rvt_qp *qp = ibqp_to_rvtqp(ibqp);
@@ -377,6 +399,12 @@ int rvt_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 	return 0;
 }
 
+/**
+ *rvt_mast_tree_empty - determine if any qps are attached to any mcast group
+ *@rdi: rvt dev struct
+ *
+ * Return: in use count
+ */
 int rvt_mcast_tree_empty(struct rvt_dev_info *rdi)
 {
 	int i;
diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
index 49180c4eb76e..273974fb7d1f 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.c
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -51,6 +51,10 @@
 #include <asm/pgtable.h>
 #include "mmap.h"
 
+/**
+ * rvt_mmap_init - init link list and lock for mem map
+ * @rdi: rvt dev struct
+ */
 void rvt_mmap_init(struct rvt_dev_info *rdi)
 {
 	INIT_LIST_HEAD(&rdi->pending_mmaps);
@@ -78,10 +82,6 @@ void rvt_release_mmap_info(struct kref *ref)
 }
 EXPORT_SYMBOL(rvt_release_mmap_info);
 
-/*
- * open and close keep track of how many times the CQ is mapped,
- * to avoid releasing it.
- */
 static void rvt_vma_open(struct vm_area_struct *vma)
 {
 	struct rvt_mmap_info *ip = vma->vm_private_data;
@@ -105,7 +105,8 @@ static const struct vm_operations_struct rvt_vm_ops = {
  * rvt_mmap - create a new mmap region
  * @context: the IB user context of the process making the mmap() call
  * @vma: the VMA to be initialized
- * Return zero if the mmap is OK. Otherwise, return an errno.
+ *
+ * Return: zero if the mmap is OK. Otherwise, return an errno.
  */
 int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 {
@@ -147,8 +148,14 @@ done:
 }
 EXPORT_SYMBOL(rvt_mmap);
 
-/*
- * Allocate information for hfi1_mmap
+/**
+ * rvt_create_mmap_info - allocate information for hfi1_mmap
+ * @rdi: rvt dev struct
+ * @size: size in bytes to map
+ * @context: user context
+ * @obj: opaque pointer to a cq, wq etc
+ *
+ * Return: rvt_mmap struct on success
  */
 struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
 					   u32 size,
@@ -180,6 +187,13 @@ struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
 }
 EXPORT_SYMBOL(rvt_create_mmap_info);
 
+/**
+ * rvt_update_mmap_info - update a mem map
+ * @rdi: rvt dev struct
+ * @ip: mmap info pointer
+ * @size: size to grow by
+ * @obj: opaque pointer to cq, wq, etc.
+ */
 void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
 			  u32 size, void *obj)
 {
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index 8bff6bbfece2..8549652ffd06 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -52,8 +52,13 @@
 #include "vt.h"
 #include "mr.h"
 
-/*
+/**
+ * rvt_driver_mr_init - Init MR resources per driver
+ * @rdi: rvt dev struct
+ *
  * Do any intilization needed when a driver registers with rdmavt.
+ *
+ * Return: 0 on success or errno on failure
  */
 int rvt_driver_mr_init(struct rvt_dev_info *rdi)
 {
@@ -98,7 +103,10 @@ int rvt_driver_mr_init(struct rvt_dev_info *rdi)
 	return 0;
 }
 
-/*
+/**
+ *rvt_mr_exit: clean up MR
+ *@rdi: rvt dev structure
+ *
  * called when drivers have unregistered or perhaps failed to register with us
  */
 void rvt_mr_exit(struct rvt_dev_info *rdi)
@@ -297,7 +305,7 @@ static void __rvt_free_mr(struct rvt_mr *mr)
  * @pd: protection domain for this memory region
  * @acc: access flags
  *
- * Returns the memory region on success, otherwise returns an errno.
+ * Return: the memory region on success, otherwise returns an errno.
  * Note that all DMA addresses should be created via the
  * struct ib_dma_mapping_ops functions (see dma.c).
  */
@@ -348,7 +356,7 @@ bail:
  * @mr_access_flags: access flags for this memory region
  * @udata: unused by the driver
  *
- * Returns the memory region on success, otherwise returns an errno.
+ * Return: the memory region on success, otherwise returns an errno.
  */
 struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			      u64 virt_addr, int mr_access_flags,
@@ -418,10 +426,11 @@ bail_umem:
  * rvt_dereg_mr - unregister and free a memory region
  * @ibmr: the memory region to free
  *
- * Returns 0 on success.
  *
  * Note that this is called to free MRs created by rvt_get_dma_mr()
  * or rvt_reg_user_mr().
+ *
+ * Returns 0 on success.
  */
 int rvt_dereg_mr(struct ib_mr *ibmr)
 {
@@ -456,7 +465,7 @@ out:
  * @mr_type: mem region type
  * @max_num_sg: Max number of segments allowed
  *
- * Return the memory region on success, otherwise return an errno.
+ * Return: the memory region on success, otherwise return an errno.
  */
 struct ib_mr *rvt_alloc_mr(struct ib_pd *pd,
 			   enum ib_mr_type mr_type,
@@ -480,7 +489,7 @@ struct ib_mr *rvt_alloc_mr(struct ib_pd *pd,
  * @mr_access_flags: access flags for this memory region
  * @fmr_attr: fast memory region attributes
  *
- * Returns the memory region on success, otherwise returns an errno.
+ * Return: the memory region on success, otherwise returns an errno.
  */
 struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
 			     struct ib_fmr_attr *fmr_attr)
@@ -537,6 +546,8 @@ bail:
  * @iova: the virtual address of the start of the fast memory region
  *
  * This may be called from interrupt context.
+ *
+ * Return: 0 on success
  */
 
 int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
@@ -580,7 +591,7 @@ int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
  * rvt_unmap_fmr - unmap fast memory regions
  * @fmr_list: the list of fast memory regions to unmap
  *
- * Returns 0 on success.
+ * Return: 0 on success.
  */
 int rvt_unmap_fmr(struct list_head *fmr_list)
 {
@@ -605,7 +616,7 @@ int rvt_unmap_fmr(struct list_head *fmr_list)
  * rvt_dealloc_fmr - deallocate a fast memory region
  * @ibfmr: the fast memory region to deallocate
  *
- * Returns 0 on success.
+ * Return: 0 on success.
  */
 int rvt_dealloc_fmr(struct ib_fmr *ibfmr)
 {
@@ -635,12 +646,13 @@ out:
  * @sge: SGE to check
  * @acc: access flags
  *
- * Return 1 if valid and successful, otherwise returns 0.
+ * Check the IB SGE for validity and initialize our internal version
+ * of it.
+ *
+ * Return: 1 if valid and successful, otherwise returns 0.
  *
  * increments the reference count upon success
  *
- * Check the IB SGE for validity and initialize our internal version
- * of it.
  */
 int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 		struct rvt_sge *isge, struct ib_sge *sge, int acc)
@@ -733,7 +745,7 @@ EXPORT_SYMBOL(rvt_lkey_ok);
  * @rkey: rkey to check
  * @acc: access flags
  *
- * Return 1 if successful, otherwise 0.
+ * Return: 1 if successful, otherwise 0.
  *
  * increments the reference count upon success
  */
diff --git a/drivers/infiniband/sw/rdmavt/pd.c b/drivers/infiniband/sw/rdmavt/pd.c
index 62fee44be3a3..d1292f324c67 100644
--- a/drivers/infiniband/sw/rdmavt/pd.c
+++ b/drivers/infiniband/sw/rdmavt/pd.c
@@ -48,6 +48,16 @@
 #include <linux/slab.h>
 #include "pd.h"
 
+/**
+ * rvt_alloc_pd - allocate a protection domain
+ * @ibdev: ib device
+ * @context: optional user context
+ * @udata: optional user data
+ *
+ * Allocate and keep track of a PD.
+ *
+ * Return: 0 on success
+ */
 struct ib_pd *rvt_alloc_pd(struct ib_device *ibdev,
 			   struct ib_ucontext *context,
 			   struct ib_udata *udata)
@@ -88,6 +98,12 @@ bail:
 	return ret;
 }
 
+/**
+ * rvt_dealloc_pd - Free PD
+ * @ibpd: Free up PD
+ *
+ * Return: always 0
+ */
 int rvt_dealloc_pd(struct ib_pd *ibpd)
 {
 	struct rvt_pd *pd = ibpd_to_rvtpd(ibpd);
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index d629911ab0ab..5809562fefda 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -162,6 +162,12 @@ static void free_qpn_table(struct rvt_qpn_table *qpt)
 		free_page((unsigned long)qpt->map[i].page);
 }
 
+/**
+ * rvt_driver_qp_init - Init driver qp resources
+ * @rdi: rvt dev strucutre
+ *
+ * Return: 0 on success
+ */
 int rvt_driver_qp_init(struct rvt_dev_info *rdi)
 {
 	int i;
@@ -262,6 +268,12 @@ static unsigned rvt_free_all_qps(struct rvt_dev_info *rdi)
 	return qp_inuse;
 }
 
+/**
+ * rvt_qp_exit - clean up qps on device exit
+ * @rdi: rvt dev structure
+ *
+ * Check for qp leaks and free resources.
+ */
 void rvt_qp_exit(struct rvt_dev_info *rdi)
 {
 	u32 qps_inuse = rvt_free_all_qps(rdi);
@@ -483,7 +495,7 @@ EXPORT_SYMBOL(rvt_reset_qp);
  * unique idea of what queue pair numbers mean. For instance there is a reserved
  * range for PSM.
  *
- * Returns the queue pair on success, otherwise returns an errno.
+ * Return: the queue pair on success, otherwise returns an errno.
  *
  * Called by the ib_create_qp() core verbs function.
  */
@@ -757,6 +769,11 @@ bail_swq:
 	return ret;
 }
 
+/**
+ * rvt_clear_mr_refs - Drop help mr refs
+ * @qp: rvt qp data structure
+ * @clr_sends: If shoudl clear send side or not
+ */
 void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends)
 {
 	unsigned n;
@@ -812,7 +829,8 @@ EXPORT_SYMBOL(rvt_clear_mr_refs);
  * @err: the receive completion error to signal if a RWQE is active
  *
  * Flushes both send and receive work queues.
- * Returns true if last WQE event should be generated.
+ *
+ * Return: true if last WQE event should be generated.
  * The QP r_lock and s_lock should be held and interrupts disabled.
  * If we are already in error state, just return.
  */
@@ -912,7 +930,11 @@ static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
 	spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags);
 }
 
-/*
+/**
+ * rvt_remove_qp - remove qp form table
+ * @rdi: rvt dev struct
+ * @qp: qp to remove
+ *
  * Remove the QP from the table so it can't be found asynchronously by
  * the receive routine.
  */
@@ -967,7 +989,7 @@ EXPORT_SYMBOL(rvt_remove_qp);
  * @attr_mask: the mask of attributes to modify
  * @udata: user data for libibverbs.so
  *
- * Returns 0 on success, otherwise returns an errno.
+ * Return: 0 on success, otherwise returns an errno.
  */
 int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		  int attr_mask, struct ib_udata *udata)
@@ -1224,10 +1246,10 @@ inval:
  * rvt_destroy_qp - destroy a queue pair
  * @ibqp: the queue pair to destroy
  *
- * Returns 0 on success.
- *
  * Note that this can be called while the QP is actively sending or
  * receiving!
+ *
+ * Return: 0 on success.
  */
 int rvt_destroy_qp(struct ib_qp *ibqp)
 {
@@ -1263,6 +1285,15 @@ int rvt_destroy_qp(struct ib_qp *ibqp)
 	return 0;
 }
 
+/**
+ * rvt_query_qp - query an ipbq
+ * @ibqp: IB qp to query
+ * @attr: attr struct to fill in
+ * @attr_mask: attr mask ignored
+ * @init_attr: struct to fill in
+ *
+ * Return: always 0
+ */
 int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		 int attr_mask, struct ib_qp_init_attr *init_attr)
 {
@@ -1321,6 +1352,8 @@ int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
  * @bad_wr: the first bad WR is put here
  *
  * This may be called from interrupt context.
+ *
+ * Return: 0 on success otherwise errno
  */
 int rvt_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 		  struct ib_recv_wr **bad_wr)
@@ -1539,6 +1572,8 @@ bail_inval_free:
  * @bad_wr: the first bad WR is put here
  *
  * This may be called from interrupt context.
+ *
+ * Return: 0 on success else errno
  */
 int rvt_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		  struct ib_send_wr **bad_wr)
@@ -1594,6 +1629,8 @@ bail:
  * @bad_wr: A pointer to the first WR to cause a problem is put here
  *
  * This may be called from interrupt context.
+ *
+ * Return: 0 on success else errno
  */
 int rvt_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
 		      struct ib_recv_wr **bad_wr)
@@ -1636,6 +1673,10 @@ int rvt_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
 	return 0;
 }
 
+/** rvt_free_qpn - Free a qpn from the bit map
+ * @qpt: QP table
+ * @qpn: queue pair number to free
+ */
 void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
 {
 	struct rvt_qpn_map *map;
@@ -1646,6 +1687,10 @@ void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
 }
 EXPORT_SYMBOL(rvt_free_qpn);
 
+/**
+ * rvt_dec_qp_cnt - decrement qp count
+ * rdi: rvt dev struct
+ */
 void rvt_dec_qp_cnt(struct rvt_dev_info *rdi)
 {
 	spin_lock(&rdi->n_qps_lock);
diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c
index 4960a89f91b2..98c492797c53 100644
--- a/drivers/infiniband/sw/rdmavt/srq.c
+++ b/drivers/infiniband/sw/rdmavt/srq.c
@@ -51,7 +51,10 @@
 
 #include "srq.h"
 
-/*
+/**
+ * rvt_driver_srq_init - init srq resources on a per driver basis
+ * @rdi: rvt dev structure
+ *
  * Do any initialization needed when a driver registers with rdmavt.
  */
 void rvt_driver_srq_init(struct rvt_dev_info *rdi)
@@ -65,6 +68,8 @@ void rvt_driver_srq_init(struct rvt_dev_info *rdi)
  * @ibpd: the protection domain of the SRQ to create
  * @srq_init_attr: the attributes of the SRQ
  * @udata: data from libibverbs when creating a user SRQ
+ *
+ * Return: Allocated srq object
  */
 struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 			      struct ib_srq_init_attr *srq_init_attr,
@@ -168,6 +173,8 @@ bail_srq:
  * @attr: the new attributes of the SRQ
  * @attr_mask: indicates which attributes to modify
  * @udata: user data for libibverbs.so
+ *
+ * Return: 0 on success
  */
 int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 		   enum ib_srq_attr_mask attr_mask,
@@ -305,6 +312,12 @@ bail_free:
 	return ret;
 }
 
+/** rvt_query_srq - query srq data
+ * @ibsrq: srq to query
+ * @attr: return info in attr
+ *
+ * Return: always 0
+ */
 int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 {
 	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
@@ -315,6 +328,12 @@ int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 	return 0;
 }
 
+/**
+ * rvt_destroy_srq - destory an srq
+ * @ibsrq: srq object to destroy
+ *
+ * Return always 0
+ */
 int rvt_destroy_srq(struct ib_srq *ibsrq)
 {
 	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 2ccf6103e5fa..f5cb09b718be 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -57,16 +57,37 @@ MODULE_DESCRIPTION("RDMA Verbs Transport Library");
 
 static int rvt_init(void)
 {
-	/* Do any work needed prior to drivers calling for registration*/
+	/*
+	 * rdmavt does not need to do anything special when it starts up. All it
+	 * needs to do is sit and wait until a driver attempts registration.
+	 */
 	return 0;
 }
 module_init(rvt_init);
 
 static void rvt_cleanup(void)
 {
+	/*
+	 * Nothing to do at exit time either. The module won't be able to be
+	 * removed until all drivers are gone which means all the dev structs
+	 * are gone so there is really nothing to do.
+	 */
 }
 module_exit(rvt_cleanup);
 
+/**
+ * rvt_alloc_device - allocate rdi
+ * @size: how big of a structure to allocate
+ * @nports: number of ports to allocate array slots for
+ *
+ * Use IB core device alloc to allocate space for the rdi which is assumed to be
+ * inside of the ib_device. Any extra space that drivers require should be
+ * included in size.
+ *
+ * We also allocate a port array based on the number of ports.
+ *
+ * Return: pointer to allocated rdi
+ */
 struct rvt_dev_info *rvt_alloc_device(size_t size, int nports)
 {
 	struct rvt_dev_info *rdi = ERR_PTR(-ENOMEM);
@@ -105,15 +126,10 @@ static int rvt_modify_device(struct ib_device *device,
 			     struct ib_device_modify *device_modify)
 {
 	/*
-	 * Change dev props. Planned support is for node desc change and sys
-	 * guid change only. This matches hfi1 and qib behavior. Other drivers
-	 * that support existing modifications will need to add their support.
+	 * There is currently no need to supply this based on qib and hfi1.
+	 * Future drivers may need to implement this though.
 	 */
 
-	/*
-	 * VT-DRIVER-API: node_desc_change()
-	 * VT-DRIVER-API: sys_guid_change()
-	 */
 	return -EOPNOTSUPP;
 }
 
@@ -123,7 +139,7 @@ static int rvt_modify_device(struct ib_device *device,
  * @port_num: port number, 1 based from ib core
  * @props: structure to hold returned properties
  *
- * Returns 0 on success
+ * Return: 0 on success
  */
 static int rvt_query_port(struct ib_device *ibdev, u8 port_num,
 			  struct ib_port_attr *props)
@@ -158,7 +174,7 @@ static int rvt_query_port(struct ib_device *ibdev, u8 port_num,
  * @port_modify_mask: How to change the port
  * @props: Structure to fill in
  *
- * Returns 0 on success
+ * Return: 0 on success
  */
 static int rvt_modify_port(struct ib_device *ibdev, u8 port_num,
 			   int port_modify_mask, struct ib_port_modify *props)
@@ -191,7 +207,7 @@ static int rvt_modify_port(struct ib_device *ibdev, u8 port_num,
  * @port_num: Port number, 1 based from ib core
  * @intex: Index into pkey table
  *
- * Returns 0 on failure pkey otherwise
+ * Return: 0 on failure pkey otherwise
  */
 static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index,
 			  u16 *pkey)
@@ -223,7 +239,7 @@ static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index,
  * @index: = Index in table
  * @gid: Gid to return
  *
- * Returns 0 on success
+ * Return: 0 on success
  */
 static int rvt_query_gid(struct ib_device *ibdev, u8 port_num,
 			 int guid_index, union ib_gid *gid)
@@ -316,6 +332,15 @@ static int rvt_get_port_immutable(struct ib_device *ibdev, u8 port_num,
 #define CHECK_DRIVER_OVERRIDE(rdi, x) \
 	rdi->ibdev.x = rdi->ibdev.x ? : rvt_ ##x
 
+/**
+ * rvt_register_device - register a driver
+ * @rdi: main dev structure for all of rdmavt operations
+ *
+ * It is up to drivers to allocate the rdi and fill in the appropriate
+ * information.
+ *
+ * Return: 0 on success otherwise an errno.
+ */
 int rvt_register_device(struct rvt_dev_info *rdi)
 {
 	/* Validate that drivers have provided the right information */
@@ -487,6 +512,10 @@ bail_no_mr:
 }
 EXPORT_SYMBOL(rvt_register_device);
 
+/**
+ * rvt_unregister_device - remove a driver
+ * @rdi: rvt dev struct
+ */
 void rvt_unregister_device(struct rvt_dev_info *rdi)
 {
 	trace_rvt_dbg(rdi, "Driver is unregistering.");
@@ -502,9 +531,16 @@ void rvt_unregister_device(struct rvt_dev_info *rdi)
 }
 EXPORT_SYMBOL(rvt_unregister_device);
 
-/*
+/**
+ * rvt_init_port - init internal data for driver port
+ * @rdi: rvt dev strut
+ * @port: rvt port
+ * @port_index: 0 based index of ports, different from IB core port num
+ *
  * Keep track of a list of ports. No need to have a detach port.
  * They persist until the driver goes away.
+ *
+ * Return: always 0
  */
 int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
 		  int port_index, u16 *pkey_table)
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index aabd2e5bc5d7..57c708dddab4 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -146,21 +146,6 @@ struct rvt_ibport {
  * Things that are driver specific, module parameters in hfi1 and qib
  */
 struct rvt_driver_params {
-	/*
-	 * driver required fields:
-	 *	node_guid
-	 *	phys_port_cnt
-	 *	dma_device
-	 *	owner
-	 * driver optional fields (rvt will provide generic value if blank):
-	 *	name
-	 *	node_desc
-	 * rvt fields, driver value ignored:
-	 *	uverbs_abi_ver
-	 *	node_type
-	 *	num_comp_vectors
-	 *	uverbs_cmd_mask
-	 */
 	struct ib_device_attr props;
 
 	/*
-- 
cgit v1.2.3


From 79a225be38932b17707009767e85d6edf450e7cc Mon Sep 17 00:00:00 2001
From: Dennis Dalessandro <dennis.dalessandro@intel.com>
Date: Sun, 14 Feb 2016 12:11:20 -0800
Subject: IB/rdmavt: Remove unnecessary exported functions

Remove exported functions which are no longer required as the
functionality has moved into rdmavt. This also requires re-ordering some
of the functions since their prototype no longer appears in a header
file. Rather than add forward declarations it is just cleaner to
re-order some of the functions.

Reviewed-by: Jubin John <jubin.john@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
---
 drivers/infiniband/sw/rdmavt/mmap.c |   4 -
 drivers/infiniband/sw/rdmavt/mmap.h |   8 ++
 drivers/infiniband/sw/rdmavt/qp.c   | 252 +++++++++++++++++-------------------
 drivers/infiniband/sw/rdmavt/srq.c  |   1 +
 drivers/infiniband/sw/rdmavt/vt.h   |   1 +
 include/rdma/rdma_vt.h              |  13 --
 include/rdma/rdmavt_qp.h            |   4 -
 7 files changed, 128 insertions(+), 155 deletions(-)

(limited to 'drivers/infiniband/sw/rdmavt/mmap.c')

diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
index 273974fb7d1f..e202b8142759 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.c
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -80,7 +80,6 @@ void rvt_release_mmap_info(struct kref *ref)
 	vfree(ip->obj);
 	kfree(ip);
 }
-EXPORT_SYMBOL(rvt_release_mmap_info);
 
 static void rvt_vma_open(struct vm_area_struct *vma)
 {
@@ -146,7 +145,6 @@ int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 done:
 	return ret;
 }
-EXPORT_SYMBOL(rvt_mmap);
 
 /**
  * rvt_create_mmap_info - allocate information for hfi1_mmap
@@ -185,7 +183,6 @@ struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
 
 	return ip;
 }
-EXPORT_SYMBOL(rvt_create_mmap_info);
 
 /**
  * rvt_update_mmap_info - update a mem map
@@ -209,4 +206,3 @@ void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
 	ip->size = size;
 	ip->obj = obj;
 }
-EXPORT_SYMBOL(rvt_update_mmap_info);
diff --git a/drivers/infiniband/sw/rdmavt/mmap.h b/drivers/infiniband/sw/rdmavt/mmap.h
index e8067471c722..fab0e7b1daf9 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.h
+++ b/drivers/infiniband/sw/rdmavt/mmap.h
@@ -51,5 +51,13 @@
 #include <rdma/rdma_vt.h>
 
 void rvt_mmap_init(struct rvt_dev_info *rdi);
+void rvt_release_mmap_info(struct kref *ref);
+int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
+					   u32 size,
+					   struct ib_ucontext *context,
+					   void *obj);
+void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
+			  u32 size, void *obj);
 
 #endif          /* DEF_RDMAVTMMAP_H */
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 441e17a0467f..dbf124db1fd1 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -389,13 +389,117 @@ static void free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
 		clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page);
 }
 
+/**
+ * rvt_clear_mr_refs - Drop help mr refs
+ * @qp: rvt qp data structure
+ * @clr_sends: If shoudl clear send side or not
+ */
+static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends)
+{
+	unsigned n;
+
+	if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags))
+		rvt_put_ss(&qp->s_rdma_read_sge);
+
+	rvt_put_ss(&qp->r_sge);
+
+	if (clr_sends) {
+		while (qp->s_last != qp->s_head) {
+			struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last);
+			unsigned i;
+
+			for (i = 0; i < wqe->wr.num_sge; i++) {
+				struct rvt_sge *sge = &wqe->sg_list[i];
+
+				rvt_put_mr(sge->mr);
+			}
+			if (qp->ibqp.qp_type == IB_QPT_UD ||
+			    qp->ibqp.qp_type == IB_QPT_SMI ||
+			    qp->ibqp.qp_type == IB_QPT_GSI)
+				atomic_dec(&ibah_to_rvtah(
+						wqe->ud_wr.ah)->refcount);
+			if (++qp->s_last >= qp->s_size)
+				qp->s_last = 0;
+			smp_wmb(); /* see qp_set_savail */
+		}
+		if (qp->s_rdma_mr) {
+			rvt_put_mr(qp->s_rdma_mr);
+			qp->s_rdma_mr = NULL;
+		}
+	}
+
+	if (qp->ibqp.qp_type != IB_QPT_RC)
+		return;
+
+	for (n = 0; n < ARRAY_SIZE(qp->s_ack_queue); n++) {
+		struct rvt_ack_entry *e = &qp->s_ack_queue[n];
+
+		if (e->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST &&
+		    e->rdma_sge.mr) {
+			rvt_put_mr(e->rdma_sge.mr);
+			e->rdma_sge.mr = NULL;
+		}
+	}
+}
+
+/**
+ * rvt_remove_qp - remove qp form table
+ * @rdi: rvt dev struct
+ * @qp: qp to remove
+ *
+ * Remove the QP from the table so it can't be found asynchronously by
+ * the receive routine.
+ */
+static void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
+{
+	struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1];
+	u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits);
+	unsigned long flags;
+	int removed = 1;
+
+	spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags);
+
+	if (rcu_dereference_protected(rvp->qp[0],
+			lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) {
+		RCU_INIT_POINTER(rvp->qp[0], NULL);
+	} else if (rcu_dereference_protected(rvp->qp[1],
+			lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) {
+		RCU_INIT_POINTER(rvp->qp[1], NULL);
+	} else {
+		struct rvt_qp *q;
+		struct rvt_qp __rcu **qpp;
+
+		removed = 0;
+		qpp = &rdi->qp_dev->qp_table[n];
+		for (; (q = rcu_dereference_protected(*qpp,
+			lockdep_is_held(&rdi->qp_dev->qpt_lock))) != NULL;
+			qpp = &q->next) {
+			if (q == qp) {
+				RCU_INIT_POINTER(*qpp,
+				     rcu_dereference_protected(qp->next,
+				     lockdep_is_held(&rdi->qp_dev->qpt_lock)));
+				removed = 1;
+				trace_rvt_qpremove(qp, n);
+				break;
+			}
+		}
+	}
+
+	spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags);
+	if (removed) {
+		synchronize_rcu();
+		if (atomic_dec_and_test(&qp->refcount))
+			wake_up(&qp->wait);
+	}
+}
+
 /**
  * reset_qp - initialize the QP state to the reset state
  * @qp: the QP to reset
  * @type: the QP type
  * r and s lock are required to be held by the caller
  */
-void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
+static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
 		  enum ib_qp_type type)
 {
 	if (qp->state != IB_QPS_RESET) {
@@ -475,7 +579,6 @@ void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
 	}
 	qp->r_sge.num_sge = 0;
 }
-EXPORT_SYMBOL(rvt_reset_qp);
 
 /**
  * rvt_create_qp - create a queue pair for a device
@@ -761,60 +864,6 @@ bail_swq:
 	return ret;
 }
 
-/**
- * rvt_clear_mr_refs - Drop help mr refs
- * @qp: rvt qp data structure
- * @clr_sends: If shoudl clear send side or not
- */
-void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends)
-{
-	unsigned n;
-
-	if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags))
-		rvt_put_ss(&qp->s_rdma_read_sge);
-
-	rvt_put_ss(&qp->r_sge);
-
-	if (clr_sends) {
-		while (qp->s_last != qp->s_head) {
-			struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last);
-			unsigned i;
-
-			for (i = 0; i < wqe->wr.num_sge; i++) {
-				struct rvt_sge *sge = &wqe->sg_list[i];
-
-				rvt_put_mr(sge->mr);
-			}
-			if (qp->ibqp.qp_type == IB_QPT_UD ||
-			    qp->ibqp.qp_type == IB_QPT_SMI ||
-			    qp->ibqp.qp_type == IB_QPT_GSI)
-				atomic_dec(&ibah_to_rvtah(
-						wqe->ud_wr.ah)->refcount);
-			if (++qp->s_last >= qp->s_size)
-				qp->s_last = 0;
-			smp_wmb(); /* see qp_set_savail */
-		}
-		if (qp->s_rdma_mr) {
-			rvt_put_mr(qp->s_rdma_mr);
-			qp->s_rdma_mr = NULL;
-		}
-	}
-
-	if (qp->ibqp.qp_type != IB_QPT_RC)
-		return;
-
-	for (n = 0; n < ARRAY_SIZE(qp->s_ack_queue); n++) {
-		struct rvt_ack_entry *e = &qp->s_ack_queue[n];
-
-		if (e->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST &&
-		    e->rdma_sge.mr) {
-			rvt_put_mr(e->rdma_sge.mr);
-			e->rdma_sge.mr = NULL;
-		}
-	}
-}
-EXPORT_SYMBOL(rvt_clear_mr_refs);
-
 /**
  * rvt_error_qp - put a QP into the error state
  * @qp: the QP to put into the error state
@@ -922,58 +971,6 @@ static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
 	spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags);
 }
 
-/**
- * rvt_remove_qp - remove qp form table
- * @rdi: rvt dev struct
- * @qp: qp to remove
- *
- * Remove the QP from the table so it can't be found asynchronously by
- * the receive routine.
- */
-void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp)
-{
-	struct rvt_ibport *rvp = rdi->ports[qp->port_num - 1];
-	u32 n = hash_32(qp->ibqp.qp_num, rdi->qp_dev->qp_table_bits);
-	unsigned long flags;
-	int removed = 1;
-
-	spin_lock_irqsave(&rdi->qp_dev->qpt_lock, flags);
-
-	if (rcu_dereference_protected(rvp->qp[0],
-			lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) {
-		RCU_INIT_POINTER(rvp->qp[0], NULL);
-	} else if (rcu_dereference_protected(rvp->qp[1],
-			lockdep_is_held(&rdi->qp_dev->qpt_lock)) == qp) {
-		RCU_INIT_POINTER(rvp->qp[1], NULL);
-	} else {
-		struct rvt_qp *q;
-		struct rvt_qp __rcu **qpp;
-
-		removed = 0;
-		qpp = &rdi->qp_dev->qp_table[n];
-		for (; (q = rcu_dereference_protected(*qpp,
-			lockdep_is_held(&rdi->qp_dev->qpt_lock))) != NULL;
-			qpp = &q->next) {
-			if (q == qp) {
-				RCU_INIT_POINTER(*qpp,
-				     rcu_dereference_protected(qp->next,
-				     lockdep_is_held(&rdi->qp_dev->qpt_lock)));
-				removed = 1;
-				trace_rvt_qpremove(qp, n);
-				break;
-			}
-		}
-	}
-
-	spin_unlock_irqrestore(&rdi->qp_dev->qpt_lock, flags);
-	if (removed) {
-		synchronize_rcu();
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
-	}
-}
-EXPORT_SYMBOL(rvt_remove_qp);
-
 /**
  * qib_modify_qp - modify the attributes of a queue pair
  * @ibqp: the queue pair who's attributes we're modifying
@@ -1234,6 +1231,19 @@ inval:
 	return -EINVAL;
 }
 
+/** rvt_free_qpn - Free a qpn from the bit map
+ * @qpt: QP table
+ * @qpn: queue pair number to free
+ */
+static void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
+{
+	struct rvt_qpn_map *map;
+
+	map = qpt->map + qpn / RVT_BITS_PER_PAGE;
+	if (map->page)
+		clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page);
+}
+
 /**
  * rvt_destroy_qp - destroy a queue pair
  * @ibqp: the queue pair to destroy
@@ -1664,29 +1674,3 @@ int rvt_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
 	}
 	return 0;
 }
-
-/** rvt_free_qpn - Free a qpn from the bit map
- * @qpt: QP table
- * @qpn: queue pair number to free
- */
-void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
-{
-	struct rvt_qpn_map *map;
-
-	map = qpt->map + qpn / RVT_BITS_PER_PAGE;
-	if (map->page)
-		clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page);
-}
-EXPORT_SYMBOL(rvt_free_qpn);
-
-/**
- * rvt_dec_qp_cnt - decrement qp count
- * rdi: rvt dev struct
- */
-void rvt_dec_qp_cnt(struct rvt_dev_info *rdi)
-{
-	spin_lock(&rdi->n_qps_lock);
-	rdi->n_qps_allocated--;
-	spin_unlock(&rdi->n_qps_lock);
-}
-EXPORT_SYMBOL(rvt_dec_qp_cnt);
diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c
index 98c492797c53..f7c48e9023de 100644
--- a/drivers/infiniband/sw/rdmavt/srq.c
+++ b/drivers/infiniband/sw/rdmavt/srq.c
@@ -50,6 +50,7 @@
 #include <linux/vmalloc.h>
 
 #include "srq.h"
+#include "vt.h"
 
 /**
  * rvt_driver_srq_init - init srq resources on a per driver basis
diff --git a/drivers/infiniband/sw/rdmavt/vt.h b/drivers/infiniband/sw/rdmavt/vt.h
index e26f9e94d1ea..6b01eaa4461b 100644
--- a/drivers/infiniband/sw/rdmavt/vt.h
+++ b/drivers/infiniband/sw/rdmavt/vt.h
@@ -60,6 +60,7 @@
 #include "mmap.h"
 #include "cq.h"
 #include "mad.h"
+#include "mmap.h"
 
 #define rvt_pr_info(rdi, fmt, ...) \
 	__rvt_pr_info(rdi->driver_f.get_pci_dev(rdi), \
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index 4c50bbb75d77..a8696551abb1 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -476,19 +476,6 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
 		u32 len, u64 vaddr, u32 rkey, int acc);
 int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 		struct rvt_sge *isge, struct ib_sge *sge, int acc);
-int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
-void rvt_release_mmap_info(struct kref *ref);
-struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
-					   u32 size,
-					   struct ib_ucontext *context,
-					   void *obj);
-void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
-			  u32 size, void *obj);
-int rvt_reg_mr(struct rvt_qp *qp, struct ib_reg_wr *wr);
 struct rvt_mcast *rvt_mcast_find(struct rvt_ibport *ibp, union ib_gid *mgid);
 
-/* Temporary export */
-void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
-		  enum ib_qp_type type);
-
 #endif          /* DEF_RDMA_VT_H */
diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h
index 1066b5d1b4d2..933f14f92da6 100644
--- a/include/rdma/rdmavt_qp.h
+++ b/include/rdma/rdmavt_qp.h
@@ -438,10 +438,6 @@ static inline struct rvt_rwqe *rvt_get_rwqe_ptr(struct rvt_rq *rq, unsigned n)
 extern const int  ib_rvt_state_ops[];
 
 struct rvt_dev_info;
-void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp);
-void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends);
 int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err);
-void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn);
-void rvt_dec_qp_cnt(struct rvt_dev_info *rdi);
 
 #endif          /* DEF_RDMAVT_INCQP_H */
-- 
cgit v1.2.3