IB/uverbs: Don't serialize with ib_uverbs_idr_mutex
Currently, all userspace verbs operations that call into the kernel
are serialized by ib_uverbs_idr_mutex. This can be a scalability
issue for some workloads, especially for devices driven by the ipath
driver, which needs to call into the kernel even for datapath
operations.
Fix this by adding reference counts to the userspace objects, and then
converting ib_uverbs_idr_mutex into a spinlock that only protects the
idrs long enough to take a reference on the object being looked up.
Because remove operations may fail, we have to do a slightly funky
two-step deletion, which is described in the comments at the top of
uverbs_cmd.c.
This also still leaves ib_uverbs_idr_lock as a single lock that is
possibly subject to contention. However, the lock hold time will only
be a single idr operation, so multiple threads should still be able to
make progress, even if ib_uverbs_idr_lock is being ping-ponged.
Surprisingly, these changes even shrink the object code:
add/remove: 23/5 grow/shrink: 4/21 up/down: 633/-693 (-60)
Signed-off-by: Roland Dreier <rolandd@cisco.com>
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index ff092a0..5ec2d49 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -66,7 +66,7 @@
static struct class *uverbs_class;
-DEFINE_MUTEX(ib_uverbs_idr_mutex);
+DEFINE_SPINLOCK(ib_uverbs_idr_lock);
DEFINE_IDR(ib_uverbs_pd_idr);
DEFINE_IDR(ib_uverbs_mr_idr);
DEFINE_IDR(ib_uverbs_mw_idr);
@@ -183,21 +183,21 @@
if (!context)
return 0;
- mutex_lock(&ib_uverbs_idr_mutex);
-
list_for_each_entry_safe(uobj, tmp, &context->ah_list, list) {
- struct ib_ah *ah = idr_find(&ib_uverbs_ah_idr, uobj->id);
- idr_remove(&ib_uverbs_ah_idr, uobj->id);
+ struct ib_ah *ah = uobj->object;
+
+ idr_remove_uobj(&ib_uverbs_ah_idr, uobj);
ib_destroy_ah(ah);
list_del(&uobj->list);
kfree(uobj);
}
list_for_each_entry_safe(uobj, tmp, &context->qp_list, list) {
- struct ib_qp *qp = idr_find(&ib_uverbs_qp_idr, uobj->id);
+ struct ib_qp *qp = uobj->object;
struct ib_uqp_object *uqp =
container_of(uobj, struct ib_uqp_object, uevent.uobject);
- idr_remove(&ib_uverbs_qp_idr, uobj->id);
+
+ idr_remove_uobj(&ib_uverbs_qp_idr, uobj);
ib_uverbs_detach_umcast(qp, uqp);
ib_destroy_qp(qp);
list_del(&uobj->list);
@@ -206,11 +206,12 @@
}
list_for_each_entry_safe(uobj, tmp, &context->cq_list, list) {
- struct ib_cq *cq = idr_find(&ib_uverbs_cq_idr, uobj->id);
+ struct ib_cq *cq = uobj->object;
struct ib_uverbs_event_file *ev_file = cq->cq_context;
struct ib_ucq_object *ucq =
container_of(uobj, struct ib_ucq_object, uobject);
- idr_remove(&ib_uverbs_cq_idr, uobj->id);
+
+ idr_remove_uobj(&ib_uverbs_cq_idr, uobj);
ib_destroy_cq(cq);
list_del(&uobj->list);
ib_uverbs_release_ucq(file, ev_file, ucq);
@@ -218,10 +219,11 @@
}
list_for_each_entry_safe(uobj, tmp, &context->srq_list, list) {
- struct ib_srq *srq = idr_find(&ib_uverbs_srq_idr, uobj->id);
+ struct ib_srq *srq = uobj->object;
struct ib_uevent_object *uevent =
container_of(uobj, struct ib_uevent_object, uobject);
- idr_remove(&ib_uverbs_srq_idr, uobj->id);
+
+ idr_remove_uobj(&ib_uverbs_srq_idr, uobj);
ib_destroy_srq(srq);
list_del(&uobj->list);
ib_uverbs_release_uevent(file, uevent);
@@ -231,11 +233,11 @@
/* XXX Free MWs */
list_for_each_entry_safe(uobj, tmp, &context->mr_list, list) {
- struct ib_mr *mr = idr_find(&ib_uverbs_mr_idr, uobj->id);
+ struct ib_mr *mr = uobj->object;
struct ib_device *mrdev = mr->device;
struct ib_umem_object *memobj;
- idr_remove(&ib_uverbs_mr_idr, uobj->id);
+ idr_remove_uobj(&ib_uverbs_mr_idr, uobj);
ib_dereg_mr(mr);
memobj = container_of(uobj, struct ib_umem_object, uobject);
@@ -246,15 +248,14 @@
}
list_for_each_entry_safe(uobj, tmp, &context->pd_list, list) {
- struct ib_pd *pd = idr_find(&ib_uverbs_pd_idr, uobj->id);
- idr_remove(&ib_uverbs_pd_idr, uobj->id);
+ struct ib_pd *pd = uobj->object;
+
+ idr_remove_uobj(&ib_uverbs_pd_idr, uobj);
ib_dealloc_pd(pd);
list_del(&uobj->list);
kfree(uobj);
}
- mutex_unlock(&ib_uverbs_idr_mutex);
-
return context->device->dealloc_ucontext(context);
}
