RDMA/mana_ib: register RDMA device with GDMA

Software client needs to register with the RDMA management interface on
the SoC to access more features, including querying device capabilities
and RC queue pair.

Signed-off-by: Long Li <longli@microsoft.com>
Link: https://lore.kernel.org/r/1702692255-23640-2-git-send-email-longli@linuxonhyperv.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c
index d4541b8707e4c75a191a09dd2890447b45895ed1..fe025e13a45c40c13004720c4de59427a6a12d34 100644 (file)
--- a/drivers/infiniband/hw/mana/device.c
+++ b/drivers/infiniband/hw/mana/device.c
@@ -68,7 +68,6 @@ static int mana_ib_probe(struct auxiliary_device *adev,
        ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
                  mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt);
 
-       dev->gdma_dev = mdev;
        dev->ib_dev.node_type = RDMA_NODE_IB_CA;
 
        /*
@@ -78,16 +77,28 @@ static int mana_ib_probe(struct auxiliary_device *adev,
        dev->ib_dev.num_comp_vectors = 1;
        dev->ib_dev.dev.parent = mdev->gdma_context->dev;
 
-       ret = ib_register_device(&dev->ib_dev, "mana_%d",
-                                mdev->gdma_context->dev);
+       ret = mana_gd_register_device(&mdev->gdma_context->mana_ib);
        if (ret) {
-               ib_dealloc_device(&dev->ib_dev);
-               return ret;
+               ibdev_err(&dev->ib_dev, "Failed to register device, ret %d",
+                         ret);
+               goto free_ib_device;
        }
+       dev->gdma_dev = &mdev->gdma_context->mana_ib;
+
+       ret = ib_register_device(&dev->ib_dev, "mana_%d",
+                                mdev->gdma_context->dev);
+       if (ret)
+               goto deregister_device;
 
        dev_set_drvdata(&adev->dev, dev);
 
        return 0;
+
+deregister_device:
+       mana_gd_deregister_device(dev->gdma_dev);
+free_ib_device:
+       ib_dealloc_device(&dev->ib_dev);
+       return ret;
 }
 
 static void mana_ib_remove(struct auxiliary_device *adev)
@@ -95,6 +106,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)
        struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev);
 
        ib_unregister_device(&dev->ib_dev);
+
+       mana_gd_deregister_device(dev->gdma_dev);
+
        ib_dealloc_device(&dev->ib_dev);
 }
 

diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index 7be4c3adb4e2bba8a4bfcecc316a6ce67a00e16a..53730306ed9b23aece5dcd8134622f76dc148a8a 100644 (file)
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -8,7 +8,7 @@
 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
                         u32 port)
 {
-       struct gdma_dev *gd = dev->gdma_dev;
+       struct gdma_dev *gd = &dev->gdma_dev->gdma_context->mana;
        struct mana_port_context *mpc;
        struct net_device *ndev;
        struct mana_context *mc;
@@ -31,7 +31,7 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
                      u32 doorbell_id)
 {
-       struct gdma_dev *mdev = dev->gdma_dev;
+       struct gdma_dev *mdev = &dev->gdma_dev->gdma_context->mana;
        struct mana_port_context *mpc;
        struct mana_context *mc;
        struct net_device *ndev;

diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 4b3b5b274e8498006102decf0e87743a98a7445e..ae45d28eef5e0bfc263505cf7fd6388aa8edaffd 100644 (file)
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -21,8 +21,8 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
        u32 req_buf_size;
        int i, err;
 
-       mdev = dev->gdma_dev;
-       gc = mdev->gdma_context;
+       gc = dev->gdma_dev->gdma_context;
+       mdev = &gc->mana;
 
        req_buf_size =
                sizeof(*req) + sizeof(mana_handle_t) * MANA_INDIRECT_TABLE_SIZE;
@@ -102,20 +102,21 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
        struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl;
        struct mana_ib_create_qp_rss_resp resp = {};
        struct mana_ib_create_qp_rss ucmd = {};
-       struct gdma_dev *gd = mdev->gdma_dev;
        mana_handle_t *mana_ind_table;
        struct mana_port_context *mpc;
+       unsigned int ind_tbl_size;
        struct mana_context *mc;
        struct net_device *ndev;
        struct mana_ib_cq *cq;
        struct mana_ib_wq *wq;
-       unsigned int ind_tbl_size;
+       struct gdma_dev *gd;
        struct ib_cq *ibcq;
        struct ib_wq *ibwq;
        int i = 0;
        u32 port;
        int ret;
 
+       gd = &mdev->gdma_dev->gdma_context->mana;
        mc = gd->driver_data;
 
        if (!udata || udata->inlen < sizeof(ucmd))
@@ -266,8 +267,8 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
        struct mana_ib_ucontext *mana_ucontext =
                rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
                                          ibucontext);
+       struct gdma_dev *gd = &mdev->gdma_dev->gdma_context->mana;
        struct mana_ib_create_qp_resp resp = {};
-       struct gdma_dev *gd = mdev->gdma_dev;
        struct mana_ib_create_qp ucmd = {};
        struct mana_obj_spec wq_spec = {};
        struct mana_obj_spec cq_spec = {};
@@ -437,7 +438,7 @@ static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
 {
        struct mana_ib_dev *mdev =
                container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
-       struct gdma_dev *gd = mdev->gdma_dev;
+       struct gdma_dev *gd = &mdev->gdma_dev->gdma_context->mana;
        struct mana_port_context *mpc;
        struct mana_context *mc;
        struct net_device *ndev;
@@ -464,7 +465,7 @@ static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata)
 {
        struct mana_ib_dev *mdev =
                container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
-       struct gdma_dev *gd = mdev->gdma_dev;
+       struct gdma_dev *gd = &mdev->gdma_dev->gdma_context->mana;
        struct ib_pd *ibpd = qp->ibqp.pd;
        struct mana_port_context *mpc;
        struct mana_context *mc;

diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index 6367de0c2c2e8fd05e369b0d6fc4f0fcd7580641..e6e71e3c357ceca2c48238d39b22e05266e48cac 100644 (file)
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -158,6 +158,9 @@ static int mana_gd_detect_devices(struct pci_dev *pdev)
                if (dev_type == GDMA_DEVICE_MANA) {
                        gc->mana.gdma_context = gc;
                        gc->mana.dev_id = dev;
+               } else if (dev_type == GDMA_DEVICE_MANA_IB) {
+                       gc->mana_ib.dev_id = dev;
+                       gc->mana_ib.gdma_context = gc;
                }
        }
 
@@ -971,6 +974,7 @@ int mana_gd_register_device(struct gdma_dev *gd)
 
        return 0;
 }
+EXPORT_SYMBOL_NS(mana_gd_register_device, NET_MANA);
 
 int mana_gd_deregister_device(struct gdma_dev *gd)
 {
@@ -1001,6 +1005,7 @@ int mana_gd_deregister_device(struct gdma_dev *gd)
 
        return err;
 }
+EXPORT_SYMBOL_NS(mana_gd_deregister_device, NET_MANA);
 
 u32 mana_gd_wq_avail_space(struct gdma_queue *wq)
 {

diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 88b6ef7ce1a6efd9218f0b700eda82cb36b947c2..000f0d7670f7513ffc8a034f2490da3b0dab3fea 100644 (file)
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -66,6 +66,7 @@ enum {
        GDMA_DEVICE_NONE        = 0,
        GDMA_DEVICE_HWC         = 1,
        GDMA_DEVICE_MANA        = 2,
+       GDMA_DEVICE_MANA_IB     = 3,
 };
 
 struct gdma_resource {
@@ -387,6 +388,9 @@ struct gdma_context {
 
        /* Azure network adapter */
        struct gdma_dev         mana;
+
+       /* Azure RDMA adapter */
+       struct gdma_dev         mana_ib;
 };
 
 #define MAX_NUM_GDMA_DEVICES   4