Implementing basic data structure handling...

(This used to be ctdb commit abc571a9948b51a436bacaae89b49b8a0e08a093)

diff --git a/ctdb/ib/ibwrapper.c b/ctdb/ib/ibwrapper.c
index 5d2d35449b67d9bf313a49f8f7271983200d7ce9..9778b4e04196e43a7ab8f7d0bff848152586abd3 100644 (file)
--- a/ctdb/ib/ibwrapper.c
+++ b/ctdb/ib/ibwrapper.c
@@ -46,22 +46,192 @@ static ibw_mr *ibw_alloc_mr(ibw_ctx_priv *pctx)
 {
 }
 
-static int ibwctx_destruct(void *ptr)
+static int ibw_ctx_priv_destruct(void *ptr)
 {
-       ibw_ctx *pctx = talloc_get_type(ptr, ibw_ctx);
+       ibw_ctx *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
        assert(pctx!=NULL);
 
+       if (pctx->cm_id) {
+               rdma_destroy_id(pctx->cm_id);
+               pctx->cm_id = NULL;
+       }
+       if (pctx->cm_channel) {
+               rdma_destroy_event_channel(pctx->cm_channel);
+               pctx->cm_channel = NULL;
+       }
+
+       /* free memory regions */
+}
+
+static int ibw_ctx_destruct(void *ptr)
+{
+       ibw_ctx *ctx = talloc_get_type(ptr, ibw_ctx);
+       assert(ctx!=NULL);
+
+       if (pconn->cm_id) {
+               rdma_destroy_id(pconn->cm_id);
+               pconn->cm_id = NULL;
+       }
+
        /* free memory regions */
 
        return 0;
 }
 
-int ibw_process_event(ibw_ctx *ctx, int fd_index);
+static int ibw_conn_priv_destruct(void *ptr)
+{
+       ibw_conn *pconn = talloc_get_type(ptr, ibw_conn_priv);
+       assert(pconn!=NULL);
+}
+
+static int ibw_conn_destruct(void *ptr)
+{
+       ibw_conn *conn = talloc_get_type(ptr, ibw_conn);
+       ibw_ctx *ctx;
+
+       assert(conn!=NULL);
+       ctx = ibw_conn->ctx;
+       assert(ctx!=NULL);
+
+       /* unhook conn from ctx's linked list */
+       assert(ctx->first_conn!=NULL);
+       assert(ctx->last_conn!=NULL);
+
+       if (conn->prev==NULL) {
+               assert(ctx->first_conn==conn);
+               ctx->first_conn = conn->next;
+       } else {
+               conn->prev->next = conn->next;
+       }
+
+       if (conn->next==NULL) {
+               assert(ctx->last_conn==conn);
+               ctx->last_conn = conn->prev;
+       } else {
+               conn->next->prev = conn->prev;
+       }
+       return 0;
+}
+
+static ibw_conn *ibw_new_conn(ibw_ctx *ctx)
+{
+       ibw_conn *conn;
+       ibw_conn_priv *pconn;
+
+       conn = talloc_zero(ctx, ibw_conn);
+       assert(conn!=NULL);
+       talloc_set_destructor(conn, ibw_conn_destruct);
+
+       pconn = talloc_zero(ctx, ibw_conn_priv);
+       assert(pconn!=NULL);
+       talloc_set_destructor(pconn, ibw_conn_priv_destruct);
+
+       conn->ctx = ctx;
+
+       /* append conn to the end of ctx's linked list */
+       conn->prev = ctx->last_conn;
+       conn->next = NULL;
+       if (ctx->first_conn) {
+               assert(ctx->last_conn!=NULL);
+               conn->prev->next = conn;
+       } else {
+               ctx->first_conn = ctx->last_conn = conn;
+       }
+
+       return conn;
+}
 
 static void ibw_process_cm_event(struct event_context *ev,
        struct fd_event *fde, uint16_t flags, void *private_data)
 {
-       if (fde->
+       int     rc;
+       ibw_ctx *ctx = talloc_get_type(private_data, ibw_ctx);
+       ibw_ctx *pctx = talloc_get_type(ctx->internal, ibw_ctx_priv);
+       struct rdma_cm_id *id;
+       struct rdma_cm_event event;
+
+       assert(ctx!=NULL);
+
+       rc = rdma_get_cm_event(cb->cm_channel, &event);
+       if (rc) {
+               ctx->state = IBWS_ERROR;
+               sprintf(ibw_lasterr, "rdma_get_cm_event error %d\n", rc);
+               DEBUG(0, ibw_lasterr);
+               return;
+       }
+       id = &event.id;
+
+       /* find whose cma_id we have */
+
+//     DEBUG(10, "cma_event type %d cma_id %p (%s)\n", event->event, cma_id,
+//               (cma_id == cb->cm_id) ? "parent" : "child");
+
+       switch (event->event) {
+       case RDMA_CM_EVENT_ADDR_RESOLVED:
+               assert(pctx->state==IWINT_INIT);
+               pctx->state = IWINT_ADDR_RESOLVED;
+               ret = rdma_resolve_route(cma_id, 2000);
+               if (ret) {
+                       cb->state = ERROR;
+                       fprintf(stderr, "rdma_resolve_route error %d\n", ret);
+                       sem_post(&cb->sem);
+               }
+               break;
+
+       case RDMA_CM_EVENT_ROUTE_RESOLVED:
+               assert(pctx->state==IWINT_ADDR_RESOLVED);
+               pctx->state = IWINT_ROUTE_RESOLVED;
+               sem_post(&cb->sem);
+               break;
+
+       case RDMA_CM_EVENT_CONNECT_REQUEST:
+               cb->state = CONNECT_REQUEST;
+               cb->child_cm_id = cma_id;
+               DEBUG_LOG("child cma %p\n", cb->child_cm_id);
+               sem_post(&cb->sem);
+               break;
+
+       case RDMA_CM_EVENT_ESTABLISHED:
+               DEBUG_LOG("ESTABLISHED\n");
+
+               /*
+                * Server will wake up when first RECV completes.
+                */
+               if (!cb->server) {
+                       cb->state = CONNECTED;
+               }
+               sem_post(&cb->sem);
+               break;
+
+       case RDMA_CM_EVENT_ADDR_ERROR:
+       case RDMA_CM_EVENT_ROUTE_ERROR:
+       case RDMA_CM_EVENT_CONNECT_ERROR:
+       case RDMA_CM_EVENT_UNREACHABLE:
+       case RDMA_CM_EVENT_REJECTED:
+               fprintf(stderr, "cma event %d, error %d\n", event->event,
+                      event->status);
+               sem_post(&cb->sem);
+               ret = -1;
+               break;
+
+       case RDMA_CM_EVENT_DISCONNECTED:
+               fprintf(stderr, "%s DISCONNECT EVENT...\n", cb->server ? "server" : "client");
+               sem_post(&cb->sem);
+               break;
+
+       case RDMA_CM_EVENT_DEVICE_REMOVAL:
+               fprintf(stderr, "cma detected device removal!!!!\n");
+               ret = -1;
+               break;
+
+       default:
+               fprintf(stderr, "oof bad type!\n");
+               sem_post(&cb->sem);
+               ret = -1;
+               break;
+       }
+
+       rdma_ack_cm_event(event);
 }
 
 static int ibw_process_init_attrs(ibw_initattr *attr, int nattr, ibw_opts *opts)
@@ -96,16 +266,17 @@ ibw_ctx *ibw_init(ibw_initattr *attr, int nattr,
        ibw_ctx *ctx = talloc_zero(NULL, ibw_ctx);
        ibw_ctx_priv *pctx;
        int     rc;
-       ibw_event_ud *event_priv;
 
+       /* initialize basic data structures */
        memset(ibw_lasterr, 0, IBW_LASTERR_BUFSIZE);
 
        assert(ctx!=NULL);
        ibw_lasterr[0] = '\0';
-       talloc_set_destructor(ctx, ibwctx_destruct);
+       talloc_set_destructor(ctx, ibw_ctx_destruct);
        ctx->userdata = userdata;
 
        pctx = talloc_zero(ctx, ibw_ctx_priv);
+       talloc_set_destructor(pctx, ibw_ctx_priv_destruct);
        ctx->internal = (void *)pctx;
        assert(pctx!=NULL);
 
@@ -126,12 +297,8 @@ ibw_ctx *ibw_init(ibw_initattr *attr, int nattr,
                goto cleanup;
        }
 
-       event_priv = talloc_zero(ctx, ibw_event_ud);
-       event_priv->ctx = ctx;
-       event_priv->id = IBWET_CM;
-
        pctx->cm_channel_event = event_add_fd(pctx->ectx, pctx,
-               pctx->cm_channel->fd, EVENT_FD_READ, ibw_process_cm_event, event_priv);
+               pctx->cm_channel->fd, EVENT_FD_READ, ibw_process_cm_event, ctx);
 
        rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, cb, RDMA_PS_TCP);
        if (rc) {
@@ -161,40 +328,56 @@ int ibw_stop(ibw_ctx *ctx)
 int ibw_bind(ibw_ctx *ctx, struct sockaddr_in *my_addr)
 {
        ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
+       int     rc;
+
+       rc = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &my_addr);
+       if (rc) {
+               sprintf(ibw_lasterr, "rdma_bind_addr error %d\n", rc);
+               return rc;
+       }
+
+       return 0;
 }
 
 int ibw_listen(ibw_ctx *ctx, int backlog)
 {
        ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
+       
+       return 0;
 }
 
 int ibw_accept(ibw_ctx *ctx, void *conn_userdata)
 {
        ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
+       
+       return 0;
 }
 
 int ibw_connect(ibw_ctx *ctx, struct sockaddr_in *serv_addr, void *conn_userdata)
 {
        ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
+               
+       return 0;
 }
 
 void ibw_disconnect(ibw_conn *conn)
 {
        ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
-}
-
-int ibw_process_event(ibw_ctx *ctx, ...)
-{
-       ibw_ctx_priv *pctx = (ibw_ctx_priv *)ctx->internal;
+       
+       return 0;
 }
 
 int ibw_alloc_send_buf(ibw_conn *conn, void **buf, void **key, int n)
 {
+       ibw_conn_priv *pconn = (ibw_ctx_priv *)ctx->internal;
+
+       return 0;
 }
 
 int ibw_send(ibw_conn *conn, void *buf, void *key, int n)
 {
-
+       ibw_conn_priv *pconn = (ibw_ctx_priv *)ctx->internal;
+       return 0;
 }
 
 const char *ibw_getLastError()

diff --git a/ctdb/ib/ibwrapper.h b/ctdb/ib/ibwrapper.h
index 8183919020fc86801a2ad3ab099c626c5cdbdac3..c5ca0272cc9bed998e71a87949eb240727c6ce4b 100644 (file)
--- a/ctdb/ib/ibwrapper.h
+++ b/ctdb/ib/ibwrapper.h
@@ -37,6 +37,8 @@ typedef struct _ibw_ctx {
 
        ibw_state_ctx state;
        void *internal;
+
+       ibw_conn *first_conn, *last_conn;
 } ibw_ctx;
 
 typedef enum {
@@ -52,6 +54,8 @@ typedef struct _ibw_conn {
 
        void *conn_userdata; /* see ibw_connect and ibw_accept */
        void *internal;
+
+       ibw_conn *prev, next;
 } ibw_conn;
 
 /*

diff --git a/ctdb/ib/ibwrapper_internal.h b/ctdb/ib/ibwrapper_internal.h
index dbf11f6273ccb13ffcc173226a76da44d9299537..df2f14a2c1459c335af1c2e7846bd1c97e977030 100644 (file)
--- a/ctdb/ib/ibwrapper_internal.h
+++ b/ctdb/ib/ibwrapper_internal.h
@@ -33,6 +33,13 @@ typedef struct _ibw_opts {
        int     ib_port;
 } ibw_opts;
 
+typedef enum {
+       IWINT_INIT = 0,
+       IWINT_ADDR_RESOLVED,
+       IWINT_ROUTE_RESOLVED,
+       IWINT_ERROR
+} ibw_state_ctx;
+
 typedef struct _ibw_ctx_priv {
        ibw_mr *avail_first;
        ibw_mr *avail_last;
@@ -45,10 +52,10 @@ typedef struct _ibw_ctx_priv {
 
        struct ibv_context     *context;
        struct ibv_pd          *pd;
+       struct rdma_cm_id       *cm_id; /* server cm id */
 
        struct rdma_event_channel *cm_channel;
        struct fd_event *cm_channel_event;
-       struct rdma_cm_id *cm_id;       /* connection on client side,*/
 
        ibw_connstate_fn_t connstate_func;
        ibw_receive_fn_t receive_func;
@@ -57,18 +64,9 @@ typedef struct _ibw_ctx_priv {
 typedef struct _ibw_conn_priv {
        struct ibv_cq   *cq;
        struct ibv_qp   *qp;
-       struct ib_cm_id *cm_id;
-} ibw_conn_priv;
 
-typedef enum {
-       IBWET_CM,
-       IBWET_VERBS
-} ibw_event_type;
-
-typedef struct _ibw_event_ud {
-       ibw_ctx *ctx;
-       ibw_event_type  id;
-} ibw_event_ud;
+       struct rdma_cm_id *cm_id; /* client's cm id */
+} ibw_conn_priv;
 
 /* 
  * Must be called in all cases after selecting/polling