rxrpc: Reduce the use of RCU in packet input

Shrink the region of rxrpc_input_packet() that is covered by the RCU read
lock so that it only covers the connection and call lookup.  This means
that the bits now outside of that can call sleepable functions such as
kmalloc and sendmsg.

Also take a ref on the conn or call we're going to use before we drop the
RCU read lock.

Signed-off-by: David Howells <dhowells@redhat.com>
cc: Marc Dionne <marc.dionne@auristor.com>
cc: linux-afs@lists.infradead.org

diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h
index 6af7298af39b9286c3a8e5e8ae077bbefe182bf7..cfd16f1e5c838ae861ffa9f95e16b1f8c56048af 100644 (file)
--- a/net/rxrpc/ar-internal.h
+++ b/net/rxrpc/ar-internal.h
@@ -961,8 +961,7 @@ void rxrpc_unpublish_service_conn(struct rxrpc_connection *);
  * input.c
  */
 void rxrpc_input_call_event(struct rxrpc_call *, struct sk_buff *);
-void rxrpc_input_implicit_end_call(struct rxrpc_sock *, struct rxrpc_connection *,
-                                  struct rxrpc_call *);
+void rxrpc_input_implicit_end_call(struct rxrpc_connection *, struct rxrpc_call *);
 
 /*
  * io_thread.c

diff --git a/net/rxrpc/call_accept.c b/net/rxrpc/call_accept.c
index 5f978b0b2404c174f068a0e3c39ebd7b0a7bbe34..beb8efa2e7a98a22865d7fee1bc12fcc1e4be22d 100644 (file)
--- a/net/rxrpc/call_accept.c
+++ b/net/rxrpc/call_accept.c
@@ -336,13 +336,13 @@ static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx,
  * If this is for a kernel service, when we allocate the call, it will have
  * three refs on it: (1) the kernel service, (2) the user_call_ID tree, (3) the
  * retainer ref obtained from the backlog buffer.  Prealloc calls for userspace
- * services only have the ref from the backlog buffer.  We want to pass this
- * ref to non-BH context to dispose of.
+ * services only have the ref from the backlog buffer.  We pass this ref to the
+ * caller.
  *
  * If we want to report an error, we mark the skb with the packet type and
  * abort code and return NULL.
  *
- * The call is returned with the user access mutex held.
+ * The call is returned with the user access mutex held and a ref on it.
  */
 struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local,
                                           struct rxrpc_sock *rx,
@@ -426,13 +426,6 @@ struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local,
 
        rxrpc_send_ping(call, skb);
 
-       /* We have to discard the prealloc queue's ref here and rely on a
-        * combination of the RCU read lock and refs held either by the socket
-        * (recvmsg queue, to-be-accepted queue or user ID tree) or the kernel
-        * service to prevent the call from being deallocated too early.
-        */
-       rxrpc_put_call(call, rxrpc_call_put_discard_prealloc);
-
        if (hlist_unhashed(&call->error_link)) {
                spin_lock(&call->peer->lock);
                hlist_add_head(&call->error_link, &call->peer->error_targets);

diff --git a/net/rxrpc/input.c b/net/rxrpc/input.c
index 42addbcf59f9808d147b1ec1cbc868f37eeee49c..01d32f817a7ac59d97c43edf40ec70faf84a91b3 100644 (file)
--- a/net/rxrpc/input.c
+++ b/net/rxrpc/input.c
@@ -1072,8 +1072,7 @@ void rxrpc_input_call_event(struct rxrpc_call *call, struct sk_buff *skb)
  *
  * TODO: If callNumber > call_id + 1, renegotiate security.
  */
-void rxrpc_input_implicit_end_call(struct rxrpc_sock *rx,
-                                  struct rxrpc_connection *conn,
+void rxrpc_input_implicit_end_call(struct rxrpc_connection *conn,
                                   struct rxrpc_call *call)
 {
        switch (READ_ONCE(call->state)) {
@@ -1091,7 +1090,7 @@ void rxrpc_input_implicit_end_call(struct rxrpc_sock *rx,
                break;
        }
 
-       spin_lock(&rx->incoming_lock);
+       spin_lock(&conn->bundle->channel_lock);
        __rxrpc_disconnect_call(conn, call);
-       spin_unlock(&rx->incoming_lock);
+       spin_unlock(&conn->bundle->channel_lock);
 }

diff --git a/net/rxrpc/io_thread.c b/net/rxrpc/io_thread.c
index 91b8ba5b90dbb46fa739035ce0d82fd02e153c40..3b6927610677c706ccdbb993f79c6923c516f921 100644 (file)
--- a/net/rxrpc/io_thread.c
+++ b/net/rxrpc/io_thread.c
@@ -257,6 +257,8 @@ static int rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb)
        if (sp->hdr.serviceId == 0)
                goto bad_message;
 
+       rcu_read_lock();
+
        if (rxrpc_to_server(sp)) {
                /* Weed out packets to services we're not offering.  Packets
                 * that would begin a call are explicitly rejected and the rest
@@ -264,7 +266,9 @@ static int rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb)
                 */
                rx = rcu_dereference(local->service);
                if (!rx || (sp->hdr.serviceId != rx->srx.srx_service &&
-                           sp->hdr.serviceId != rx->second_service)) {
+                           sp->hdr.serviceId != rx->second_service)
+                   ) {
+                       rcu_read_unlock();
                        if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA &&
                            sp->hdr.seq == 1)
                                goto unsupported_service;
@@ -293,7 +297,12 @@ static int rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb)
                if (sp->hdr.callNumber == 0) {
                        /* Connection-level packet */
                        _debug("CONN %p {%d}", conn, conn->debug_id);
-                       rxrpc_post_packet_to_conn(conn, skb);
+                       conn = rxrpc_get_connection_maybe(conn, rxrpc_conn_get_conn_input);
+                       rcu_read_unlock();
+                       if (conn) {
+                               rxrpc_post_packet_to_conn(conn, skb);
+                               rxrpc_put_connection(conn, rxrpc_conn_put_conn_input);
+                       }
                        return 0;
                }
 
@@ -305,20 +314,26 @@ static int rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb)
                chan = &conn->channels[channel];
 
                /* Ignore really old calls */
-               if (sp->hdr.callNumber < chan->last_call)
+               if (sp->hdr.callNumber < chan->last_call) {
+                       rcu_read_unlock();
                        return 0;
+               }
 
                if (sp->hdr.callNumber == chan->last_call) {
                        if (chan->call ||
-                           sp->hdr.type == RXRPC_PACKET_TYPE_ABORT)
+                           sp->hdr.type == RXRPC_PACKET_TYPE_ABORT) {
+                               rcu_read_unlock();
                                return 0;
+                       }
 
                        /* For the previous service call, if completed
                         * successfully, we discard all further packets.
                         */
                        if (rxrpc_conn_is_service(conn) &&
-                           chan->last_type == RXRPC_PACKET_TYPE_ACK)
+                           chan->last_type == RXRPC_PACKET_TYPE_ACK) {
+                               rcu_read_unlock();
                                return 0;
+                       }
 
                        /* But otherwise we need to retransmit the final packet
                         * from data cached in the connection record.
@@ -328,20 +343,32 @@ static int rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb)
                                                    sp->hdr.seq,
                                                    sp->hdr.serial,
                                                    sp->hdr.flags);
-                       rxrpc_post_packet_to_conn(conn, skb);
+                       conn = rxrpc_get_connection_maybe(conn, rxrpc_conn_get_call_input);
+                       rcu_read_unlock();
+                       if (conn) {
+                               rxrpc_post_packet_to_conn(conn, skb);
+                               rxrpc_put_connection(conn, rxrpc_conn_put_call_input);
+                       }
                        return 0;
                }
 
                call = rcu_dereference(chan->call);
 
                if (sp->hdr.callNumber > chan->call_id) {
-                       if (rxrpc_to_client(sp))
+                       if (rxrpc_to_client(sp)) {
+                               rcu_read_unlock();
                                goto reject_packet;
-                       if (call)
-                               rxrpc_input_implicit_end_call(rx, conn, call);
-                       call = NULL;
+                       }
+                       if (call) {
+                               rxrpc_input_implicit_end_call(conn, call);
+                               chan->call = NULL;
+                               call = NULL;
+                       }
                }
 
+               if (call && !rxrpc_try_get_call(call, rxrpc_call_get_input))
+                       call = NULL;
+
                if (call) {
                        if (sp->hdr.serviceId != call->dest_srx.srx_service)
                                call->dest_srx.srx_service = sp->hdr.serviceId;
@@ -352,23 +379,33 @@ static int rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb)
                }
        }
 
-       if (!call || refcount_read(&call->ref) == 0) {
+       if (!call) {
                if (rxrpc_to_client(sp) ||
-                   sp->hdr.type != RXRPC_PACKET_TYPE_DATA)
+                   sp->hdr.type != RXRPC_PACKET_TYPE_DATA) {
+                       rcu_read_unlock();
                        goto bad_message;
-               if (sp->hdr.seq != 1)
+               }
+               if (sp->hdr.seq != 1) {
+                       rcu_read_unlock();
                        return 0;
+               }
                call = rxrpc_new_incoming_call(local, rx, skb);
-               if (!call)
+               if (!call) {
+                       rcu_read_unlock();
                        goto reject_packet;
+               }
        }
 
+       rcu_read_unlock();
+
        /* Process a call packet. */
        rxrpc_input_call_event(call, skb);
+       rxrpc_put_call(call, rxrpc_call_put_input);
        trace_rxrpc_rx_done(0, 0);
        return 0;
 
 wrong_security:
+       rcu_read_unlock();
        trace_rxrpc_abort(0, "SEC", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
                          RXKADINCONSISTENCY, EBADMSG);
        skb->priority = RXKADINCONSISTENCY;
@@ -381,6 +418,7 @@ unsupported_service:
        goto post_abort;
 
 reupgrade:
+       rcu_read_unlock();
        trace_rxrpc_abort(0, "UPG", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
                          RX_PROTOCOL_ERROR, EBADMSG);
        goto protocol_error;
@@ -433,9 +471,7 @@ int rxrpc_io_thread(void *data)
                        switch (skb->mark) {
                        case RXRPC_SKB_MARK_PACKET:
                                skb->priority = 0;
-                               rcu_read_lock();
                                rxrpc_input_packet(local, &skb);
-                               rcu_read_unlock();
                                trace_rxrpc_rx_done(skb->mark, skb->priority);
                                rxrpc_free_skb(skb, rxrpc_skb_put_input);
                                break;