rds: zerocopy Tx support.

[sfrench/cifs-2.6.git] / net / rds / message.c

diff --git a/net/rds/message.c b/net/rds/message.c
index bf1a656b198a06eeae26db7c760d6173d87950a8..651834513481a3db69f62e884f1b2e9f84d833f8 100644 (file)
--- a/net/rds/message.c
+++ b/net/rds/message.c
@@ -341,12 +341,14 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in
        return rm;
 }
 
-int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from)
+int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
+                              bool zcopy)
 {
        unsigned long to_copy, nbytes;
        unsigned long sg_off;
        struct scatterlist *sg;
        int ret = 0;
+       int length = iov_iter_count(from);
 
        rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));
 
@@ -356,6 +358,53 @@ int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from)
        sg = rm->data.op_sg;
        sg_off = 0; /* Dear gcc, sg->page will be null from kzalloc. */
 
+       if (zcopy) {
+               int total_copied = 0;
+               struct sk_buff *skb;
+
+               skb = alloc_skb(SO_EE_ORIGIN_MAX_ZCOOKIES * sizeof(u32),
+                               GFP_KERNEL);
+               if (!skb)
+                       return -ENOMEM;
+               rm->data.op_mmp_znotifier = RDS_ZCOPY_SKB(skb);
+               if (mm_account_pinned_pages(&rm->data.op_mmp_znotifier->z_mmp,
+                                           length)) {
+                       ret = -ENOMEM;
+                       goto err;
+               }
+               while (iov_iter_count(from)) {
+                       struct page *pages;
+                       size_t start;
+                       ssize_t copied;
+
+                       copied = iov_iter_get_pages(from, &pages, PAGE_SIZE,
+                                                   1, &start);
+                       if (copied < 0) {
+                               struct mmpin *mmp;
+                               int i;
+
+                               for (i = 0; i < rm->data.op_nents; i++)
+                                       put_page(sg_page(&rm->data.op_sg[i]));
+                               mmp = &rm->data.op_mmp_znotifier->z_mmp;
+                               mm_unaccount_pinned_pages(mmp);
+                               ret = -EFAULT;
+                               goto err;
+                       }
+                       total_copied += copied;
+                       iov_iter_advance(from, copied);
+                       length -= copied;
+                       sg_set_page(sg, pages, copied, start);
+                       rm->data.op_nents++;
+                       sg++;
+               }
+               WARN_ON_ONCE(length != 0);
+               return ret;
+err:
+               consume_skb(skb);
+               rm->data.op_mmp_znotifier = NULL;
+               return ret;
+       } /* zcopy */
+
        while (iov_iter_count(from)) {
                if (!sg_page(sg)) {
                        ret = rds_page_remainder_alloc(sg, iov_iter_count(from),