Merge branch 'work.get_user_pages_fast' of git://git.kernel.org/pub/scm/linux/kernel...
[sfrench/cifs-2.6.git] / net / ceph / pagevec.c
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/ceph/ceph_debug.h>
3
4 #include <linux/module.h>
5 #include <linux/sched.h>
6 #include <linux/slab.h>
7 #include <linux/file.h>
8 #include <linux/namei.h>
9 #include <linux/writeback.h>
10
11 #include <linux/ceph/libceph.h>
12
13 /*
14  * build a vector of user pages
15  */
16 struct page **ceph_get_direct_page_vector(const void __user *data,
17                                           int num_pages, bool write_page)
18 {
19         struct page **pages;
20         int got = 0;
21         int rc = 0;
22
23         pages = kmalloc(sizeof(*pages) * num_pages, GFP_NOFS);
24         if (!pages)
25                 return ERR_PTR(-ENOMEM);
26
27         while (got < num_pages) {
28                 rc = get_user_pages_fast(
29                     (unsigned long)data + ((unsigned long)got * PAGE_SIZE),
30                     num_pages - got, write_page, pages + got);
31                 if (rc < 0)
32                         break;
33                 BUG_ON(rc == 0);
34                 got += rc;
35         }
36         if (rc < 0)
37                 goto fail;
38         return pages;
39
40 fail:
41         ceph_put_page_vector(pages, got, false);
42         return ERR_PTR(rc);
43 }
44 EXPORT_SYMBOL(ceph_get_direct_page_vector);
45
46 void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty)
47 {
48         int i;
49
50         for (i = 0; i < num_pages; i++) {
51                 if (dirty)
52                         set_page_dirty_lock(pages[i]);
53                 put_page(pages[i]);
54         }
55         kvfree(pages);
56 }
57 EXPORT_SYMBOL(ceph_put_page_vector);
58
59 void ceph_release_page_vector(struct page **pages, int num_pages)
60 {
61         int i;
62
63         for (i = 0; i < num_pages; i++)
64                 __free_pages(pages[i], 0);
65         kfree(pages);
66 }
67 EXPORT_SYMBOL(ceph_release_page_vector);
68
69 /*
70  * allocate a vector new pages
71  */
72 struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags)
73 {
74         struct page **pages;
75         int i;
76
77         pages = kmalloc(sizeof(*pages) * num_pages, flags);
78         if (!pages)
79                 return ERR_PTR(-ENOMEM);
80         for (i = 0; i < num_pages; i++) {
81                 pages[i] = __page_cache_alloc(flags);
82                 if (pages[i] == NULL) {
83                         ceph_release_page_vector(pages, i);
84                         return ERR_PTR(-ENOMEM);
85                 }
86         }
87         return pages;
88 }
89 EXPORT_SYMBOL(ceph_alloc_page_vector);
90
91 /*
92  * copy user data into a page vector
93  */
94 int ceph_copy_user_to_page_vector(struct page **pages,
95                                          const void __user *data,
96                                          loff_t off, size_t len)
97 {
98         int i = 0;
99         int po = off & ~PAGE_MASK;
100         int left = len;
101         int l, bad;
102
103         while (left > 0) {
104                 l = min_t(int, PAGE_SIZE-po, left);
105                 bad = copy_from_user(page_address(pages[i]) + po, data, l);
106                 if (bad == l)
107                         return -EFAULT;
108                 data += l - bad;
109                 left -= l - bad;
110                 po += l - bad;
111                 if (po == PAGE_SIZE) {
112                         po = 0;
113                         i++;
114                 }
115         }
116         return len;
117 }
118 EXPORT_SYMBOL(ceph_copy_user_to_page_vector);
119
120 void ceph_copy_to_page_vector(struct page **pages,
121                                     const void *data,
122                                     loff_t off, size_t len)
123 {
124         int i = 0;
125         size_t po = off & ~PAGE_MASK;
126         size_t left = len;
127
128         while (left > 0) {
129                 size_t l = min_t(size_t, PAGE_SIZE-po, left);
130
131                 memcpy(page_address(pages[i]) + po, data, l);
132                 data += l;
133                 left -= l;
134                 po += l;
135                 if (po == PAGE_SIZE) {
136                         po = 0;
137                         i++;
138                 }
139         }
140 }
141 EXPORT_SYMBOL(ceph_copy_to_page_vector);
142
143 void ceph_copy_from_page_vector(struct page **pages,
144                                     void *data,
145                                     loff_t off, size_t len)
146 {
147         int i = 0;
148         size_t po = off & ~PAGE_MASK;
149         size_t left = len;
150
151         while (left > 0) {
152                 size_t l = min_t(size_t, PAGE_SIZE-po, left);
153
154                 memcpy(data, page_address(pages[i]) + po, l);
155                 data += l;
156                 left -= l;
157                 po += l;
158                 if (po == PAGE_SIZE) {
159                         po = 0;
160                         i++;
161                 }
162         }
163 }
164 EXPORT_SYMBOL(ceph_copy_from_page_vector);
165
166 /*
167  * Zero an extent within a page vector.  Offset is relative to the
168  * start of the first page.
169  */
170 void ceph_zero_page_vector_range(int off, int len, struct page **pages)
171 {
172         int i = off >> PAGE_SHIFT;
173
174         off &= ~PAGE_MASK;
175
176         dout("zero_page_vector_page %u~%u\n", off, len);
177
178         /* leading partial page? */
179         if (off) {
180                 int end = min((int)PAGE_SIZE, off + len);
181                 dout("zeroing %d %p head from %d\n", i, pages[i],
182                      (int)off);
183                 zero_user_segment(pages[i], off, end);
184                 len -= (end - off);
185                 i++;
186         }
187         while (len >= PAGE_SIZE) {
188                 dout("zeroing %d %p len=%d\n", i, pages[i], len);
189                 zero_user_segment(pages[i], 0, PAGE_SIZE);
190                 len -= PAGE_SIZE;
191                 i++;
192         }
193         /* trailing partial page? */
194         if (len) {
195                 dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len);
196                 zero_user_segment(pages[i], 0, len);
197         }
198 }
199 EXPORT_SYMBOL(ceph_zero_page_vector_range);
200