xarray: Add range store functionality

This version of xa_store_range() really only supports load and store.
Our only user only needs basic load and store functionality, so there's
no need to do the extra work to support marking and overlapping stores
correctly yet.

Signed-off-by: Matthew Wilcox <willy@infradead.org>

diff --git a/Documentation/core-api/xarray.rst b/Documentation/core-api/xarray.rst
index 463e4c798ec1bfeae019e7bfcba64d70617af043..a4e705108f428e970b4c93be376ed3644dcb3649 100644 (file)
--- a/Documentation/core-api/xarray.rst
+++ b/Documentation/core-api/xarray.rst
@@ -98,6 +98,13 @@ the XArray by calling :c:func:`xa_for_each`.  You may prefer to use
 :c:func:`xa_find` or :c:func:`xa_find_after` to move to the next present
 entry in the XArray.
 
+Calling :c:func:`xa_store_range` stores the same entry in a range
+of indices.  If you do this, some of the other operations will behave
+in a slightly odd way.  For example, marking the entry at one index
+may result in the entry being marked at some, but not all of the other
+indices.  Storing into one index may result in the entry retrieved by
+some, but not all of the other indices changing.
+
 Finally, you can remove all entries from an XArray by calling
 :c:func:`xa_destroy`.  If the XArray entries are pointers, you may wish
 to free the entries first.  You can do this by iterating over all present
@@ -156,6 +163,7 @@ Takes xa_lock internally:
  * :c:func:`xa_erase_bh`
  * :c:func:`xa_erase_irq`
  * :c:func:`xa_cmpxchg`
+ * :c:func:`xa_store_range`
  * :c:func:`xa_alloc`
  * :c:func:`xa_alloc_bh`
  * :c:func:`xa_alloc_irq`

diff --git a/include/linux/xarray.h b/include/linux/xarray.h
index e0b57af52e9b2a469079a25171efa2034192f86d..d9514928ddacb0b873b2824100599d5daa05ad50 100644 (file)
--- a/include/linux/xarray.h
+++ b/include/linux/xarray.h
@@ -292,6 +292,8 @@ void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
 void *xa_cmpxchg(struct xarray *, unsigned long index,
                        void *old, void *entry, gfp_t);
 int xa_reserve(struct xarray *, unsigned long index, gfp_t);
+void *xa_store_range(struct xarray *, unsigned long first, unsigned long last,
+                       void *entry, gfp_t);
 bool xa_get_mark(struct xarray *, unsigned long index, xa_mark_t);
 void xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
 void xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);

diff --git a/lib/test_xarray.c b/lib/test_xarray.c
index 0f06a93b4d0e8f4e4a3e55cdf3d65466310e279a..aa47754150cee9a0374ca16195f17c061225a538 100644 (file)
--- a/lib/test_xarray.c
+++ b/lib/test_xarray.c
@@ -1039,6 +1039,39 @@ static noinline void check_create_range(struct xarray *xa)
        check_create_range_3();
 }
 
+static noinline void __check_store_range(struct xarray *xa, unsigned long first,
+               unsigned long last)
+{
+#ifdef CONFIG_XARRAY_MULTI
+       xa_store_range(xa, first, last, xa_mk_value(first), GFP_KERNEL);
+
+       XA_BUG_ON(xa, xa_load(xa, first) != xa_mk_value(first));
+       XA_BUG_ON(xa, xa_load(xa, last) != xa_mk_value(first));
+       XA_BUG_ON(xa, xa_load(xa, first - 1) != NULL);
+       XA_BUG_ON(xa, xa_load(xa, last + 1) != NULL);
+
+       xa_store_range(xa, first, last, NULL, GFP_KERNEL);
+#endif
+
+       XA_BUG_ON(xa, !xa_empty(xa));
+}
+
+static noinline void check_store_range(struct xarray *xa)
+{
+       unsigned long i, j;
+
+       for (i = 0; i < 128; i++) {
+               for (j = i; j < 128; j++) {
+                       __check_store_range(xa, i, j);
+                       __check_store_range(xa, 128 + i, 128 + j);
+                       __check_store_range(xa, 4095 + i, 4095 + j);
+                       __check_store_range(xa, 4096 + i, 4096 + j);
+                       __check_store_range(xa, 123456 + i, 123456 + j);
+                       __check_store_range(xa, UINT_MAX + i, UINT_MAX + j);
+               }
+       }
+}
+
 static LIST_HEAD(shadow_nodes);
 
 static void test_update_node(struct xa_node *node)
@@ -1184,6 +1217,7 @@ static int xarray_checks(void)
        check_destroy(&array);
        check_move(&array);
        check_create_range(&array);
+       check_store_range(&array);
        check_store_iter(&array);
 
        check_workingset(&array, 0);

diff --git a/lib/xarray.c b/lib/xarray.c
index 9a0d49d4b5f07177088c67d1c538771e67662a0e..8b176f009c0877853087772b18cd53ef15b2c099 100644 (file)
--- a/lib/xarray.c
+++ b/lib/xarray.c
@@ -376,6 +376,14 @@ static void *xas_alloc(struct xa_state *xas, unsigned int shift)
        return node;
 }
 
+#ifdef CONFIG_XARRAY_MULTI
+/* Returns the number of indices covered by a given xa_state */
+static unsigned long xas_size(const struct xa_state *xas)
+{
+       return (xas->xa_sibs + 1UL) << xas->xa_shift;
+}
+#endif
+
 /*
  * Use this to calculate the maximum index that will need to be created
  * in order to add the entry described by @xas.  Because we cannot store a
@@ -388,8 +396,7 @@ static unsigned long xas_max(struct xa_state *xas)
 
 #ifdef CONFIG_XARRAY_MULTI
        if (xas->xa_shift || xas->xa_sibs) {
-               unsigned long mask;
-               mask = (((xas->xa_sibs + 1UL) << xas->xa_shift) - 1);
+               unsigned long mask = xas_size(xas) - 1;
                max |= mask;
                if (mask == max)
                        max++;
@@ -1517,6 +1524,92 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp)
 }
 EXPORT_SYMBOL(xa_reserve);
 
+#ifdef CONFIG_XARRAY_MULTI
+static void xas_set_range(struct xa_state *xas, unsigned long first,
+               unsigned long last)
+{
+       unsigned int shift = 0;
+       unsigned long sibs = last - first;
+       unsigned int offset = XA_CHUNK_MASK;
+
+       xas_set(xas, first);
+
+       while ((first & XA_CHUNK_MASK) == 0) {
+               if (sibs < XA_CHUNK_MASK)
+                       break;
+               if ((sibs == XA_CHUNK_MASK) && (offset < XA_CHUNK_MASK))
+                       break;
+               shift += XA_CHUNK_SHIFT;
+               if (offset == XA_CHUNK_MASK)
+                       offset = sibs & XA_CHUNK_MASK;
+               sibs >>= XA_CHUNK_SHIFT;
+               first >>= XA_CHUNK_SHIFT;
+       }
+
+       offset = first & XA_CHUNK_MASK;
+       if (offset + sibs > XA_CHUNK_MASK)
+               sibs = XA_CHUNK_MASK - offset;
+       if ((((first + sibs + 1) << shift) - 1) > last)
+               sibs -= 1;
+
+       xas->xa_shift = shift;
+       xas->xa_sibs = sibs;
+}
+
+/**
+ * xa_store_range() - Store this entry at a range of indices in the XArray.
+ * @xa: XArray.
+ * @first: First index to affect.
+ * @last: Last index to affect.
+ * @entry: New entry.
+ * @gfp: Memory allocation flags.
+ *
+ * After this function returns, loads from any index between @first and @last,
+ * inclusive will return @entry.
+ * Storing into an existing multislot entry updates the entry of every index.
+ * The marks associated with @index are unaffected unless @entry is %NULL.
+ *
+ * Context: Process context.  Takes and releases the xa_lock.  May sleep
+ * if the @gfp flags permit.
+ * Return: %NULL on success, xa_err(-EINVAL) if @entry cannot be stored in
+ * an XArray, or xa_err(-ENOMEM) if memory allocation failed.
+ */
+void *xa_store_range(struct xarray *xa, unsigned long first,
+               unsigned long last, void *entry, gfp_t gfp)
+{
+       XA_STATE(xas, xa, 0);
+
+       if (WARN_ON_ONCE(xa_is_internal(entry)))
+               return XA_ERROR(-EINVAL);
+       if (last < first)
+               return XA_ERROR(-EINVAL);
+
+       do {
+               xas_lock(&xas);
+               if (entry) {
+                       unsigned int order = (last == ~0UL) ? 64 :
+                                               ilog2(last + 1);
+                       xas_set_order(&xas, last, order);
+                       xas_create(&xas);
+                       if (xas_error(&xas))
+                               goto unlock;
+               }
+               do {
+                       xas_set_range(&xas, first, last);
+                       xas_store(&xas, entry);
+                       if (xas_error(&xas))
+                               goto unlock;
+                       first += xas_size(&xas);
+               } while (first <= last);
+unlock:
+               xas_unlock(&xas);
+       } while (xas_nomem(&xas, gfp));
+
+       return xas_result(&xas, NULL);
+}
+EXPORT_SYMBOL(xa_store_range);
+#endif /* CONFIG_XARRAY_MULTI */
+
 /**
  * __xa_alloc() - Find somewhere to store this entry in the XArray.
  * @xa: XArray.