x86: explicitly align IO accesses in memcpy_{to,from}io

[sfrench/cifs-2.6.git] / arch / x86 / lib / iomem.c

diff --git a/arch/x86/lib/iomem.c b/arch/x86/lib/iomem.c
index 66894675f3c8d3ac65c4efdae5039d904110c9b2..df50451d94ef7edf095defd0253aa7bb329123ab 100644 (file)
--- a/arch/x86/lib/iomem.c
+++ b/arch/x86/lib/iomem.c
@@ -2,8 +2,11 @@
 #include <linux/module.h>
 #include <linux/io.h>
 
+#define movs(type,to,from) \
+       asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory")
+
 /* Originally from i386/string.h */
-static __always_inline void __iomem_memcpy(void *to, const void *from, size_t n)
+static __always_inline void rep_movs(void *to, const void *from, size_t n)
 {
        unsigned long d0, d1, d2;
        asm volatile("rep ; movsl\n\t"
@@ -21,13 +24,37 @@ static __always_inline void __iomem_memcpy(void *to, const void *from, size_t n)
 
 void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n)
 {
-       __iomem_memcpy(to, (const void *)from, n);
+       if (unlikely(!n))
+               return;
+
+       /* Align any unaligned source IO */
+       if (unlikely(1 & (unsigned long)from)) {
+               movs("b", to, from);
+               n--;
+       }
+       if (n > 1 && unlikely(2 & (unsigned long)from)) {
+               movs("w", to, from);
+               n-=2;
+       }
+       rep_movs(to, (const void *)from, n);
 }
 EXPORT_SYMBOL(memcpy_fromio);
 
 void memcpy_toio(volatile void __iomem *to, const void *from, size_t n)
 {
-       __iomem_memcpy((void *)to, (const void *) from, n);
+       if (unlikely(!n))
+               return;
+
+       /* Align any unaligned destination IO */
+       if (unlikely(1 & (unsigned long)to)) {
+               movs("b", to, from);
+               n--;
+       }
+       if (n > 1 && unlikely(2 & (unsigned long)to)) {
+               movs("w", to, from);
+               n-=2;
+       }
+       rep_movs((void *)to, (const void *) from, n);
 }
 EXPORT_SYMBOL(memcpy_toio);