/* SPDX-License-Identifier: GPL-2.0 */ /* * * Optimized version of the standard memcpy() function * * Inputs: * in0: destination address * in1: source address * in2: number of bytes to copy * Output: * no return value * * Copyright (C) 2000-2001 Hewlett-Packard Co * Stephane Eranian * David Mosberger-Tang */ #include #include GLOBAL_ENTRY(memcpy) # define MEM_LAT 21 /* latency to memory */ # define dst r2 # define src r3 # define retval r8 # define saved_pfs r9 # define saved_lc r10 # define saved_pr r11 # define cnt r16 # define src2 r17 # define t0 r18 # define t1 r19 # define t2 r20 # define t3 r21 # define t4 r22 # define src_end r23 # define N (MEM_LAT + 4) # define Nrot ((N + 7) & ~7) /* * First, check if everything (src, dst, len) is a multiple of eight. If * so, we handle everything with no taken branches (other than the loop * itself) and a small icache footprint. Otherwise, we jump off to * the more general copy routine handling arbitrary * sizes/alignment etc. */ .prologue .save ar.pfs, saved_pfs alloc saved_pfs=ar.pfs,3,Nrot,0,Nrot .save ar.lc, saved_lc mov saved_lc=ar.lc or t0=in0,in1 ;; or t0=t0,in2 .save pr, saved_pr mov saved_pr=pr .body cmp.eq p6,p0=in2,r0 // zero length? mov retval=in0 // return dst (p6) br.ret.spnt.many rp // zero length, return immediately ;; mov dst=in0 // copy because of rotation shr.u cnt=in2,3 // number of 8-byte words to copy mov pr.rot=1<<16 ;; adds cnt=-1,cnt // br.ctop is repeat/until cmp.gtu p7,p0=16,in2 // copying less than 16 bytes? mov ar.ec=N ;; and t0=0x7,t0 mov ar.lc=cnt ;; cmp.ne p6,p0=t0,r0 mov src=in1 // copy because of rotation (p7) br.cond.spnt.few .memcpy_short (p6) br.cond.spnt.few .memcpy_long ;; nop.m 0 ;; nop.m 0 nop.i 0 ;; nop.m 0 ;; .rotr val[N] .rotp p[N] .align 32 1: { .mib (p[0]) ld8 val[0]=[src],8 nop.i 0 brp.loop.imp 1b, 2f } 2: { .mfb (p[N-1])st8 [dst]=val[N-1],8 nop.f 0 br.ctop.dptk.few 1b } ;; mov ar.lc=saved_lc mov pr=saved_pr,-1 mov ar.pfs=saved_pfs br.ret.sptk.many rp /* * Small (<16 bytes) unaligned copying is done via a simple byte-at-the-time * copy loop. This performs relatively poorly on Itanium, but it doesn't * get used very often (gcc inlines small copies) and due to atomicity * issues, we want to avoid read-modify-write of entire words. */ .align 32 .memcpy_short: adds cnt=-1,in2 // br.ctop is repeat/until mov ar.ec=MEM_LAT brp.loop.imp 1f, 2f ;; mov ar.lc=cnt ;; nop.m 0 ;; nop.m 0 nop.i 0 ;; nop.m 0 ;; nop.m 0 ;; /* * It is faster to put a stop bit in the loop here because it makes * the pipeline shorter (and latency is what matters on short copies). */ .align 32 1: { .mib (p[0]) ld1 val[0]=[src],1 nop.i 0 brp.loop.imp 1b, 2f } ;; 2: { .mfb (p[MEM_LAT-1])st1 [dst]=val[MEM_LAT-1],1 nop.f 0 br.ctop.dptk.few 1b } ;; mov ar.lc=saved_lc mov pr=saved_pr,-1 mov ar.pfs=saved_pfs br.ret.sptk.many rp /* * Large (>= 16 bytes) copying is done in a fancy way. Latency isn't * an overriding concern here, but throughput is. We first do * sub-word copying until the destination is aligned, then we check * if the source is also aligned. If so, we do a simple load/store-loop * until there are less than 8 bytes left over and then we do the tail, * by storing the last few bytes using sub-word copying. If the source * is not aligned, we branch off to the non-congruent loop. * * stage: op: * 0 ld * : * MEM_LAT+3 shrp * MEM_LAT+4 st * * On Itanium, the pipeline itself runs without stalls. However, br.ctop * seems to introduce an unavoidable bubble in the pipeline so the overall * latency is 2 cycles/iteration. This gives us a _copy_ throughput * of 4 byte/cycle. Still not bad. */ # undef N # undef Nrot # define N (MEM_LAT + 5) /* number of stages */ # define Nrot ((N+1 + 2 + 7) & ~7) /* number of rotating regs */ #define LOG_LOOP_SIZE 6 .memcpy_long: alloc t3=ar.pfs,3,Nrot,0,Nrot // resize register frame and t0=-8,src // t0 = src & ~7 and t2=7,src // t2 = src & 7 ;; ld8 t0=[t0] // t0 = 1st source word adds src2=7,src // src2 = (src + 7) sub t4=r0,dst // t4 = -dst ;; and src2=-8,src2 // src2 = (src + 7) & ~7 shl t2=t2,3 // t2 = 8*(src & 7) shl t4=t4,3 // t4 = 8*(dst & 7) ;; ld8 t1=[src2] // t1 = 1st source word if src is 8-byte aligned, 2nd otherwise sub t3=64,t2 // t3 = 64-8*(src & 7) shr.u t0=t0,t2 ;; add src_end=src,in2 shl t1=t1,t3 mov pr=t4,0x38 // (p5,p4,p3)=(dst & 7) ;; or t0=t0,t1 mov cnt=r0 adds src_end=-1,src_end ;; (p3) st1 [dst]=t0,1 (p3) shr.u t0=t0,8 (p3) adds cnt=1,cnt ;; (p4) st2 [dst]=t0,2 (p4) shr.u t0=t0,16 (p4) adds cnt=2,cnt ;; (p5) st4 [dst]=t0,4 (p5) adds cnt=4,cnt and src_end=-8,src_end // src_end = last word of source buffer ;; // At this point, dst is aligned to 8 bytes and there at least 16-7=9 bytes left to copy: 1:{ add src=cnt,src // make src point to remainder of source buffer sub cnt=in2,cnt // cnt = number of bytes left to copy mov t4=ip } ;; and src2=-8,src // align source pointer adds t4=.memcpy_loops-1b,t4 mov ar.ec=N and t0=7,src // t0 = src & 7 shr.u t2=cnt,3 // t2 = number of 8-byte words left to copy shl cnt=cnt,3 // move bits 0-2 to 3-5 ;; .rotr val[N+1], w[2] .rotp p[N] cmp.ne p6,p0=t0,r0 // is src aligned, too? shl t0=t0,LOG_LOOP_SIZE // t0 = 8*(src & 7) adds t2=-1,t2 // br.ctop is repeat/until ;; add t4=t0,t4 mov pr=cnt,0x38 // set (p5,p4,p3) to # of bytes last-word bytes to copy mov ar.lc=t2 ;; nop.m 0 ;; nop.m 0 nop.i 0 ;; nop.m 0 ;; (p6) ld8 val[1]=[src2],8 // prime the pump... mov b6=t4 br.sptk.few b6 ;; .memcpy_tail: // At this point, (p5,p4,p3) are set to the number of bytes left to copy (which is // less than 8) and t0 contains the last few bytes of the src buffer: (p5) st4 [dst]=t0,4 (p5) shr.u t0=t0,32 mov ar.lc=saved_lc ;; (p4) st2 [dst]=t0,2 (p4) shr.u t0=t0,16 mov ar.pfs=saved_pfs ;; (p3) st1 [dst]=t0 mov pr=saved_pr,-1 br.ret.sptk.many rp /////////////////////////////////////////////////////// .align 64 #define COPY(shift,index) \ 1: { .mib \ (p[0]) ld8 val[0]=[src2],8; \ (p[MEM_LAT+3]) shrp w[0]=val[MEM_LAT+3],val[MEM_LAT+4-index],shift; \ brp.loop.imp 1b, 2f \ }; \ 2: { .mfb \ (p[MEM_LAT+4]) st8 [dst]=w[1],8; \ nop.f 0; \ br.ctop.dptk.few 1b; \ }; \ ;; \ ld8 val[N-1]=[src_end]; /* load last word (may be same as val[N]) */ \ ;; \ shrp t0=val[N-1],val[N-index],shift; \ br .memcpy_tail .memcpy_loops: COPY(0, 1) /* no point special casing this---it doesn't go any faster without shrp */ COPY(8, 0) COPY(16, 0) COPY(24, 0) COPY(32, 0) COPY(40, 0) COPY(48, 0) COPY(56, 0) END(memcpy) EXPORT_SYMBOL(memcpy)