1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <asm/asmmacro.h>
17
18GLOBAL_ENTRY(memcpy)
19
20# define MEM_LAT 21
21
22# define dst r2
23# define src r3
24# define retval r8
25# define saved_pfs r9
26# define saved_lc r10
27# define saved_pr r11
28# define cnt r16
29# define src2 r17
30# define t0 r18
31# define t1 r19
32# define t2 r20
33# define t3 r21
34# define t4 r22
35# define src_end r23
36
37# define N (MEM_LAT + 4)
38# define Nrot ((N + 7) & ~7)
39
40
41
42
43
44
45
46
47 .prologue
48 .save ar.pfs, saved_pfs
49 alloc saved_pfs=ar.pfs,3,Nrot,0,Nrot
50 .save ar.lc, saved_lc
51 mov saved_lc=ar.lc
52 or t0=in0,in1
53 ;;
54
55 or t0=t0,in2
56 .save pr, saved_pr
57 mov saved_pr=pr
58
59 .body
60
61 cmp.eq p6,p0=in2,r0
62 mov retval=in0
63(p6) br.ret.spnt.many rp
64 ;;
65
66 mov dst=in0
67 shr.u cnt=in2,3
68 mov pr.rot=1<<16
69 ;;
70
71 adds cnt=-1,cnt
72 cmp.gtu p7,p0=16,in2
73 mov ar.ec=N
74 ;;
75
76 and t0=0x7,t0
77 mov ar.lc=cnt
78 ;;
79 cmp.ne p6,p0=t0,r0
80
81 mov src=in1
82(p7) br.cond.spnt.few .memcpy_short
83(p6) br.cond.spnt.few .memcpy_long
84 ;;
85 nop.m 0
86 ;;
87 nop.m 0
88 nop.i 0
89 ;;
90 nop.m 0
91 ;;
92 .rotr val[N]
93 .rotp p[N]
94 .align 32
951: { .mib
96(p[0]) ld8 val[0]=[src],8
97 nop.i 0
98 brp.loop.imp 1b, 2f
99}
1002: { .mfb
101(p[N-1])st8 [dst]=val[N-1],8
102 nop.f 0
103 br.ctop.dptk.few 1b
104}
105 ;;
106 mov ar.lc=saved_lc
107 mov pr=saved_pr,-1
108 mov ar.pfs=saved_pfs
109 br.ret.sptk.many rp
110
111
112
113
114
115
116
117 .align 32
118.memcpy_short:
119 adds cnt=-1,in2
120 mov ar.ec=MEM_LAT
121 brp.loop.imp 1f, 2f
122 ;;
123 mov ar.lc=cnt
124 ;;
125 nop.m 0
126 ;;
127 nop.m 0
128 nop.i 0
129 ;;
130 nop.m 0
131 ;;
132 nop.m 0
133 ;;
134
135
136
137
138 .align 32
1391: { .mib
140(p[0]) ld1 val[0]=[src],1
141 nop.i 0
142 brp.loop.imp 1b, 2f
143} ;;
1442: { .mfb
145(p[MEM_LAT-1])st1 [dst]=val[MEM_LAT-1],1
146 nop.f 0
147 br.ctop.dptk.few 1b
148} ;;
149 mov ar.lc=saved_lc
150 mov pr=saved_pr,-1
151 mov ar.pfs=saved_pfs
152 br.ret.sptk.many rp
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174# undef N
175# undef Nrot
176# define N (MEM_LAT + 5)
177# define Nrot ((N+1 + 2 + 7) & ~7)
178
179#define LOG_LOOP_SIZE 6
180
181.memcpy_long:
182 alloc t3=ar.pfs,3,Nrot,0,Nrot
183 and t0=-8,src
184 and t2=7,src
185 ;;
186 ld8 t0=[t0]
187 adds src2=7,src
188 sub t4=r0,dst
189 ;;
190 and src2=-8,src2
191 shl t2=t2,3
192 shl t4=t4,3
193 ;;
194 ld8 t1=[src2]
195 sub t3=64,t2
196 shr.u t0=t0,t2
197 ;;
198 add src_end=src,in2
199 shl t1=t1,t3
200 mov pr=t4,0x38
201 ;;
202 or t0=t0,t1
203 mov cnt=r0
204 adds src_end=-1,src_end
205 ;;
206(p3) st1 [dst]=t0,1
207(p3) shr.u t0=t0,8
208(p3) adds cnt=1,cnt
209 ;;
210(p4) st2 [dst]=t0,2
211(p4) shr.u t0=t0,16
212(p4) adds cnt=2,cnt
213 ;;
214(p5) st4 [dst]=t0,4
215(p5) adds cnt=4,cnt
216 and src_end=-8,src_end
217 ;;
218
219
220
2211:{ add src=cnt,src
222 sub cnt=in2,cnt
223 mov t4=ip
224 } ;;
225 and src2=-8,src
226 adds t4=.memcpy_loops-1b,t4
227 mov ar.ec=N
228
229 and t0=7,src
230 shr.u t2=cnt,3
231 shl cnt=cnt,3
232 ;;
233
234 .rotr val[N+1], w[2]
235 .rotp p[N]
236
237 cmp.ne p6,p0=t0,r0
238 shl t0=t0,LOG_LOOP_SIZE
239 adds t2=-1,t2
240 ;;
241 add t4=t0,t4
242 mov pr=cnt,0x38
243 mov ar.lc=t2
244 ;;
245 nop.m 0
246 ;;
247 nop.m 0
248 nop.i 0
249 ;;
250 nop.m 0
251 ;;
252(p6) ld8 val[1]=[src2],8
253 mov b6=t4
254 br.sptk.few b6
255 ;;
256
257.memcpy_tail:
258
259
260(p5) st4 [dst]=t0,4
261(p5) shr.u t0=t0,32
262 mov ar.lc=saved_lc
263 ;;
264(p4) st2 [dst]=t0,2
265(p4) shr.u t0=t0,16
266 mov ar.pfs=saved_pfs
267 ;;
268(p3) st1 [dst]=t0
269 mov pr=saved_pr,-1
270 br.ret.sptk.many rp
271
272
273 .align 64
274
275#define COPY(shift,index) \
276 1: { .mib \
277 (p[0]) ld8 val[0]=[src2],8; \
278 (p[MEM_LAT+3]) shrp w[0]=val[MEM_LAT+3],val[MEM_LAT+4-index],shift; \
279 brp.loop.imp 1b, 2f \
280 }; \
281 2: { .mfb \
282 (p[MEM_LAT+4]) st8 [dst]=w[1],8; \
283 nop.f 0; \
284 br.ctop.dptk.few 1b; \
285 }; \
286 ;; \
287 ld8 val[N-1]=[src_end]; \
288 ;; \
289 shrp t0=val[N-1],val[N-index],shift; \
290 br .memcpy_tail
291.memcpy_loops:
292 COPY(0, 1)
293 COPY(8, 0)
294 COPY(16, 0)
295 COPY(24, 0)
296 COPY(32, 0)
297 COPY(40, 0)
298 COPY(48, 0)
299 COPY(56, 0)
300
301END(memcpy)
302