1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <asm/asmmacro.h>
18#include <asm/export.h>
19
20GLOBAL_ENTRY(memcpy)
21
22# define MEM_LAT 21
23
24# define dst r2
25# define src r3
26# define retval r8
27# define saved_pfs r9
28# define saved_lc r10
29# define saved_pr r11
30# define cnt r16
31# define src2 r17
32# define t0 r18
33# define t1 r19
34# define t2 r20
35# define t3 r21
36# define t4 r22
37# define src_end r23
38
39# define N (MEM_LAT + 4)
40# define Nrot ((N + 7) & ~7)
41
42
43
44
45
46
47
48
49 .prologue
50 .save ar.pfs, saved_pfs
51 alloc saved_pfs=ar.pfs,3,Nrot,0,Nrot
52 .save ar.lc, saved_lc
53 mov saved_lc=ar.lc
54 or t0=in0,in1
55 ;;
56
57 or t0=t0,in2
58 .save pr, saved_pr
59 mov saved_pr=pr
60
61 .body
62
63 cmp.eq p6,p0=in2,r0
64 mov retval=in0
65(p6) br.ret.spnt.many rp
66 ;;
67
68 mov dst=in0
69 shr.u cnt=in2,3
70 mov pr.rot=1<<16
71 ;;
72
73 adds cnt=-1,cnt
74 cmp.gtu p7,p0=16,in2
75 mov ar.ec=N
76 ;;
77
78 and t0=0x7,t0
79 mov ar.lc=cnt
80 ;;
81 cmp.ne p6,p0=t0,r0
82
83 mov src=in1
84(p7) br.cond.spnt.few .memcpy_short
85(p6) br.cond.spnt.few .memcpy_long
86 ;;
87 nop.m 0
88 ;;
89 nop.m 0
90 nop.i 0
91 ;;
92 nop.m 0
93 ;;
94 .rotr val[N]
95 .rotp p[N]
96 .align 32
971: { .mib
98(p[0]) ld8 val[0]=[src],8
99 nop.i 0
100 brp.loop.imp 1b, 2f
101}
1022: { .mfb
103(p[N-1])st8 [dst]=val[N-1],8
104 nop.f 0
105 br.ctop.dptk.few 1b
106}
107 ;;
108 mov ar.lc=saved_lc
109 mov pr=saved_pr,-1
110 mov ar.pfs=saved_pfs
111 br.ret.sptk.many rp
112
113
114
115
116
117
118
119 .align 32
120.memcpy_short:
121 adds cnt=-1,in2
122 mov ar.ec=MEM_LAT
123 brp.loop.imp 1f, 2f
124 ;;
125 mov ar.lc=cnt
126 ;;
127 nop.m 0
128 ;;
129 nop.m 0
130 nop.i 0
131 ;;
132 nop.m 0
133 ;;
134 nop.m 0
135 ;;
136
137
138
139
140 .align 32
1411: { .mib
142(p[0]) ld1 val[0]=[src],1
143 nop.i 0
144 brp.loop.imp 1b, 2f
145} ;;
1462: { .mfb
147(p[MEM_LAT-1])st1 [dst]=val[MEM_LAT-1],1
148 nop.f 0
149 br.ctop.dptk.few 1b
150} ;;
151 mov ar.lc=saved_lc
152 mov pr=saved_pr,-1
153 mov ar.pfs=saved_pfs
154 br.ret.sptk.many rp
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176# undef N
177# undef Nrot
178# define N (MEM_LAT + 5)
179# define Nrot ((N+1 + 2 + 7) & ~7)
180
181#define LOG_LOOP_SIZE 6
182
183.memcpy_long:
184 alloc t3=ar.pfs,3,Nrot,0,Nrot
185 and t0=-8,src
186 and t2=7,src
187 ;;
188 ld8 t0=[t0]
189 adds src2=7,src
190 sub t4=r0,dst
191 ;;
192 and src2=-8,src2
193 shl t2=t2,3
194 shl t4=t4,3
195 ;;
196 ld8 t1=[src2]
197 sub t3=64,t2
198 shr.u t0=t0,t2
199 ;;
200 add src_end=src,in2
201 shl t1=t1,t3
202 mov pr=t4,0x38
203 ;;
204 or t0=t0,t1
205 mov cnt=r0
206 adds src_end=-1,src_end
207 ;;
208(p3) st1 [dst]=t0,1
209(p3) shr.u t0=t0,8
210(p3) adds cnt=1,cnt
211 ;;
212(p4) st2 [dst]=t0,2
213(p4) shr.u t0=t0,16
214(p4) adds cnt=2,cnt
215 ;;
216(p5) st4 [dst]=t0,4
217(p5) adds cnt=4,cnt
218 and src_end=-8,src_end
219 ;;
220
221
222
2231:{ add src=cnt,src
224 sub cnt=in2,cnt
225 mov t4=ip
226 } ;;
227 and src2=-8,src
228 adds t4=.memcpy_loops-1b,t4
229 mov ar.ec=N
230
231 and t0=7,src
232 shr.u t2=cnt,3
233 shl cnt=cnt,3
234 ;;
235
236 .rotr val[N+1], w[2]
237 .rotp p[N]
238
239 cmp.ne p6,p0=t0,r0
240 shl t0=t0,LOG_LOOP_SIZE
241 adds t2=-1,t2
242 ;;
243 add t4=t0,t4
244 mov pr=cnt,0x38
245 mov ar.lc=t2
246 ;;
247 nop.m 0
248 ;;
249 nop.m 0
250 nop.i 0
251 ;;
252 nop.m 0
253 ;;
254(p6) ld8 val[1]=[src2],8
255 mov b6=t4
256 br.sptk.few b6
257 ;;
258
259.memcpy_tail:
260
261
262(p5) st4 [dst]=t0,4
263(p5) shr.u t0=t0,32
264 mov ar.lc=saved_lc
265 ;;
266(p4) st2 [dst]=t0,2
267(p4) shr.u t0=t0,16
268 mov ar.pfs=saved_pfs
269 ;;
270(p3) st1 [dst]=t0
271 mov pr=saved_pr,-1
272 br.ret.sptk.many rp
273
274
275 .align 64
276
277#define COPY(shift,index) \
278 1: { .mib \
279 (p[0]) ld8 val[0]=[src2],8; \
280 (p[MEM_LAT+3]) shrp w[0]=val[MEM_LAT+3],val[MEM_LAT+4-index],shift; \
281 brp.loop.imp 1b, 2f \
282 }; \
283 2: { .mfb \
284 (p[MEM_LAT+4]) st8 [dst]=w[1],8; \
285 nop.f 0; \
286 br.ctop.dptk.few 1b; \
287 }; \
288 ;; \
289 ld8 val[N-1]=[src_end]; \
290 ;; \
291 shrp t0=val[N-1],val[N-index],shift; \
292 br .memcpy_tail
293.memcpy_loops:
294 COPY(0, 1)
295 COPY(8, 0)
296 COPY(16, 0)
297 COPY(24, 0)
298 COPY(32, 0)
299 COPY(40, 0)
300 COPY(48, 0)
301 COPY(56, 0)
302
303END(memcpy)
304EXPORT_SYMBOL(memcpy)
305