1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#include <linux/linkage.h>
41#include <asm/inst.h>
42
43
44.align 16
45
46
47
48
49
50
51
52.Lconstant_R2R1:
53 .octa 0x00000001c6e415960000000154442bd4
54
55
56
57
58
59
60
61.Lconstant_R4R3:
62 .octa 0x00000000ccaa009e00000001751997d0
63
64
65
66
67.Lconstant_R5:
68 .octa 0x00000000000000000000000163cd6124
69.Lconstant_mask32:
70 .octa 0x000000000000000000000000FFFFFFFF
71
72
73
74
75
76
77.Lconstant_RUpoly:
78 .octa 0x00000001F701164100000001DB710641
79
80#define CONSTANT %xmm0
81
82#ifdef __x86_64__
83#define BUF %rdi
84#define LEN %rsi
85#define CRC %edx
86#else
87#define BUF %eax
88#define LEN %edx
89#define CRC %ecx
90#endif
91
92
93
94.text
95
96
97
98
99
100
101
102
103
104
105ENTRY(crc32_pclmul_le_16)
106 movdqa (BUF), %xmm1
107 movdqa 0x10(BUF), %xmm2
108 movdqa 0x20(BUF), %xmm3
109 movdqa 0x30(BUF), %xmm4
110 movd CRC, CONSTANT
111 pxor CONSTANT, %xmm1
112 sub $0x40, LEN
113 add $0x40, BUF
114#ifndef __x86_64__
115
116 call delta
117delta:
118 pop %ecx
119#endif
120 cmp $0x40, LEN
121 jb less_64
122
123#ifdef __x86_64__
124 movdqa .Lconstant_R2R1(%rip), CONSTANT
125#else
126 movdqa .Lconstant_R2R1 - delta(%ecx), CONSTANT
127#endif
128
129loop_64:
130 prefetchnta 0x40(BUF)
131 movdqa %xmm1, %xmm5
132 movdqa %xmm2, %xmm6
133 movdqa %xmm3, %xmm7
134#ifdef __x86_64__
135 movdqa %xmm4, %xmm8
136#endif
137 PCLMULQDQ 00, CONSTANT, %xmm1
138 PCLMULQDQ 00, CONSTANT, %xmm2
139 PCLMULQDQ 00, CONSTANT, %xmm3
140#ifdef __x86_64__
141 PCLMULQDQ 00, CONSTANT, %xmm4
142#endif
143 PCLMULQDQ 0x11, CONSTANT, %xmm5
144 PCLMULQDQ 0x11, CONSTANT, %xmm6
145 PCLMULQDQ 0x11, CONSTANT, %xmm7
146#ifdef __x86_64__
147 PCLMULQDQ 0x11, CONSTANT, %xmm8
148#endif
149 pxor %xmm5, %xmm1
150 pxor %xmm6, %xmm2
151 pxor %xmm7, %xmm3
152#ifdef __x86_64__
153 pxor %xmm8, %xmm4
154#else
155
156 movdqa %xmm4, %xmm5
157 PCLMULQDQ 00, CONSTANT, %xmm4
158 PCLMULQDQ 0x11, CONSTANT, %xmm5
159 pxor %xmm5, %xmm4
160#endif
161
162 pxor (BUF), %xmm1
163 pxor 0x10(BUF), %xmm2
164 pxor 0x20(BUF), %xmm3
165 pxor 0x30(BUF), %xmm4
166
167 sub $0x40, LEN
168 add $0x40, BUF
169 cmp $0x40, LEN
170 jge loop_64
171less_64:
172#ifdef __x86_64__
173 movdqa .Lconstant_R4R3(%rip), CONSTANT
174#else
175 movdqa .Lconstant_R4R3 - delta(%ecx), CONSTANT
176#endif
177 prefetchnta (BUF)
178
179 movdqa %xmm1, %xmm5
180 PCLMULQDQ 0x00, CONSTANT, %xmm1
181 PCLMULQDQ 0x11, CONSTANT, %xmm5
182 pxor %xmm5, %xmm1
183 pxor %xmm2, %xmm1
184
185 movdqa %xmm1, %xmm5
186 PCLMULQDQ 0x00, CONSTANT, %xmm1
187 PCLMULQDQ 0x11, CONSTANT, %xmm5
188 pxor %xmm5, %xmm1
189 pxor %xmm3, %xmm1
190
191 movdqa %xmm1, %xmm5
192 PCLMULQDQ 0x00, CONSTANT, %xmm1
193 PCLMULQDQ 0x11, CONSTANT, %xmm5
194 pxor %xmm5, %xmm1
195 pxor %xmm4, %xmm1
196
197 cmp $0x10, LEN
198 jb fold_64
199loop_16:
200 movdqa %xmm1, %xmm5
201 PCLMULQDQ 0x00, CONSTANT, %xmm1
202 PCLMULQDQ 0x11, CONSTANT, %xmm5
203 pxor %xmm5, %xmm1
204 pxor (BUF), %xmm1
205 sub $0x10, LEN
206 add $0x10, BUF
207 cmp $0x10, LEN
208 jge loop_16
209
210fold_64:
211
212
213 PCLMULQDQ 0x01, %xmm1, CONSTANT
214 psrldq $0x08, %xmm1
215 pxor CONSTANT, %xmm1
216
217
218 movdqa %xmm1, %xmm2
219#ifdef __x86_64__
220 movdqa .Lconstant_R5(%rip), CONSTANT
221 movdqa .Lconstant_mask32(%rip), %xmm3
222#else
223 movdqa .Lconstant_R5 - delta(%ecx), CONSTANT
224 movdqa .Lconstant_mask32 - delta(%ecx), %xmm3
225#endif
226 psrldq $0x04, %xmm2
227 pand %xmm3, %xmm1
228 PCLMULQDQ 0x00, CONSTANT, %xmm1
229 pxor %xmm2, %xmm1
230
231
232#ifdef __x86_64__
233 movdqa .Lconstant_RUpoly(%rip), CONSTANT
234#else
235 movdqa .Lconstant_RUpoly - delta(%ecx), CONSTANT
236#endif
237 movdqa %xmm1, %xmm2
238 pand %xmm3, %xmm1
239 PCLMULQDQ 0x10, CONSTANT, %xmm1
240 pand %xmm3, %xmm1
241 PCLMULQDQ 0x00, CONSTANT, %xmm1
242 pxor %xmm2, %xmm1
243 PEXTRD 0x01, %xmm1, %eax
244
245 ret
246ENDPROC(crc32_pclmul_le_16)
247