1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46#include <asm/inst.h>
47#include <linux/linkage.h>
48
49
50
51.macro LABEL prefix n
52\prefix\n\():
53.endm
54
55.macro JMPTBL_ENTRY i
56.word crc_\i - crc_array
57.endm
58
59.macro JNC_LESS_THAN j
60 jnc less_than_\j
61.endm
62
63
64
65
66
67#define SMALL_SIZE 200
68
69.if (SMALL_SIZE > 255)
70.error "SMALL_ SIZE must be < 256"
71.endif
72
73
74
75ENTRY(crc_pcl)
76#define bufp %rdi
77#define bufp_dw %edi
78#define bufp_w %di
79#define bufp_b %dil
80#define bufptmp %rcx
81#define block_0 %rcx
82#define block_1 %rdx
83#define block_2 %r11
84#define len %rsi
85#define len_dw %esi
86#define len_w %si
87#define len_b %sil
88#define crc_init_arg %rdx
89#define tmp %rbx
90#define crc_init %r8
91#define crc_init_dw %r8d
92#define crc1 %r9
93#define crc2 %r10
94
95 pushq %rbx
96 pushq %rdi
97 pushq %rsi
98
99
100 mov crc_init_arg, crc_init
101
102
103
104
105
106 mov bufp, bufptmp
107 neg bufp
108 and $7, bufp
109
110 je proc_block
111
112
113
114 cmp $8, len
115 jae do_align
116
117
118 shl $32-3+1, len_dw
119 jmp less_than_8_post_shl1
120
121do_align:
122
123 movq (bufptmp), tmp
124 add bufp, bufptmp
125
126 sub bufp, len
127align_loop:
128 crc32b %bl, crc_init_dw
129 shr $8, tmp
130 dec bufp
131 jne align_loop
132
133proc_block:
134
135
136
137
138
139
140 movq len, tmp
141
142 cmpq $128*24, len
143 jae full_block
144
145continue_block:
146 cmpq $SMALL_SIZE, len
147 jb small
148
149
150 movq $2731, %rax
151 mul len_dw
152 shrq $16, %rax
153
154
155
156
157
158
159
160
161
162 lea (bufptmp, %rax, 8), block_0
163 lea (block_0, %rax, 8), block_1
164 lea (block_1, %rax, 8), block_2
165
166 xor crc1, crc1
167 xor crc2, crc2
168
169
170 lea jump_table(%rip), bufp
171 movzxw (bufp, %rax, 2), len
172 offset=crc_array-jump_table
173 lea offset(bufp, len, 1), bufp
174 jmp *bufp
175
176
177
178
179full_block:
180 movq $128,%rax
181 lea 128*8*2(block_0), block_1
182 lea 128*8*3(block_0), block_2
183 add $128*8*1, block_0
184
185 xor crc1,crc1
186 xor crc2,crc2
187
188
189
190
191
192
193
194crc_array:
195 i=128
196.rept 128-1
197.altmacro
198LABEL crc_ %i
199.noaltmacro
200 crc32q -i*8(block_0), crc_init
201 crc32q -i*8(block_1), crc1
202 crc32q -i*8(block_2), crc2
203 i=(i-1)
204.endr
205
206.altmacro
207LABEL crc_ %i
208.noaltmacro
209 crc32q -i*8(block_0), crc_init
210 crc32q -i*8(block_1), crc1
211
212
213 mov block_2, block_0
214
215
216
217
218
219 lea (K_table-16)(%rip), bufp
220 shlq $3, %rax
221 subq %rax, tmp
222 shlq $1, %rax
223 subq %rax, tmp
224
225 addq %rax, bufp
226
227 movdqa (bufp), %xmm0
228
229 movq crc_init, %xmm1
230 PCLMULQDQ 0x00,%xmm0,%xmm1
231
232 movq crc1, %xmm2
233 PCLMULQDQ 0x10, %xmm0, %xmm2
234
235 pxor %xmm2,%xmm1
236 movq %xmm1, %rax
237 xor -i*8(block_2), %rax
238 mov crc2, crc_init
239 crc32 %rax, crc_init
240
241
242
243
244
245LABEL crc_ 0
246 mov tmp, len
247 cmp $128*24, tmp
248 jae full_block
249 cmp $24, tmp
250 jae continue_block
251
252less_than_24:
253 shl $32-4, len_dw
254
255 jnc less_than_16
256 crc32q (bufptmp), crc_init
257 crc32q 8(bufptmp), crc_init
258 jz do_return
259 add $16, bufptmp
260
261
262
263 shl $2, len_dw
264 jmp less_than_8_post_shl1
265
266
267
268
269small:
270 shl $32-8, len_dw
271 j=256
272.rept 5
273.altmacro
274LABEL less_than_ %j
275
276 j=(j/2)
277 shl $1, len_dw
278 JNC_LESS_THAN %j
279.noaltmacro
280 i=0
281.rept (j/8)
282 crc32q i(bufptmp), crc_init
283 i=i+8
284.endr
285 jz do_return
286 add $j, bufptmp
287.endr
288
289less_than_8:
290
291 shl $1, len_dw
292less_than_8_post_shl1:
293 jnc less_than_4
294 crc32l (bufptmp), crc_init_dw
295 jz do_return
296 add $4, bufptmp
297less_than_4:
298
299 shl $1, len_dw
300 jnc less_than_2
301 crc32w (bufptmp), crc_init_dw
302 jz do_return
303 add $2, bufptmp
304less_than_2:
305
306 shl $1, len_dw
307 jnc less_than_1
308 crc32b (bufptmp), crc_init_dw
309less_than_1:
310do_return:
311 movq crc_init, %rax
312 popq %rsi
313 popq %rdi
314 popq %rbx
315 ret
316
317
318
319
320.align 4
321jump_table:
322 i=0
323.rept 129
324.altmacro
325JMPTBL_ENTRY %i
326.noaltmacro
327 i=i+1
328.endr
329
330ENDPROC(crc_pcl)
331
332
333
334
335
336.data
337.align 64
338K_table:
339 .quad 0x14cd00bd6,0x105ec76f0
340 .quad 0x0ba4fc28e,0x14cd00bd6
341 .quad 0x1d82c63da,0x0f20c0dfe
342 .quad 0x09e4addf8,0x0ba4fc28e
343 .quad 0x039d3b296,0x1384aa63a
344 .quad 0x102f9b8a2,0x1d82c63da
345 .quad 0x14237f5e6,0x01c291d04
346 .quad 0x00d3b6092,0x09e4addf8
347 .quad 0x0c96cfdc0,0x0740eef02
348 .quad 0x18266e456,0x039d3b296
349 .quad 0x0daece73e,0x0083a6eec
350 .quad 0x0ab7aff2a,0x102f9b8a2
351 .quad 0x1248ea574,0x1c1733996
352 .quad 0x083348832,0x14237f5e6
353 .quad 0x12c743124,0x02ad91c30
354 .quad 0x0b9e02b86,0x00d3b6092
355 .quad 0x018b33a4e,0x06992cea2
356 .quad 0x1b331e26a,0x0c96cfdc0
357 .quad 0x17d35ba46,0x07e908048
358 .quad 0x1bf2e8b8a,0x18266e456
359 .quad 0x1a3e0968a,0x11ed1f9d8
360 .quad 0x0ce7f39f4,0x0daece73e
361 .quad 0x061d82e56,0x0f1d0f55e
362 .quad 0x0d270f1a2,0x0ab7aff2a
363 .quad 0x1c3f5f66c,0x0a87ab8a8
364 .quad 0x12ed0daac,0x1248ea574
365 .quad 0x065863b64,0x08462d800
366 .quad 0x11eef4f8e,0x083348832
367 .quad 0x1ee54f54c,0x071d111a8
368 .quad 0x0b3e32c28,0x12c743124
369 .quad 0x0064f7f26,0x0ffd852c6
370 .quad 0x0dd7e3b0c,0x0b9e02b86
371 .quad 0x0f285651c,0x0dcb17aa4
372 .quad 0x010746f3c,0x018b33a4e
373 .quad 0x1c24afea4,0x0f37c5aee
374 .quad 0x0271d9844,0x1b331e26a
375 .quad 0x08e766a0c,0x06051d5a2
376 .quad 0x093a5f730,0x17d35ba46
377 .quad 0x06cb08e5c,0x11d5ca20e
378 .quad 0x06b749fb2,0x1bf2e8b8a
379 .quad 0x1167f94f2,0x021f3d99c
380 .quad 0x0cec3662e,0x1a3e0968a
381 .quad 0x19329634a,0x08f158014
382 .quad 0x0e6fc4e6a,0x0ce7f39f4
383 .quad 0x08227bb8a,0x1a5e82106
384 .quad 0x0b0cd4768,0x061d82e56
385 .quad 0x13c2b89c4,0x188815ab2
386 .quad 0x0d7a4825c,0x0d270f1a2
387 .quad 0x10f5ff2ba,0x105405f3e
388 .quad 0x00167d312,0x1c3f5f66c
389 .quad 0x0f6076544,0x0e9adf796
390 .quad 0x026f6a60a,0x12ed0daac
391 .quad 0x1a2adb74e,0x096638b34
392 .quad 0x19d34af3a,0x065863b64
393 .quad 0x049c3cc9c,0x1e50585a0
394 .quad 0x068bce87a,0x11eef4f8e
395 .quad 0x1524fa6c6,0x19f1c69dc
396 .quad 0x16cba8aca,0x1ee54f54c
397 .quad 0x042d98888,0x12913343e
398 .quad 0x1329d9f7e,0x0b3e32c28
399 .quad 0x1b1c69528,0x088f25a3a
400 .quad 0x02178513a,0x0064f7f26
401 .quad 0x0e0ac139e,0x04e36f0b0
402 .quad 0x0170076fa,0x0dd7e3b0c
403 .quad 0x141a1a2e2,0x0bd6f81f8
404 .quad 0x16ad828b4,0x0f285651c
405 .quad 0x041d17b64,0x19425cbba
406 .quad 0x1fae1cc66,0x010746f3c
407 .quad 0x1a75b4b00,0x18db37e8a
408 .quad 0x0f872e54c,0x1c24afea4
409 .quad 0x01e41e9fc,0x04c144932
410 .quad 0x086d8e4d2,0x0271d9844
411 .quad 0x160f7af7a,0x052148f02
412 .quad 0x05bb8f1bc,0x08e766a0c
413 .quad 0x0a90fd27a,0x0a3c6f37a
414 .quad 0x0b3af077a,0x093a5f730
415 .quad 0x04984d782,0x1d22c238e
416 .quad 0x0ca6ef3ac,0x06cb08e5c
417 .quad 0x0234e0b26,0x063ded06a
418 .quad 0x1d88abd4a,0x06b749fb2
419 .quad 0x04597456a,0x04d56973c
420 .quad 0x0e9e28eb4,0x1167f94f2
421 .quad 0x07b3ff57a,0x19385bf2e
422 .quad 0x0c9c8b782,0x0cec3662e
423 .quad 0x13a9cba9e,0x0e417f38a
424 .quad 0x093e106a4,0x19329634a
425 .quad 0x167001a9c,0x14e727980
426 .quad 0x1ddffc5d4,0x0e6fc4e6a
427 .quad 0x00df04680,0x0d104b8fc
428 .quad 0x02342001e,0x08227bb8a
429 .quad 0x00a2a8d7e,0x05b397730
430 .quad 0x168763fa6,0x0b0cd4768
431 .quad 0x1ed5a407a,0x0e78eb416
432 .quad 0x0d2c3ed1a,0x13c2b89c4
433 .quad 0x0995a5724,0x1641378f0
434 .quad 0x19b1afbc4,0x0d7a4825c
435 .quad 0x109ffedc0,0x08d96551c
436 .quad 0x0f2271e60,0x10f5ff2ba
437 .quad 0x00b0bf8ca,0x00bf80dd2
438 .quad 0x123888b7a,0x00167d312
439 .quad 0x1e888f7dc,0x18dcddd1c
440 .quad 0x002ee03b2,0x0f6076544
441 .quad 0x183e8d8fe,0x06a45d2b2
442 .quad 0x133d7a042,0x026f6a60a
443 .quad 0x116b0f50c,0x1dd3e10e8
444 .quad 0x05fabe670,0x1a2adb74e
445 .quad 0x130004488,0x0de87806c
446 .quad 0x000bcf5f6,0x19d34af3a
447 .quad 0x18f0c7078,0x014338754
448 .quad 0x017f27698,0x049c3cc9c
449 .quad 0x058ca5f00,0x15e3e77ee
450 .quad 0x1af900c24,0x068bce87a
451 .quad 0x0b5cfca28,0x0dd07448e
452 .quad 0x0ded288f8,0x1524fa6c6
453 .quad 0x059f229bc,0x1d8048348
454 .quad 0x06d390dec,0x16cba8aca
455 .quad 0x037170390,0x0a3e3e02c
456 .quad 0x06353c1cc,0x042d98888
457 .quad 0x0c4584f5c,0x0d73c7bea
458 .quad 0x1f16a3418,0x1329d9f7e
459 .quad 0x0531377e2,0x185137662
460 .quad 0x1d8d9ca7c,0x1b1c69528
461 .quad 0x0b25b29f2,0x18a08b5bc
462 .quad 0x19fb2a8b0,0x02178513a
463 .quad 0x1a08fe6ac,0x1da758ae0
464 .quad 0x045cddf4e,0x0e0ac139e
465 .quad 0x1a91647f2,0x169cf9eb0
466 .quad 0x1a0f717c4,0x0170076fa
467