1
2
3
4
5#include <stddef.h>
6#include <stdint.h>
7
8#include <rte_cpuflags.h>
9#include <rte_common.h>
10#include <rte_net_crc.h>
11#include <rte_log.h>
12#include <rte_vect.h>
13
14#include "net_crc.h"
15
16
17#define CRC32_ETH_POLYNOMIAL 0x04c11db7UL
18#define CRC16_CCITT_POLYNOMIAL 0x1021U
19
20#define CRC_LUT_SIZE 256
21
22
23static uint32_t crc32_eth_lut[CRC_LUT_SIZE];
24static uint32_t crc16_ccitt_lut[CRC_LUT_SIZE];
25
26static uint32_t
27rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len);
28
29static uint32_t
30rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len);
31
32static uint32_t
33rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len);
34
35static uint32_t
36rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len);
37
38typedef uint32_t
39(*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len);
40
41static rte_net_crc_handler handlers_default[] = {
42 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_default_handler,
43 [RTE_NET_CRC32_ETH] = rte_crc32_eth_default_handler,
44};
45
46static const rte_net_crc_handler *handlers = handlers_default;
47
48static const rte_net_crc_handler handlers_scalar[] = {
49 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler,
50 [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler,
51};
52#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
53static const rte_net_crc_handler handlers_avx512[] = {
54 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler,
55 [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler,
56};
57#endif
58#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
59static const rte_net_crc_handler handlers_sse42[] = {
60 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler,
61 [RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler,
62};
63#endif
64#ifdef CC_ARM64_NEON_PMULL_SUPPORT
65static const rte_net_crc_handler handlers_neon[] = {
66 [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler,
67 [RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler,
68};
69#endif
70
71static uint16_t max_simd_bitwidth;
72
73#define NET_LOG(level, fmt, args...) \
74 rte_log(RTE_LOG_ ## level, libnet_logtype, "%s(): " fmt "\n", \
75 __func__, ## args)
76
77RTE_LOG_REGISTER_DEFAULT(libnet_logtype, INFO);
78
79
80
81
82
83
84
85
86
87
88
89
90static uint32_t
91reflect_32bits(uint32_t val)
92{
93 uint32_t i, res = 0;
94
95 for (i = 0; i < 32; i++)
96 if ((val & (1U << i)) != 0)
97 res |= (uint32_t)(1U << (31 - i));
98
99 return res;
100}
101
102static void
103crc32_eth_init_lut(uint32_t poly,
104 uint32_t *lut)
105{
106 uint32_t i, j;
107
108 for (i = 0; i < CRC_LUT_SIZE; i++) {
109 uint32_t crc = reflect_32bits(i);
110
111 for (j = 0; j < 8; j++) {
112 if (crc & 0x80000000L)
113 crc = (crc << 1) ^ poly;
114 else
115 crc <<= 1;
116 }
117 lut[i] = reflect_32bits(crc);
118 }
119}
120
121static __rte_always_inline uint32_t
122crc32_eth_calc_lut(const uint8_t *data,
123 uint32_t data_len,
124 uint32_t crc,
125 const uint32_t *lut)
126{
127 while (data_len--)
128 crc = lut[(crc ^ *data++) & 0xffL] ^ (crc >> 8);
129
130 return crc;
131}
132
133static void
134rte_net_crc_scalar_init(void)
135{
136
137 crc32_eth_init_lut(CRC32_ETH_POLYNOMIAL, crc32_eth_lut);
138
139
140 crc32_eth_init_lut(CRC16_CCITT_POLYNOMIAL << 16, crc16_ccitt_lut);
141}
142
143static inline uint32_t
144rte_crc16_ccitt_handler(const uint8_t *data, uint32_t data_len)
145{
146
147 return (uint16_t)~crc32_eth_calc_lut(data,
148 data_len,
149 0xffff,
150 crc16_ccitt_lut);
151}
152
153static inline uint32_t
154rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len)
155{
156
157 return ~crc32_eth_calc_lut(data,
158 data_len,
159 0xffffffffUL,
160 crc32_eth_lut);
161}
162
163
164
165#define AVX512_VPCLMULQDQ_CPU_SUPPORTED ( \
166 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && \
167 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && \
168 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && \
169 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && \
170 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && \
171 rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ) \
172)
173
174static const rte_net_crc_handler *
175avx512_vpclmulqdq_get_handlers(void)
176{
177#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
178 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED &&
179 max_simd_bitwidth >= RTE_VECT_SIMD_512)
180 return handlers_avx512;
181#endif
182 NET_LOG(INFO, "Requirements not met, can't use AVX512\n");
183 return NULL;
184}
185
186static void
187avx512_vpclmulqdq_init(void)
188{
189#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT
190 if (AVX512_VPCLMULQDQ_CPU_SUPPORTED)
191 rte_net_crc_avx512_init();
192#endif
193}
194
195
196
197#define SSE42_PCLMULQDQ_CPU_SUPPORTED \
198 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ)
199
200static const rte_net_crc_handler *
201sse42_pclmulqdq_get_handlers(void)
202{
203#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
204 if (SSE42_PCLMULQDQ_CPU_SUPPORTED &&
205 max_simd_bitwidth >= RTE_VECT_SIMD_128)
206 return handlers_sse42;
207#endif
208 NET_LOG(INFO, "Requirements not met, can't use SSE\n");
209 return NULL;
210}
211
212static void
213sse42_pclmulqdq_init(void)
214{
215#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
216 if (SSE42_PCLMULQDQ_CPU_SUPPORTED)
217 rte_net_crc_sse42_init();
218#endif
219}
220
221
222
223#define NEON_PMULL_CPU_SUPPORTED \
224 rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)
225
226static const rte_net_crc_handler *
227neon_pmull_get_handlers(void)
228{
229#ifdef CC_ARM64_NEON_PMULL_SUPPORT
230 if (NEON_PMULL_CPU_SUPPORTED &&
231 max_simd_bitwidth >= RTE_VECT_SIMD_128)
232 return handlers_neon;
233#endif
234 NET_LOG(INFO, "Requirements not met, can't use NEON\n");
235 return NULL;
236}
237
238static void
239neon_pmull_init(void)
240{
241#ifdef CC_ARM64_NEON_PMULL_SUPPORT
242 if (NEON_PMULL_CPU_SUPPORTED)
243 rte_net_crc_neon_init();
244#endif
245}
246
247
248
249static uint32_t
250rte_crc16_ccitt_default_handler(const uint8_t *data, uint32_t data_len)
251{
252 handlers = NULL;
253 if (max_simd_bitwidth == 0)
254 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
255
256 handlers = avx512_vpclmulqdq_get_handlers();
257 if (handlers != NULL)
258 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
259 handlers = sse42_pclmulqdq_get_handlers();
260 if (handlers != NULL)
261 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
262 handlers = neon_pmull_get_handlers();
263 if (handlers != NULL)
264 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
265 handlers = handlers_scalar;
266 return handlers[RTE_NET_CRC16_CCITT](data, data_len);
267}
268
269static uint32_t
270rte_crc32_eth_default_handler(const uint8_t *data, uint32_t data_len)
271{
272 handlers = NULL;
273 if (max_simd_bitwidth == 0)
274 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
275
276 handlers = avx512_vpclmulqdq_get_handlers();
277 if (handlers != NULL)
278 return handlers[RTE_NET_CRC32_ETH](data, data_len);
279 handlers = sse42_pclmulqdq_get_handlers();
280 if (handlers != NULL)
281 return handlers[RTE_NET_CRC32_ETH](data, data_len);
282 handlers = neon_pmull_get_handlers();
283 if (handlers != NULL)
284 return handlers[RTE_NET_CRC32_ETH](data, data_len);
285 handlers = handlers_scalar;
286 return handlers[RTE_NET_CRC32_ETH](data, data_len);
287}
288
289
290
291void
292rte_net_crc_set_alg(enum rte_net_crc_alg alg)
293{
294 handlers = NULL;
295 if (max_simd_bitwidth == 0)
296 max_simd_bitwidth = rte_vect_get_max_simd_bitwidth();
297
298 switch (alg) {
299 case RTE_NET_CRC_AVX512:
300 handlers = avx512_vpclmulqdq_get_handlers();
301 if (handlers != NULL)
302 break;
303
304 case RTE_NET_CRC_SSE42:
305 handlers = sse42_pclmulqdq_get_handlers();
306 break;
307 case RTE_NET_CRC_NEON:
308 handlers = neon_pmull_get_handlers();
309
310 case RTE_NET_CRC_SCALAR:
311
312 default:
313 break;
314 }
315
316 if (handlers == NULL)
317 handlers = handlers_scalar;
318}
319
320uint32_t
321rte_net_crc_calc(const void *data,
322 uint32_t data_len,
323 enum rte_net_crc_type type)
324{
325 uint32_t ret;
326 rte_net_crc_handler f_handle;
327
328 f_handle = handlers[type];
329 ret = f_handle(data, data_len);
330
331 return ret;
332}
333
334
335RTE_INIT(rte_net_crc_init)
336{
337 rte_net_crc_scalar_init();
338 sse42_pclmulqdq_init();
339 avx512_vpclmulqdq_init();
340 neon_pmull_init();
341}
342
