1
2
3
4#include <string.h>
5
6#include <rte_common.h>
7#include <rte_malloc.h>
8
9#include "rte_port_sym_crypto.h"
10
11
12
13
14#ifdef RTE_PORT_STATS_COLLECT
15
16#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \
17 (port)->stats.n_pkts_in += (val)
18#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \
19 (port)->stats.n_pkts_drop += (val)
20
21#else
22
23#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val)
24#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val)
25
26#endif
27
28struct rte_port_sym_crypto_reader {
29 struct rte_port_in_stats stats;
30
31 uint8_t cryptodev_id;
32 uint16_t queue_id;
33 struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX];
34 rte_port_sym_crypto_reader_callback_fn f_callback;
35 void *arg_callback;
36};
37
38static void *
39rte_port_sym_crypto_reader_create(void *params, int socket_id)
40{
41 struct rte_port_sym_crypto_reader_params *conf =
42 params;
43 struct rte_port_sym_crypto_reader *port;
44
45
46 if (conf == NULL) {
47 RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__);
48 return NULL;
49 }
50
51
52 port = rte_zmalloc_socket("PORT", sizeof(*port),
53 RTE_CACHE_LINE_SIZE, socket_id);
54 if (port == NULL) {
55 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
56 return NULL;
57 }
58
59
60 port->cryptodev_id = conf->cryptodev_id;
61 port->queue_id = conf->queue_id;
62 port->f_callback = conf->f_callback;
63 port->arg_callback = conf->arg_callback;
64
65 return port;
66}
67
68static int
69rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts)
70{
71 struct rte_port_sym_crypto_reader *p =
72 port;
73 uint16_t rx_ops_cnt, i, n = 0;
74
75 rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id,
76 p->ops, n_pkts);
77
78 for (i = 0; i < rx_ops_cnt; i++) {
79 struct rte_crypto_op *op = p->ops[i];
80
81
82 if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) {
83 rte_pktmbuf_free(op->sym->m_src);
84 continue;
85 }
86
87 pkts[n++] = op->sym->m_src;
88 }
89
90 if (p->f_callback)
91 (*p->f_callback)(pkts, n, p->arg_callback);
92
93 RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n);
94 RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_ops_cnt - n);
95
96 return n;
97}
98
99static int
100rte_port_sym_crypto_reader_free(void *port)
101{
102 if (port == NULL) {
103 RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
104 return -EINVAL;
105 }
106
107 rte_free(port);
108
109 return 0;
110}
111
112static int rte_port_sym_crypto_reader_stats_read(void *port,
113 struct rte_port_in_stats *stats, int clear)
114{
115 struct rte_port_sym_crypto_reader *p =
116 port;
117
118 if (stats != NULL)
119 memcpy(stats, &p->stats, sizeof(p->stats));
120
121 if (clear)
122 memset(&p->stats, 0, sizeof(p->stats));
123
124 return 0;
125}
126
127
128
129
130#ifdef RTE_PORT_STATS_COLLECT
131
132#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \
133 (port)->stats.n_pkts_in += (val)
134#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \
135 (port)->stats.n_pkts_drop += (val)
136
137#else
138
139#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val)
140#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val)
141
142#endif
143
144struct rte_port_sym_crypto_writer {
145 struct rte_port_out_stats stats;
146
147 struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
148
149 uint32_t tx_burst_sz;
150 uint32_t tx_buf_count;
151 uint64_t bsz_mask;
152
153 uint8_t cryptodev_id;
154 uint16_t queue_id;
155 uint16_t crypto_op_offset;
156};
157
158static void *
159rte_port_sym_crypto_writer_create(void *params, int socket_id)
160{
161 struct rte_port_sym_crypto_writer_params *conf =
162 params;
163 struct rte_port_sym_crypto_writer *port;
164
165
166 if ((conf == NULL) ||
167 (conf->tx_burst_sz == 0) ||
168 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
169 (!rte_is_power_of_2(conf->tx_burst_sz))) {
170 RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
171 return NULL;
172 }
173
174
175 port = rte_zmalloc_socket("PORT", sizeof(*port),
176 RTE_CACHE_LINE_SIZE, socket_id);
177 if (port == NULL) {
178 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
179 return NULL;
180 }
181
182
183 port->tx_burst_sz = conf->tx_burst_sz;
184 port->tx_buf_count = 0;
185 port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
186
187 port->cryptodev_id = conf->cryptodev_id;
188 port->queue_id = conf->queue_id;
189 port->crypto_op_offset = conf->crypto_op_offset;
190
191 return port;
192}
193
194static inline void
195send_burst(struct rte_port_sym_crypto_writer *p)
196{
197 uint32_t nb_tx;
198
199 nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id,
200 p->tx_buf, p->tx_buf_count);
201
202 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count -
203 nb_tx);
204 for (; nb_tx < p->tx_buf_count; nb_tx++)
205 rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src);
206
207 p->tx_buf_count = 0;
208}
209
210static int
211rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt)
212{
213 struct rte_port_sym_crypto_writer *p =
214 port;
215
216 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
217 RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset);
218 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
219 if (p->tx_buf_count >= p->tx_burst_sz)
220 send_burst(p);
221
222 return 0;
223}
224
225static int
226rte_port_sym_crypto_writer_tx_bulk(void *port,
227 struct rte_mbuf **pkts,
228 uint64_t pkts_mask)
229{
230 struct rte_port_sym_crypto_writer *p =
231 port;
232 uint64_t bsz_mask = p->bsz_mask;
233 uint32_t tx_buf_count = p->tx_buf_count;
234 uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
235 ((pkts_mask & bsz_mask) ^ bsz_mask);
236
237 if (expr == 0) {
238 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
239 uint32_t i;
240
241 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts);
242
243 for (i = 0; i < n_pkts; i++)
244 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
245 RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
246 p->crypto_op_offset);
247
248 if (p->tx_buf_count >= p->tx_burst_sz)
249 send_burst(p);
250 } else {
251 for (; pkts_mask;) {
252 uint32_t pkt_index = __builtin_ctzll(pkts_mask);
253 uint64_t pkt_mask = 1LLU << pkt_index;
254 struct rte_mbuf *pkt = pkts[pkt_index];
255
256 p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *)
257 RTE_MBUF_METADATA_UINT8_PTR(pkt,
258 p->crypto_op_offset);
259
260 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
261 pkts_mask &= ~pkt_mask;
262 }
263
264 p->tx_buf_count = tx_buf_count;
265 if (tx_buf_count >= p->tx_burst_sz)
266 send_burst(p);
267 }
268
269 return 0;
270}
271
272static int
273rte_port_sym_crypto_writer_flush(void *port)
274{
275 struct rte_port_sym_crypto_writer *p =
276 port;
277
278 if (p->tx_buf_count > 0)
279 send_burst(p);
280
281 return 0;
282}
283
284static int
285rte_port_sym_crypto_writer_free(void *port)
286{
287 if (port == NULL) {
288 RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
289 return -EINVAL;
290 }
291
292 rte_port_sym_crypto_writer_flush(port);
293 rte_free(port);
294
295 return 0;
296}
297
298static int rte_port_sym_crypto_writer_stats_read(void *port,
299 struct rte_port_out_stats *stats, int clear)
300{
301 struct rte_port_sym_crypto_writer *p =
302 port;
303
304 if (stats != NULL)
305 memcpy(stats, &p->stats, sizeof(p->stats));
306
307 if (clear)
308 memset(&p->stats, 0, sizeof(p->stats));
309
310 return 0;
311}
312
313
314
315
316#ifdef RTE_PORT_STATS_COLLECT
317
318#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \
319 (port)->stats.n_pkts_in += (val)
320#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \
321 (port)->stats.n_pkts_drop += (val)
322
323#else
324
325#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val)
326#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val)
327
328#endif
329
330struct rte_port_sym_crypto_writer_nodrop {
331 struct rte_port_out_stats stats;
332
333 struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
334 uint32_t tx_burst_sz;
335 uint32_t tx_buf_count;
336 uint64_t bsz_mask;
337 uint64_t n_retries;
338
339 uint8_t cryptodev_id;
340 uint16_t queue_id;
341 uint16_t crypto_op_offset;
342};
343
344static void *
345rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id)
346{
347 struct rte_port_sym_crypto_writer_nodrop_params *conf =
348 params;
349 struct rte_port_sym_crypto_writer_nodrop *port;
350
351
352 if ((conf == NULL) ||
353 (conf->tx_burst_sz == 0) ||
354 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
355 (!rte_is_power_of_2(conf->tx_burst_sz))) {
356 RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
357 return NULL;
358 }
359
360
361 port = rte_zmalloc_socket("PORT", sizeof(*port),
362 RTE_CACHE_LINE_SIZE, socket_id);
363 if (port == NULL) {
364 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
365 return NULL;
366 }
367
368
369 port->cryptodev_id = conf->cryptodev_id;
370 port->queue_id = conf->queue_id;
371 port->crypto_op_offset = conf->crypto_op_offset;
372 port->tx_burst_sz = conf->tx_burst_sz;
373 port->tx_buf_count = 0;
374 port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
375
376
377
378
379
380
381 port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries;
382
383 return port;
384}
385
386static inline void
387send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p)
388{
389 uint32_t nb_tx = 0, i;
390
391 nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id,
392 p->tx_buf, p->tx_buf_count);
393
394
395 if (nb_tx >= p->tx_buf_count) {
396 p->tx_buf_count = 0;
397 return;
398 }
399
400 for (i = 0; i < p->n_retries; i++) {
401 nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id,
402 p->queue_id, p->tx_buf + nb_tx,
403 p->tx_buf_count - nb_tx);
404
405
406 if (nb_tx >= p->tx_buf_count) {
407 p->tx_buf_count = 0;
408 return;
409 }
410 }
411
412
413 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p,
414 p->tx_buf_count - nb_tx);
415 for ( ; nb_tx < p->tx_buf_count; nb_tx++)
416 rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src);
417
418 p->tx_buf_count = 0;
419}
420
421static int
422rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt)
423{
424 struct rte_port_sym_crypto_writer_nodrop *p =
425 port;
426
427 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
428 RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset);
429 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
430 if (p->tx_buf_count >= p->tx_burst_sz)
431 send_burst_nodrop(p);
432
433 return 0;
434}
435
436static int
437rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port,
438 struct rte_mbuf **pkts,
439 uint64_t pkts_mask)
440{
441 struct rte_port_sym_crypto_writer_nodrop *p =
442 port;
443
444 uint64_t bsz_mask = p->bsz_mask;
445 uint32_t tx_buf_count = p->tx_buf_count;
446 uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
447 ((pkts_mask & bsz_mask) ^ bsz_mask);
448
449 if (expr == 0) {
450 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
451 uint32_t i;
452
453 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts);
454
455 for (i = 0; i < n_pkts; i++)
456 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
457 RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
458 p->crypto_op_offset);
459
460 if (p->tx_buf_count >= p->tx_burst_sz)
461 send_burst_nodrop(p);
462 } else {
463 for ( ; pkts_mask; ) {
464 uint32_t pkt_index = __builtin_ctzll(pkts_mask);
465 uint64_t pkt_mask = 1LLU << pkt_index;
466 struct rte_mbuf *pkt = pkts[pkt_index];
467
468 p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *)
469 RTE_MBUF_METADATA_UINT8_PTR(pkt,
470 p->crypto_op_offset);
471 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p,
472 1);
473 pkts_mask &= ~pkt_mask;
474 }
475
476 p->tx_buf_count = tx_buf_count;
477 if (tx_buf_count >= p->tx_burst_sz)
478 send_burst_nodrop(p);
479 }
480
481 return 0;
482}
483
484static int
485rte_port_sym_crypto_writer_nodrop_flush(void *port)
486{
487 struct rte_port_sym_crypto_writer_nodrop *p =
488 port;
489
490 if (p->tx_buf_count > 0)
491 send_burst_nodrop(p);
492
493 return 0;
494}
495
496static int
497rte_port_sym_crypto_writer_nodrop_free(void *port)
498{
499 if (port == NULL) {
500 RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
501 return -EINVAL;
502 }
503
504 rte_port_sym_crypto_writer_nodrop_flush(port);
505 rte_free(port);
506
507 return 0;
508}
509
510static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port,
511 struct rte_port_out_stats *stats, int clear)
512{
513 struct rte_port_sym_crypto_writer_nodrop *p =
514 port;
515
516 if (stats != NULL)
517 memcpy(stats, &p->stats, sizeof(p->stats));
518
519 if (clear)
520 memset(&p->stats, 0, sizeof(p->stats));
521
522 return 0;
523}
524
525
526
527
528
529struct rte_port_in_ops rte_port_sym_crypto_reader_ops = {
530 .f_create = rte_port_sym_crypto_reader_create,
531 .f_free = rte_port_sym_crypto_reader_free,
532 .f_rx = rte_port_sym_crypto_reader_rx,
533 .f_stats = rte_port_sym_crypto_reader_stats_read,
534};
535
536struct rte_port_out_ops rte_port_sym_crypto_writer_ops = {
537 .f_create = rte_port_sym_crypto_writer_create,
538 .f_free = rte_port_sym_crypto_writer_free,
539 .f_tx = rte_port_sym_crypto_writer_tx,
540 .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk,
541 .f_flush = rte_port_sym_crypto_writer_flush,
542 .f_stats = rte_port_sym_crypto_writer_stats_read,
543};
544
545struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops = {
546 .f_create = rte_port_sym_crypto_writer_nodrop_create,
547 .f_free = rte_port_sym_crypto_writer_nodrop_free,
548 .f_tx = rte_port_sym_crypto_writer_nodrop_tx,
549 .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk,
550 .f_flush = rte_port_sym_crypto_writer_nodrop_flush,
551 .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read,
552};
553
