1
2
3
4
5#include <stdbool.h>
6#include <stdio.h>
7#include <stdlib.h>
8#include <stdint.h>
9#include <inttypes.h>
10#include <sys/types.h>
11#include <netinet/in.h>
12#include <netinet/ip.h>
13#include <netinet/ip6.h>
14#include <string.h>
15#include <sys/queue.h>
16#include <stdarg.h>
17#include <errno.h>
18#include <signal.h>
19#include <getopt.h>
20
21#include <rte_common.h>
22#include <rte_bitmap.h>
23#include <rte_byteorder.h>
24#include <rte_log.h>
25#include <rte_eal.h>
26#include <rte_launch.h>
27#include <rte_cycles.h>
28#include <rte_prefetch.h>
29#include <rte_lcore.h>
30#include <rte_per_lcore.h>
31#include <rte_branch_prediction.h>
32#include <rte_interrupts.h>
33#include <rte_random.h>
34#include <rte_debug.h>
35#include <rte_ether.h>
36#include <rte_ethdev.h>
37#include <rte_mempool.h>
38#include <rte_mbuf.h>
39#include <rte_acl.h>
40#include <rte_lpm.h>
41#include <rte_lpm6.h>
42#include <rte_hash.h>
43#include <rte_jhash.h>
44#include <rte_cryptodev.h>
45#include <rte_security.h>
46#include <rte_eventdev.h>
47#include <rte_ip.h>
48#include <rte_ip_frag.h>
49#include <rte_alarm.h>
50#include <rte_telemetry.h>
51
52#include "event_helper.h"
53#include "flow.h"
54#include "ipsec.h"
55#include "ipsec_worker.h"
56#include "parser.h"
57#include "sad.h"
58
59#if defined(__ARM_NEON)
60#include "ipsec_lpm_neon.h"
61#endif
62
63volatile bool force_quit;
64
65#define MAX_JUMBO_PKT_LEN 9600
66
67#define MEMPOOL_CACHE_SIZE 256
68
69#define CDEV_MAP_ENTRIES 16384
70#define CDEV_MP_CACHE_SZ 64
71#define CDEV_MP_CACHE_MULTIPLIER 1.5
72#define MAX_QUEUE_PAIRS 1
73
74#define MAX_LCORE_PARAMS 1024
75
76
77
78
79#define IPSEC_SECGW_RX_DESC_DEFAULT 1024
80#define IPSEC_SECGW_TX_DESC_DEFAULT 1024
81static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT;
82static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT;
83
84
85
86
87static uint32_t qp_desc_nb = 2048;
88
89#define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \
90 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \
91 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \
92 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \
93 0, 0)
94
95#define FRAG_TBL_BUCKET_ENTRIES 4
96#define MAX_FRAG_TTL_NS (10LL * NS_PER_S)
97
98#define MTU_TO_FRAMELEN(x) ((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN)
99
100struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = {
101 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) },
102 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) },
103 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) },
104 { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) }
105};
106
107
108
109
110
111xmm_t val_eth[RTE_MAX_ETHPORTS];
112
113struct flow_info flow_info_tbl[RTE_MAX_ETHPORTS];
114
115#define CMD_LINE_OPT_CONFIG "config"
116#define CMD_LINE_OPT_SINGLE_SA "single-sa"
117#define CMD_LINE_OPT_CRYPTODEV_MASK "cryptodev_mask"
118#define CMD_LINE_OPT_TRANSFER_MODE "transfer-mode"
119#define CMD_LINE_OPT_SCHEDULE_TYPE "event-schedule-type"
120#define CMD_LINE_OPT_RX_OFFLOAD "rxoffload"
121#define CMD_LINE_OPT_TX_OFFLOAD "txoffload"
122#define CMD_LINE_OPT_REASSEMBLE "reassemble"
123#define CMD_LINE_OPT_MTU "mtu"
124#define CMD_LINE_OPT_FRAG_TTL "frag-ttl"
125#define CMD_LINE_OPT_EVENT_VECTOR "event-vector"
126#define CMD_LINE_OPT_VECTOR_SIZE "vector-size"
127#define CMD_LINE_OPT_VECTOR_TIMEOUT "vector-tmo"
128#define CMD_LINE_OPT_VECTOR_POOL_SZ "vector-pool-sz"
129#define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
130#define CMD_LINE_OPT_QP_DESC_NB "desc-nb"
131
132#define CMD_LINE_ARG_EVENT "event"
133#define CMD_LINE_ARG_POLL "poll"
134#define CMD_LINE_ARG_ORDERED "ordered"
135#define CMD_LINE_ARG_ATOMIC "atomic"
136#define CMD_LINE_ARG_PARALLEL "parallel"
137
138enum {
139
140
141
142
143
144 CMD_LINE_OPT_MIN_NUM = 256,
145 CMD_LINE_OPT_CONFIG_NUM,
146 CMD_LINE_OPT_SINGLE_SA_NUM,
147 CMD_LINE_OPT_CRYPTODEV_MASK_NUM,
148 CMD_LINE_OPT_TRANSFER_MODE_NUM,
149 CMD_LINE_OPT_SCHEDULE_TYPE_NUM,
150 CMD_LINE_OPT_RX_OFFLOAD_NUM,
151 CMD_LINE_OPT_TX_OFFLOAD_NUM,
152 CMD_LINE_OPT_REASSEMBLE_NUM,
153 CMD_LINE_OPT_MTU_NUM,
154 CMD_LINE_OPT_FRAG_TTL_NUM,
155 CMD_LINE_OPT_EVENT_VECTOR_NUM,
156 CMD_LINE_OPT_VECTOR_SIZE_NUM,
157 CMD_LINE_OPT_VECTOR_TIMEOUT_NUM,
158 CMD_LINE_OPT_VECTOR_POOL_SZ_NUM,
159 CMD_LINE_OPT_PER_PORT_POOL_NUM,
160 CMD_LINE_OPT_QP_DESC_NB_NUM,
161};
162
163static const struct option lgopts[] = {
164 {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM},
165 {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM},
166 {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM},
167 {CMD_LINE_OPT_TRANSFER_MODE, 1, 0, CMD_LINE_OPT_TRANSFER_MODE_NUM},
168 {CMD_LINE_OPT_SCHEDULE_TYPE, 1, 0, CMD_LINE_OPT_SCHEDULE_TYPE_NUM},
169 {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM},
170 {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM},
171 {CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM},
172 {CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM},
173 {CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM},
174 {CMD_LINE_OPT_EVENT_VECTOR, 0, 0, CMD_LINE_OPT_EVENT_VECTOR_NUM},
175 {CMD_LINE_OPT_VECTOR_SIZE, 1, 0, CMD_LINE_OPT_VECTOR_SIZE_NUM},
176 {CMD_LINE_OPT_VECTOR_TIMEOUT, 1, 0, CMD_LINE_OPT_VECTOR_TIMEOUT_NUM},
177 {CMD_LINE_OPT_VECTOR_POOL_SZ, 1, 0, CMD_LINE_OPT_VECTOR_POOL_SZ_NUM},
178 {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PER_PORT_POOL_NUM},
179 {CMD_LINE_OPT_QP_DESC_NB, 1, 0, CMD_LINE_OPT_QP_DESC_NB_NUM},
180 {NULL, 0, 0, 0}
181};
182
183uint32_t unprotected_port_mask;
184uint32_t single_sa_idx;
185
186static uint32_t enabled_port_mask;
187static uint64_t enabled_cryptodev_mask = UINT64_MAX;
188static int32_t promiscuous_on;
189static int32_t numa_on = 1;
190static uint32_t nb_lcores;
191uint32_t single_sa;
192uint32_t nb_bufs_in_pool;
193
194
195
196
197
198static uint64_t dev_rx_offload = UINT64_MAX;
199static uint64_t dev_tx_offload = UINT64_MAX;
200
201
202
203
204uint32_t frag_tbl_sz;
205static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
206uint32_t mtu_size = RTE_ETHER_MTU;
207static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS;
208static uint32_t stats_interval;
209
210
211struct app_sa_prm app_sa_prm = {
212 .enable = 0,
213 .cache_sz = SA_CACHE_SZ,
214 .udp_encap = 0
215 };
216static const char *cfgfile;
217
218struct lcore_params {
219 uint16_t port_id;
220 uint8_t queue_id;
221 uint8_t lcore_id;
222} __rte_cache_aligned;
223
224static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
225
226static struct lcore_params *lcore_params;
227static uint16_t nb_lcore_params;
228
229static struct rte_hash *cdev_map_in;
230static struct rte_hash *cdev_map_out;
231
232struct lcore_conf lcore_conf[RTE_MAX_LCORE];
233
234static struct rte_eth_conf port_conf = {
235 .rxmode = {
236 .mq_mode = RTE_ETH_MQ_RX_RSS,
237 .split_hdr_size = 0,
238 .offloads = RTE_ETH_RX_OFFLOAD_CHECKSUM,
239 },
240 .rx_adv_conf = {
241 .rss_conf = {
242 .rss_key = NULL,
243 .rss_hf = RTE_ETH_RSS_IP | RTE_ETH_RSS_UDP |
244 RTE_ETH_RSS_TCP | RTE_ETH_RSS_SCTP,
245 },
246 },
247 .txmode = {
248 .mq_mode = RTE_ETH_MQ_TX_NONE,
249 },
250};
251
252struct socket_ctx socket_ctx[NB_SOCKETS];
253
254bool per_port_pool;
255
256uint16_t wrkr_flags;
257
258
259
260
261
262static int
263multi_seg_required(void)
264{
265 return (MTU_TO_FRAMELEN(mtu_size) + RTE_PKTMBUF_HEADROOM >
266 frame_buf_size || frag_tbl_sz != 0);
267}
268
269
270struct ipsec_core_statistics core_statistics[RTE_MAX_LCORE];
271
272
273static void
274print_stats_cb(__rte_unused void *param)
275{
276 uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
277 float burst_percent, rx_per_call, tx_per_call;
278 unsigned int coreid;
279
280 total_packets_dropped = 0;
281 total_packets_tx = 0;
282 total_packets_rx = 0;
283
284 const char clr[] = { 27, '[', '2', 'J', '\0' };
285 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
286
287
288 printf("%s%s", clr, topLeft);
289
290 printf("\nCore statistics ====================================");
291
292 for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
293
294 if (rte_lcore_is_enabled(coreid) == 0)
295 continue;
296 burst_percent = (float)(core_statistics[coreid].burst_rx * 100)/
297 core_statistics[coreid].rx;
298 rx_per_call = (float)(core_statistics[coreid].rx)/
299 core_statistics[coreid].rx_call;
300 tx_per_call = (float)(core_statistics[coreid].tx)/
301 core_statistics[coreid].tx_call;
302 printf("\nStatistics for core %u ------------------------------"
303 "\nPackets received: %20"PRIu64
304 "\nPackets sent: %24"PRIu64
305 "\nPackets dropped: %21"PRIu64
306 "\nBurst percent: %23.2f"
307 "\nPackets per Rx call: %17.2f"
308 "\nPackets per Tx call: %17.2f",
309 coreid,
310 core_statistics[coreid].rx,
311 core_statistics[coreid].tx,
312 core_statistics[coreid].dropped,
313 burst_percent,
314 rx_per_call,
315 tx_per_call);
316
317 total_packets_dropped += core_statistics[coreid].dropped;
318 total_packets_tx += core_statistics[coreid].tx;
319 total_packets_rx += core_statistics[coreid].rx;
320 }
321 printf("\nAggregate statistics ==============================="
322 "\nTotal packets received: %14"PRIu64
323 "\nTotal packets sent: %18"PRIu64
324 "\nTotal packets dropped: %15"PRIu64,
325 total_packets_rx,
326 total_packets_tx,
327 total_packets_dropped);
328 printf("\n====================================================\n");
329
330 rte_eal_alarm_set(stats_interval * US_PER_S, print_stats_cb, NULL);
331}
332
333static void
334split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num)
335{
336 uint32_t i, n4, n6;
337 struct ip *ip;
338 struct rte_mbuf *m;
339
340 n4 = trf->ip4.num;
341 n6 = trf->ip6.num;
342
343 for (i = 0; i < num; i++) {
344
345 m = mb[i];
346 ip = rte_pktmbuf_mtod(m, struct ip *);
347
348 if (ip->ip_v == IPVERSION) {
349 trf->ip4.pkts[n4] = m;
350 trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m,
351 uint8_t *, offsetof(struct ip, ip_p));
352 n4++;
353 } else if (ip->ip_v == IP6_VERSION) {
354 trf->ip6.pkts[n6] = m;
355 trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m,
356 uint8_t *,
357 offsetof(struct ip6_hdr, ip6_nxt));
358 n6++;
359 } else
360 free_pkts(&m, 1);
361 }
362
363 trf->ip4.num = n4;
364 trf->ip6.num = n6;
365}
366
367
368static inline void
369process_pkts_inbound(struct ipsec_ctx *ipsec_ctx,
370 struct ipsec_traffic *traffic)
371{
372 unsigned int lcoreid = rte_lcore_id();
373 uint16_t nb_pkts_in, n_ip4, n_ip6;
374
375 n_ip4 = traffic->ip4.num;
376 n_ip6 = traffic->ip6.num;
377
378 if (app_sa_prm.enable == 0) {
379 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts,
380 traffic->ipsec.num, MAX_PKT_BURST);
381 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in);
382 } else {
383 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts,
384 traffic->ipsec.saptr, traffic->ipsec.num);
385 ipsec_process(ipsec_ctx, traffic);
386 }
387
388 inbound_sp_sa(ipsec_ctx->sp4_ctx,
389 ipsec_ctx->sa_ctx, &traffic->ip4, n_ip4,
390 &core_statistics[lcoreid].inbound.spd4);
391
392 inbound_sp_sa(ipsec_ctx->sp6_ctx,
393 ipsec_ctx->sa_ctx, &traffic->ip6, n_ip6,
394 &core_statistics[lcoreid].inbound.spd6);
395}
396
397static inline void
398outbound_spd_lookup(struct sp_ctx *sp,
399 struct traffic_type *ip,
400 struct traffic_type *ipsec,
401 struct ipsec_spd_stats *stats)
402{
403 struct rte_mbuf *m;
404 uint32_t i, j, sa_idx;
405
406 if (ip->num == 0 || sp == NULL)
407 return;
408
409 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res,
410 ip->num, DEFAULT_MAX_CATEGORIES);
411
412 for (i = 0, j = 0; i < ip->num; i++) {
413 m = ip->pkts[i];
414 sa_idx = ip->res[i] - 1;
415
416 if (unlikely(ip->res[i] == DISCARD)) {
417 free_pkts(&m, 1);
418
419 stats->discard++;
420 } else if (unlikely(ip->res[i] == BYPASS)) {
421 ip->pkts[j++] = m;
422
423 stats->bypass++;
424 } else {
425 ipsec->res[ipsec->num] = sa_idx;
426 ipsec->pkts[ipsec->num++] = m;
427
428 stats->protect++;
429 }
430 }
431 ip->num = j;
432}
433
434static inline void
435process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
436 struct ipsec_traffic *traffic)
437{
438 struct rte_mbuf *m;
439 uint16_t idx, nb_pkts_out, i;
440 unsigned int lcoreid = rte_lcore_id();
441
442
443 free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
444
445 traffic->ipsec.num = 0;
446
447 outbound_spd_lookup(ipsec_ctx->sp4_ctx,
448 &traffic->ip4, &traffic->ipsec,
449 &core_statistics[lcoreid].outbound.spd4);
450
451 outbound_spd_lookup(ipsec_ctx->sp6_ctx,
452 &traffic->ip6, &traffic->ipsec,
453 &core_statistics[lcoreid].outbound.spd6);
454
455 if (app_sa_prm.enable == 0) {
456
457 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts,
458 traffic->ipsec.res, traffic->ipsec.num,
459 MAX_PKT_BURST);
460
461 for (i = 0; i < nb_pkts_out; i++) {
462 m = traffic->ipsec.pkts[i];
463 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *);
464 if (ip->ip_v == IPVERSION) {
465 idx = traffic->ip4.num++;
466 traffic->ip4.pkts[idx] = m;
467 } else {
468 idx = traffic->ip6.num++;
469 traffic->ip6.pkts[idx] = m;
470 }
471 }
472 } else {
473 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res,
474 traffic->ipsec.saptr, traffic->ipsec.num);
475 ipsec_process(ipsec_ctx, traffic);
476 }
477}
478
479static inline void
480process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
481 struct ipsec_traffic *traffic)
482{
483 struct rte_mbuf *m;
484 uint32_t nb_pkts_in, i, idx;
485
486 if (app_sa_prm.enable == 0) {
487
488 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts,
489 traffic->ipsec.num, MAX_PKT_BURST);
490
491 for (i = 0; i < nb_pkts_in; i++) {
492 m = traffic->ipsec.pkts[i];
493 struct ip *ip = rte_pktmbuf_mtod(m, struct ip *);
494 if (ip->ip_v == IPVERSION) {
495 idx = traffic->ip4.num++;
496 traffic->ip4.pkts[idx] = m;
497 } else {
498 idx = traffic->ip6.num++;
499 traffic->ip6.pkts[idx] = m;
500 }
501 }
502 } else {
503 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts,
504 traffic->ipsec.saptr, traffic->ipsec.num);
505 ipsec_process(ipsec_ctx, traffic);
506 }
507}
508
509static inline void
510process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,
511 struct ipsec_traffic *traffic)
512{
513 struct rte_mbuf *m;
514 uint32_t nb_pkts_out, i, n;
515 struct ip *ip;
516
517
518 free_pkts(traffic->ipsec.pkts, traffic->ipsec.num);
519
520 n = 0;
521
522 for (i = 0; i < traffic->ip4.num; i++) {
523 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i];
524 traffic->ipsec.res[n++] = single_sa_idx;
525 }
526
527 for (i = 0; i < traffic->ip6.num; i++) {
528 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i];
529 traffic->ipsec.res[n++] = single_sa_idx;
530 }
531
532 traffic->ip4.num = 0;
533 traffic->ip6.num = 0;
534 traffic->ipsec.num = n;
535
536 if (app_sa_prm.enable == 0) {
537
538 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts,
539 traffic->ipsec.res, traffic->ipsec.num,
540 MAX_PKT_BURST);
541
542
543 m = traffic->ipsec.pkts[0];
544 ip = rte_pktmbuf_mtod(m, struct ip *);
545 if (ip->ip_v == IPVERSION) {
546 traffic->ip4.num = nb_pkts_out;
547 for (i = 0; i < nb_pkts_out; i++)
548 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i];
549 } else {
550 traffic->ip6.num = nb_pkts_out;
551 for (i = 0; i < nb_pkts_out; i++)
552 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i];
553 }
554 } else {
555 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res,
556 traffic->ipsec.saptr, traffic->ipsec.num);
557 ipsec_process(ipsec_ctx, traffic);
558 }
559}
560
561static inline void
562process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts,
563 uint8_t nb_pkts, uint16_t portid, struct rte_security_ctx *ctx)
564{
565 struct ipsec_traffic traffic;
566
567 prepare_traffic(ctx, pkts, &traffic, nb_pkts);
568
569 if (unlikely(single_sa)) {
570 if (is_unprotected_port(portid))
571 process_pkts_inbound_nosp(&qconf->inbound, &traffic);
572 else
573 process_pkts_outbound_nosp(&qconf->outbound, &traffic);
574 } else {
575 if (is_unprotected_port(portid))
576 process_pkts_inbound(&qconf->inbound, &traffic);
577 else
578 process_pkts_outbound(&qconf->outbound, &traffic);
579 }
580
581#if defined __ARM_NEON
582
583 route4_pkts_neon(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num,
584 qconf->outbound.ipv4_offloads, true);
585 route6_pkts_neon(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num);
586#else
587 route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num,
588 qconf->outbound.ipv4_offloads, true);
589 route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num);
590#endif
591}
592
593static inline void
594drain_crypto_buffers(struct lcore_conf *qconf)
595{
596 uint32_t i;
597 struct ipsec_ctx *ctx;
598
599
600 ctx = &qconf->inbound;
601 for (i = 0; i != ctx->nb_qps; i++) {
602 if (ctx->tbl[i].len != 0)
603 enqueue_cop_burst(ctx->tbl + i);
604 }
605
606
607 ctx = &qconf->outbound;
608 for (i = 0; i != ctx->nb_qps; i++) {
609 if (ctx->tbl[i].len != 0)
610 enqueue_cop_burst(ctx->tbl + i);
611 }
612}
613
614static void
615drain_inbound_crypto_queues(const struct lcore_conf *qconf,
616 struct ipsec_ctx *ctx)
617{
618 uint32_t n;
619 struct ipsec_traffic trf;
620 unsigned int lcoreid = rte_lcore_id();
621
622 if (app_sa_prm.enable == 0) {
623
624
625 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts,
626 RTE_DIM(trf.ipsec.pkts));
627
628 trf.ip4.num = 0;
629 trf.ip6.num = 0;
630
631
632 split46_traffic(&trf, trf.ipsec.pkts, n);
633 } else
634 ipsec_cqp_process(ctx, &trf);
635
636
637 if (trf.ip4.num != 0) {
638 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0,
639 &core_statistics[lcoreid].inbound.spd4);
640 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num,
641 qconf->outbound.ipv4_offloads, true);
642 }
643
644
645 if (trf.ip6.num != 0) {
646 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0,
647 &core_statistics[lcoreid].inbound.spd6);
648 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num);
649 }
650}
651
652static void
653drain_outbound_crypto_queues(const struct lcore_conf *qconf,
654 struct ipsec_ctx *ctx)
655{
656 uint32_t n;
657 struct ipsec_traffic trf;
658
659 if (app_sa_prm.enable == 0) {
660
661
662 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts,
663 RTE_DIM(trf.ipsec.pkts));
664
665 trf.ip4.num = 0;
666 trf.ip6.num = 0;
667
668
669 split46_traffic(&trf, trf.ipsec.pkts, n);
670 } else
671 ipsec_cqp_process(ctx, &trf);
672
673
674 if (trf.ip4.num != 0)
675 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num,
676 qconf->outbound.ipv4_offloads, true);
677
678
679 if (trf.ip6.num != 0)
680 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num);
681}
682
683
684void
685ipsec_poll_mode_worker(void)
686{
687 struct rte_mbuf *pkts[MAX_PKT_BURST];
688 uint32_t lcore_id;
689 uint64_t prev_tsc, diff_tsc, cur_tsc;
690 int32_t i, nb_rx;
691 uint16_t portid;
692 uint8_t queueid;
693 struct lcore_conf *qconf;
694 int32_t rc, socket_id;
695 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
696 / US_PER_S * BURST_TX_DRAIN_US;
697 struct lcore_rx_queue *rxql;
698
699 prev_tsc = 0;
700 lcore_id = rte_lcore_id();
701 qconf = &lcore_conf[lcore_id];
702 rxql = qconf->rx_queue_list;
703 socket_id = rte_lcore_to_socket_id(lcore_id);
704
705 qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4;
706 qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6;
707 qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in;
708 qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in;
709 qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in;
710 qconf->inbound.cdev_map = cdev_map_in;
711 qconf->inbound.lcore_id = lcore_id;
712 qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out;
713 qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out;
714 qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out;
715 qconf->outbound.cdev_map = cdev_map_out;
716 qconf->outbound.lcore_id = lcore_id;
717 qconf->frag.pool_indir = socket_ctx[socket_id].mbuf_pool_indir;
718
719 rc = ipsec_sad_lcore_cache_init(app_sa_prm.cache_sz);
720 if (rc != 0) {
721 RTE_LOG(ERR, IPSEC,
722 "SAD cache init on lcore %u, failed with code: %d\n",
723 lcore_id, rc);
724 return;
725 }
726
727 if (qconf->nb_rx_queue == 0) {
728 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n",
729 lcore_id);
730 return;
731 }
732
733 RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id);
734
735 for (i = 0; i < qconf->nb_rx_queue; i++) {
736 portid = rxql[i].port_id;
737 queueid = rxql[i].queue_id;
738 RTE_LOG(INFO, IPSEC,
739 " -- lcoreid=%u portid=%u rxqueueid=%hhu\n",
740 lcore_id, portid, queueid);
741 }
742
743 while (!force_quit) {
744 cur_tsc = rte_rdtsc();
745
746
747 diff_tsc = cur_tsc - prev_tsc;
748
749 if (unlikely(diff_tsc > drain_tsc)) {
750 drain_tx_buffers(qconf);
751 drain_crypto_buffers(qconf);
752 prev_tsc = cur_tsc;
753 }
754
755 for (i = 0; i < qconf->nb_rx_queue; ++i) {
756
757
758 portid = rxql[i].port_id;
759 queueid = rxql[i].queue_id;
760 nb_rx = rte_eth_rx_burst(portid, queueid,
761 pkts, MAX_PKT_BURST);
762
763 if (nb_rx > 0) {
764 core_stats_update_rx(nb_rx);
765 process_pkts(qconf, pkts, nb_rx, portid,
766 rxql->sec_ctx);
767 }
768
769
770 if (is_unprotected_port(portid))
771 drain_inbound_crypto_queues(qconf,
772 &qconf->inbound);
773 else
774 drain_outbound_crypto_queues(qconf,
775 &qconf->outbound);
776 }
777 }
778}
779
780int
781check_flow_params(uint16_t fdir_portid, uint8_t fdir_qid)
782{
783 uint16_t i;
784 uint16_t portid;
785 uint8_t queueid;
786
787 for (i = 0; i < nb_lcore_params; ++i) {
788 portid = lcore_params_array[i].port_id;
789 if (portid == fdir_portid) {
790 queueid = lcore_params_array[i].queue_id;
791 if (queueid == fdir_qid)
792 break;
793 }
794
795 if (i == nb_lcore_params - 1)
796 return -1;
797 }
798
799 return 1;
800}
801
802static int32_t
803check_poll_mode_params(struct eh_conf *eh_conf)
804{
805 uint8_t lcore;
806 uint16_t portid;
807 uint16_t i;
808 int32_t socket_id;
809
810 if (!eh_conf)
811 return -EINVAL;
812
813 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_POLL)
814 return 0;
815
816 if (lcore_params == NULL) {
817 printf("Error: No port/queue/core mappings\n");
818 return -1;
819 }
820
821 for (i = 0; i < nb_lcore_params; ++i) {
822 lcore = lcore_params[i].lcore_id;
823 if (!rte_lcore_is_enabled(lcore)) {
824 printf("error: lcore %hhu is not enabled in "
825 "lcore mask\n", lcore);
826 return -1;
827 }
828 socket_id = rte_lcore_to_socket_id(lcore);
829 if (socket_id != 0 && numa_on == 0) {
830 printf("warning: lcore %hhu is on socket %d "
831 "with numa off\n",
832 lcore, socket_id);
833 }
834 portid = lcore_params[i].port_id;
835 if ((enabled_port_mask & (1 << portid)) == 0) {
836 printf("port %u is not enabled in port mask\n", portid);
837 return -1;
838 }
839 if (!rte_eth_dev_is_valid_port(portid)) {
840 printf("port %u is not present on the board\n", portid);
841 return -1;
842 }
843 }
844 return 0;
845}
846
847static uint8_t
848get_port_nb_rx_queues(const uint16_t port)
849{
850 int32_t queue = -1;
851 uint16_t i;
852
853 for (i = 0; i < nb_lcore_params; ++i) {
854 if (lcore_params[i].port_id == port &&
855 lcore_params[i].queue_id > queue)
856 queue = lcore_params[i].queue_id;
857 }
858 return (uint8_t)(++queue);
859}
860
861static int32_t
862init_lcore_rx_queues(void)
863{
864 uint16_t i, nb_rx_queue;
865 uint8_t lcore;
866
867 for (i = 0; i < nb_lcore_params; ++i) {
868 lcore = lcore_params[i].lcore_id;
869 nb_rx_queue = lcore_conf[lcore].nb_rx_queue;
870 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
871 printf("error: too many queues (%u) for lcore: %u\n",
872 nb_rx_queue + 1, lcore);
873 return -1;
874 }
875 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
876 lcore_params[i].port_id;
877 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
878 lcore_params[i].queue_id;
879 lcore_conf[lcore].nb_rx_queue++;
880 }
881 return 0;
882}
883
884
885static void
886print_usage(const char *prgname)
887{
888 fprintf(stderr, "%s [EAL options] --"
889 " -p PORTMASK"
890 " [-P]"
891 " [-u PORTMASK]"
892 " [-j FRAMESIZE]"
893 " [-l]"
894 " [-w REPLAY_WINDOW_SIZE]"
895 " [-e]"
896 " [-a]"
897 " [-c]"
898 " [-t STATS_INTERVAL]"
899 " [-s NUMBER_OF_MBUFS_IN_PKT_POOL]"
900 " -f CONFIG_FILE"
901 " --config (port,queue,lcore)[,(port,queue,lcore)]"
902 " [--single-sa SAIDX]"
903 " [--cryptodev_mask MASK]"
904 " [--transfer-mode MODE]"
905 " [--event-schedule-type TYPE]"
906 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]"
907 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]"
908 " [--" CMD_LINE_OPT_REASSEMBLE " REASSEMBLE_TABLE_SIZE]"
909 " [--" CMD_LINE_OPT_MTU " MTU]"
910 " [--event-vector]"
911 " [--vector-size SIZE]"
912 " [--vector-tmo TIMEOUT in ns]"
913 " [--" CMD_LINE_OPT_QP_DESC_NB " NUMBER_OF_DESC]"
914 "\n\n"
915 " -p PORTMASK: Hexadecimal bitmask of ports to configure\n"
916 " -P : Enable promiscuous mode\n"
917 " -u PORTMASK: Hexadecimal bitmask of unprotected ports\n"
918 " -j FRAMESIZE: Data buffer size, minimum (and default)\n"
919 " value: RTE_MBUF_DEFAULT_BUF_SIZE\n"
920 " -l enables code-path that uses librte_ipsec\n"
921 " -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n"
922 " size for each SA\n"
923 " -e enables ESN\n"
924 " -a enables SA SQN atomic behaviour\n"
925 " -c specifies inbound SAD cache size,\n"
926 " zero value disables the cache (default value: 128)\n"
927 " -t specifies statistics screen update interval,\n"
928 " zero disables statistics screen (default value: 0)\n"
929 " -s number of mbufs in packet pool, if not specified number\n"
930 " of mbufs will be calculated based on number of cores,\n"
931 " ports and crypto queues\n"
932 " -f CONFIG_FILE: Configuration file\n"
933 " --config (port,queue,lcore): Rx queue configuration. In poll\n"
934 " mode determines which queues from\n"
935 " which ports are mapped to which cores.\n"
936 " In event mode this option is not used\n"
937 " as packets are dynamically scheduled\n"
938 " to cores by HW.\n"
939 " --single-sa SAIDX: In poll mode use single SA index for\n"
940 " outbound traffic, bypassing the SP\n"
941 " In event mode selects driver submode,\n"
942 " SA index value is ignored\n"
943 " --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n"
944 " devices to configure\n"
945 " --transfer-mode MODE\n"
946 " \"poll\" : Packet transfer via polling (default)\n"
947 " \"event\" : Packet transfer via event device\n"
948 " --event-schedule-type TYPE queue schedule type, used only when\n"
949 " transfer mode is set to event\n"
950 " \"ordered\" : Ordered (default)\n"
951 " \"atomic\" : Atomic\n"
952 " \"parallel\" : Parallel\n"
953 " --" CMD_LINE_OPT_RX_OFFLOAD
954 ": bitmask of the RX HW offload capabilities to enable/use\n"
955 " (RTE_ETH_RX_OFFLOAD_*)\n"
956 " --" CMD_LINE_OPT_TX_OFFLOAD
957 ": bitmask of the TX HW offload capabilities to enable/use\n"
958 " (RTE_ETH_TX_OFFLOAD_*)\n"
959 " --" CMD_LINE_OPT_REASSEMBLE " NUM"
960 ": max number of entries in reassemble(fragment) table\n"
961 " (zero (default value) disables reassembly)\n"
962 " --" CMD_LINE_OPT_MTU " MTU"
963 ": MTU value on all ports (default value: 1500)\n"
964 " outgoing packets with bigger size will be fragmented\n"
965 " incoming packets with bigger size will be discarded\n"
966 " --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS"
967 ": fragments lifetime in nanoseconds, default\n"
968 " and maximum value is 10.000.000.000 ns (10 s)\n"
969 " --event-vector enables event vectorization\n"
970 " --vector-size Max vector size (default value: 16)\n"
971 " --vector-tmo Max vector timeout in nanoseconds"
972 " (default value: 102400)\n"
973 " --" CMD_LINE_OPT_PER_PORT_POOL " Enable per port mbuf pool\n"
974 " --" CMD_LINE_OPT_VECTOR_POOL_SZ " Vector pool size\n"
975 " (default value is based on mbuf count)\n"
976 " --" CMD_LINE_OPT_QP_DESC_NB " DESC_NB"
977 ": Number of descriptors per queue pair (default value: 2048)\n"
978 "\n",
979 prgname);
980}
981
982static int
983parse_mask(const char *str, uint64_t *val)
984{
985 char *end;
986 unsigned long t;
987
988 errno = 0;
989 t = strtoul(str, &end, 0);
990 if (errno != 0 || end[0] != 0)
991 return -EINVAL;
992
993 *val = t;
994 return 0;
995}
996
997static int32_t
998parse_portmask(const char *portmask)
999{
1000 char *end = NULL;
1001 unsigned long pm;
1002
1003 errno = 0;
1004
1005
1006 pm = strtoul(portmask, &end, 16);
1007 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
1008 return -1;
1009
1010 if ((pm == 0) && errno)
1011 return -1;
1012
1013 return pm;
1014}
1015
1016static int64_t
1017parse_decimal(const char *str)
1018{
1019 char *end = NULL;
1020 uint64_t num;
1021
1022 num = strtoull(str, &end, 10);
1023 if ((str[0] == '\0') || (end == NULL) || (*end != '\0')
1024 || num > INT64_MAX)
1025 return -1;
1026
1027 return num;
1028}
1029
1030static int32_t
1031parse_config(const char *q_arg)
1032{
1033 char s[256];
1034 const char *p, *p0 = q_arg;
1035 char *end;
1036 enum fieldnames {
1037 FLD_PORT = 0,
1038 FLD_QUEUE,
1039 FLD_LCORE,
1040 _NUM_FLD
1041 };
1042 unsigned long int_fld[_NUM_FLD];
1043 char *str_fld[_NUM_FLD];
1044 int32_t i;
1045 uint32_t size;
1046
1047 nb_lcore_params = 0;
1048
1049 while ((p = strchr(p0, '(')) != NULL) {
1050 ++p;
1051 p0 = strchr(p, ')');
1052 if (p0 == NULL)
1053 return -1;
1054
1055 size = p0 - p;
1056 if (size >= sizeof(s))
1057 return -1;
1058
1059 snprintf(s, sizeof(s), "%.*s", size, p);
1060 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
1061 _NUM_FLD)
1062 return -1;
1063 for (i = 0; i < _NUM_FLD; i++) {
1064 errno = 0;
1065 int_fld[i] = strtoul(str_fld[i], &end, 0);
1066 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
1067 return -1;
1068 }
1069 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
1070 printf("exceeded max number of lcore params: %hu\n",
1071 nb_lcore_params);
1072 return -1;
1073 }
1074 lcore_params_array[nb_lcore_params].port_id =
1075 (uint8_t)int_fld[FLD_PORT];
1076 lcore_params_array[nb_lcore_params].queue_id =
1077 (uint8_t)int_fld[FLD_QUEUE];
1078 lcore_params_array[nb_lcore_params].lcore_id =
1079 (uint8_t)int_fld[FLD_LCORE];
1080 ++nb_lcore_params;
1081 }
1082 lcore_params = lcore_params_array;
1083 return 0;
1084}
1085
1086static void
1087print_app_sa_prm(const struct app_sa_prm *prm)
1088{
1089 printf("librte_ipsec usage: %s\n",
1090 (prm->enable == 0) ? "disabled" : "enabled");
1091
1092 printf("replay window size: %u\n", prm->window_size);
1093 printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled");
1094 printf("SA flags: %#" PRIx64 "\n", prm->flags);
1095 printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns);
1096}
1097
1098static int
1099parse_transfer_mode(struct eh_conf *conf, const char *optarg)
1100{
1101 if (!strcmp(CMD_LINE_ARG_POLL, optarg))
1102 conf->mode = EH_PKT_TRANSFER_MODE_POLL;
1103 else if (!strcmp(CMD_LINE_ARG_EVENT, optarg))
1104 conf->mode = EH_PKT_TRANSFER_MODE_EVENT;
1105 else {
1106 printf("Unsupported packet transfer mode\n");
1107 return -EINVAL;
1108 }
1109
1110 return 0;
1111}
1112
1113static int
1114parse_schedule_type(struct eh_conf *conf, const char *optarg)
1115{
1116 struct eventmode_conf *em_conf = NULL;
1117
1118
1119 em_conf = conf->mode_params;
1120
1121 if (!strcmp(CMD_LINE_ARG_ORDERED, optarg))
1122 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED;
1123 else if (!strcmp(CMD_LINE_ARG_ATOMIC, optarg))
1124 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ATOMIC;
1125 else if (!strcmp(CMD_LINE_ARG_PARALLEL, optarg))
1126 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_PARALLEL;
1127 else {
1128 printf("Unsupported queue schedule type\n");
1129 return -EINVAL;
1130 }
1131
1132 return 0;
1133}
1134
1135static int32_t
1136parse_args(int32_t argc, char **argv, struct eh_conf *eh_conf)
1137{
1138 int opt;
1139 int64_t ret;
1140 char **argvopt;
1141 int32_t option_index;
1142 char *prgname = argv[0];
1143 int32_t f_present = 0;
1144 struct eventmode_conf *em_conf = NULL;
1145
1146 argvopt = argv;
1147
1148 while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:c:t:s:",
1149 lgopts, &option_index)) != EOF) {
1150
1151 switch (opt) {
1152 case 'p':
1153 enabled_port_mask = parse_portmask(optarg);
1154 if (enabled_port_mask == 0) {
1155 printf("invalid portmask\n");
1156 print_usage(prgname);
1157 return -1;
1158 }
1159 break;
1160 case 'P':
1161 printf("Promiscuous mode selected\n");
1162 promiscuous_on = 1;
1163 break;
1164 case 'u':
1165 unprotected_port_mask = parse_portmask(optarg);
1166 if (unprotected_port_mask == 0) {
1167 printf("invalid unprotected portmask\n");
1168 print_usage(prgname);
1169 return -1;
1170 }
1171 break;
1172 case 'f':
1173 if (f_present == 1) {
1174 printf("\"-f\" option present more than "
1175 "once!\n");
1176 print_usage(prgname);
1177 return -1;
1178 }
1179 cfgfile = optarg;
1180 f_present = 1;
1181 break;
1182
1183 case 's':
1184 ret = parse_decimal(optarg);
1185 if (ret < 0) {
1186 printf("Invalid number of buffers in a pool: "
1187 "%s\n", optarg);
1188 print_usage(prgname);
1189 return -1;
1190 }
1191
1192 nb_bufs_in_pool = ret;
1193 break;
1194
1195 case 'j':
1196 ret = parse_decimal(optarg);
1197 if (ret < RTE_MBUF_DEFAULT_BUF_SIZE ||
1198 ret > UINT16_MAX) {
1199 printf("Invalid frame buffer size value: %s\n",
1200 optarg);
1201 print_usage(prgname);
1202 return -1;
1203 }
1204 frame_buf_size = ret;
1205 printf("Custom frame buffer size %u\n", frame_buf_size);
1206 break;
1207 case 'l':
1208 app_sa_prm.enable = 1;
1209 break;
1210 case 'w':
1211 app_sa_prm.window_size = parse_decimal(optarg);
1212 break;
1213 case 'e':
1214 app_sa_prm.enable_esn = 1;
1215 break;
1216 case 'a':
1217 app_sa_prm.enable = 1;
1218 app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM;
1219 break;
1220 case 'c':
1221 ret = parse_decimal(optarg);
1222 if (ret < 0) {
1223 printf("Invalid SA cache size: %s\n", optarg);
1224 print_usage(prgname);
1225 return -1;
1226 }
1227 app_sa_prm.cache_sz = ret;
1228 break;
1229 case 't':
1230 ret = parse_decimal(optarg);
1231 if (ret < 0) {
1232 printf("Invalid interval value: %s\n", optarg);
1233 print_usage(prgname);
1234 return -1;
1235 }
1236 stats_interval = ret;
1237 break;
1238 case CMD_LINE_OPT_CONFIG_NUM:
1239 ret = parse_config(optarg);
1240 if (ret) {
1241 printf("Invalid config\n");
1242 print_usage(prgname);
1243 return -1;
1244 }
1245 break;
1246 case CMD_LINE_OPT_SINGLE_SA_NUM:
1247 ret = parse_decimal(optarg);
1248 if (ret == -1 || ret > UINT32_MAX) {
1249 printf("Invalid argument[sa_idx]\n");
1250 print_usage(prgname);
1251 return -1;
1252 }
1253
1254
1255 single_sa = 1;
1256 single_sa_idx = ret;
1257 eh_conf->ipsec_mode = EH_IPSEC_MODE_TYPE_DRIVER;
1258 wrkr_flags |= SS_F;
1259 printf("Configured with single SA index %u\n",
1260 single_sa_idx);
1261 break;
1262 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM:
1263 ret = parse_portmask(optarg);
1264 if (ret == -1) {
1265 printf("Invalid argument[portmask]\n");
1266 print_usage(prgname);
1267 return -1;
1268 }
1269
1270
1271 enabled_cryptodev_mask = ret;
1272 break;
1273
1274 case CMD_LINE_OPT_TRANSFER_MODE_NUM:
1275 ret = parse_transfer_mode(eh_conf, optarg);
1276 if (ret < 0) {
1277 printf("Invalid packet transfer mode\n");
1278 print_usage(prgname);
1279 return -1;
1280 }
1281 break;
1282
1283 case CMD_LINE_OPT_SCHEDULE_TYPE_NUM:
1284 ret = parse_schedule_type(eh_conf, optarg);
1285 if (ret < 0) {
1286 printf("Invalid queue schedule type\n");
1287 print_usage(prgname);
1288 return -1;
1289 }
1290 break;
1291
1292 case CMD_LINE_OPT_RX_OFFLOAD_NUM:
1293 ret = parse_mask(optarg, &dev_rx_offload);
1294 if (ret != 0) {
1295 printf("Invalid argument for \'%s\': %s\n",
1296 CMD_LINE_OPT_RX_OFFLOAD, optarg);
1297 print_usage(prgname);
1298 return -1;
1299 }
1300 break;
1301 case CMD_LINE_OPT_TX_OFFLOAD_NUM:
1302 ret = parse_mask(optarg, &dev_tx_offload);
1303 if (ret != 0) {
1304 printf("Invalid argument for \'%s\': %s\n",
1305 CMD_LINE_OPT_TX_OFFLOAD, optarg);
1306 print_usage(prgname);
1307 return -1;
1308 }
1309 break;
1310 case CMD_LINE_OPT_REASSEMBLE_NUM:
1311 ret = parse_decimal(optarg);
1312 if (ret < 0 || ret > UINT32_MAX) {
1313 printf("Invalid argument for \'%s\': %s\n",
1314 CMD_LINE_OPT_REASSEMBLE, optarg);
1315 print_usage(prgname);
1316 return -1;
1317 }
1318 frag_tbl_sz = ret;
1319 break;
1320 case CMD_LINE_OPT_MTU_NUM:
1321 ret = parse_decimal(optarg);
1322 if (ret < 0 || ret > RTE_IPV4_MAX_PKT_LEN) {
1323 printf("Invalid argument for \'%s\': %s\n",
1324 CMD_LINE_OPT_MTU, optarg);
1325 print_usage(prgname);
1326 return -1;
1327 }
1328 mtu_size = ret;
1329 break;
1330 case CMD_LINE_OPT_FRAG_TTL_NUM:
1331 ret = parse_decimal(optarg);
1332 if (ret < 0 || ret > MAX_FRAG_TTL_NS) {
1333 printf("Invalid argument for \'%s\': %s\n",
1334 CMD_LINE_OPT_MTU, optarg);
1335 print_usage(prgname);
1336 return -1;
1337 }
1338 frag_ttl_ns = ret;
1339 break;
1340 case CMD_LINE_OPT_EVENT_VECTOR_NUM:
1341 em_conf = eh_conf->mode_params;
1342 em_conf->ext_params.event_vector = 1;
1343 break;
1344 case CMD_LINE_OPT_VECTOR_SIZE_NUM:
1345 ret = parse_decimal(optarg);
1346
1347 if (ret > MAX_PKT_BURST_VEC) {
1348 printf("Invalid argument for \'%s\': %s\n",
1349 CMD_LINE_OPT_VECTOR_SIZE, optarg);
1350 print_usage(prgname);
1351 return -1;
1352 }
1353 em_conf = eh_conf->mode_params;
1354 em_conf->ext_params.vector_size = ret;
1355 break;
1356 case CMD_LINE_OPT_VECTOR_TIMEOUT_NUM:
1357 ret = parse_decimal(optarg);
1358
1359 em_conf = eh_conf->mode_params;
1360 em_conf->vector_tmo_ns = ret;
1361 break;
1362 case CMD_LINE_OPT_VECTOR_POOL_SZ_NUM:
1363 ret = parse_decimal(optarg);
1364
1365 em_conf = eh_conf->mode_params;
1366 em_conf->vector_pool_sz = ret;
1367 break;
1368 case CMD_LINE_OPT_PER_PORT_POOL_NUM:
1369 per_port_pool = 1;
1370 break;
1371 case CMD_LINE_OPT_QP_DESC_NB_NUM:
1372 qp_desc_nb = parse_decimal(optarg);
1373 break;
1374 default:
1375 print_usage(prgname);
1376 return -1;
1377 }
1378 }
1379
1380 if (f_present == 0) {
1381 printf("Mandatory option \"-f\" not present\n");
1382 return -1;
1383 }
1384
1385
1386 if (multi_seg_required()) {
1387
1388 app_sa_prm.enable = 1;
1389 printf("frame buf size: %u, mtu: %u, "
1390 "number of reassemble entries: %u\n"
1391 "multi-segment support is required\n",
1392 frame_buf_size, mtu_size, frag_tbl_sz);
1393 }
1394
1395 print_app_sa_prm(&app_sa_prm);
1396
1397 if (optind >= 0)
1398 argv[optind-1] = prgname;
1399
1400 ret = optind-1;
1401 optind = 1;
1402 return ret;
1403}
1404
1405static void
1406print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr)
1407{
1408 char buf[RTE_ETHER_ADDR_FMT_SIZE];
1409 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr);
1410 printf("%s%s", name, buf);
1411}
1412
1413
1414
1415
1416int
1417add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr)
1418{
1419 if (port >= RTE_DIM(ethaddr_tbl))
1420 return -EINVAL;
1421
1422 ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr);
1423 rte_ether_addr_copy((struct rte_ether_addr *)ðaddr_tbl[port].dst,
1424 (struct rte_ether_addr *)(val_eth + port));
1425 return 0;
1426}
1427
1428
1429static void
1430check_all_ports_link_status(uint32_t port_mask)
1431{
1432#define CHECK_INTERVAL 100
1433#define MAX_CHECK_TIME 90
1434 uint16_t portid;
1435 uint8_t count, all_ports_up, print_flag = 0;
1436 struct rte_eth_link link;
1437 int ret;
1438 char link_status_text[RTE_ETH_LINK_MAX_STR_LEN];
1439
1440 printf("\nChecking link status");
1441 fflush(stdout);
1442 for (count = 0; count <= MAX_CHECK_TIME; count++) {
1443 all_ports_up = 1;
1444 RTE_ETH_FOREACH_DEV(portid) {
1445 if ((port_mask & (1 << portid)) == 0)
1446 continue;
1447 memset(&link, 0, sizeof(link));
1448 ret = rte_eth_link_get_nowait(portid, &link);
1449 if (ret < 0) {
1450 all_ports_up = 0;
1451 if (print_flag == 1)
1452 printf("Port %u link get failed: %s\n",
1453 portid, rte_strerror(-ret));
1454 continue;
1455 }
1456
1457 if (print_flag == 1) {
1458 rte_eth_link_to_str(link_status_text,
1459 sizeof(link_status_text), &link);
1460 printf("Port %d %s\n", portid,
1461 link_status_text);
1462 continue;
1463 }
1464
1465 if (link.link_status == RTE_ETH_LINK_DOWN) {
1466 all_ports_up = 0;
1467 break;
1468 }
1469 }
1470
1471 if (print_flag == 1)
1472 break;
1473
1474 if (all_ports_up == 0) {
1475 printf(".");
1476 fflush(stdout);
1477 rte_delay_ms(CHECK_INTERVAL);
1478 }
1479
1480
1481 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1482 print_flag = 1;
1483 printf("done\n");
1484 }
1485 }
1486}
1487
1488static int32_t
1489add_mapping(const char *str, uint16_t cdev_id,
1490 uint16_t qp, struct lcore_params *params,
1491 struct ipsec_ctx *ipsec_ctx,
1492 const struct rte_cryptodev_capabilities *cipher,
1493 const struct rte_cryptodev_capabilities *auth,
1494 const struct rte_cryptodev_capabilities *aead)
1495{
1496 int32_t ret = 0;
1497 unsigned long i;
1498 struct cdev_key key = { 0 };
1499
1500 key.lcore_id = params->lcore_id;
1501 if (cipher)
1502 key.cipher_algo = cipher->sym.cipher.algo;
1503 if (auth)
1504 key.auth_algo = auth->sym.auth.algo;
1505 if (aead)
1506 key.aead_algo = aead->sym.aead.algo;
1507
1508 ret = rte_hash_lookup(ipsec_ctx->cdev_map, &key);
1509 if (ret != -ENOENT)
1510 return 0;
1511
1512 for (i = 0; i < ipsec_ctx->nb_qps; i++)
1513 if (ipsec_ctx->tbl[i].id == cdev_id)
1514 break;
1515
1516 if (i == ipsec_ctx->nb_qps) {
1517 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) {
1518 printf("Maximum number of crypto devices assigned to "
1519 "a core, increase MAX_QP_PER_LCORE value\n");
1520 return 0;
1521 }
1522 ipsec_ctx->tbl[i].id = cdev_id;
1523 ipsec_ctx->tbl[i].qp = qp;
1524 ipsec_ctx->nb_qps++;
1525 printf("%s cdev mapping: lcore %u using cdev %u qp %u "
1526 "(cdev_id_qp %lu)\n", str, key.lcore_id,
1527 cdev_id, qp, i);
1528 }
1529
1530 ret = rte_hash_add_key_data(ipsec_ctx->cdev_map, &key, (void *)i);
1531 if (ret < 0) {
1532 printf("Failed to insert cdev mapping for (lcore %u, "
1533 "cdev %u, qp %u), errno %d\n",
1534 key.lcore_id, ipsec_ctx->tbl[i].id,
1535 ipsec_ctx->tbl[i].qp, ret);
1536 return 0;
1537 }
1538
1539 return 1;
1540}
1541
1542static int32_t
1543add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id,
1544 uint16_t qp, struct lcore_params *params)
1545{
1546 int32_t ret = 0;
1547 const struct rte_cryptodev_capabilities *i, *j;
1548 struct lcore_conf *qconf;
1549 struct ipsec_ctx *ipsec_ctx;
1550 const char *str;
1551
1552 qconf = &lcore_conf[params->lcore_id];
1553
1554 if (!is_unprotected_port(params->port_id)) {
1555 ipsec_ctx = &qconf->outbound;
1556 ipsec_ctx->cdev_map = cdev_map_out;
1557 str = "Outbound";
1558 } else {
1559 ipsec_ctx = &qconf->inbound;
1560 ipsec_ctx->cdev_map = cdev_map_in;
1561 str = "Inbound";
1562 }
1563
1564
1565 if (!(dev_info->feature_flags &
1566 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
1567 return ret;
1568
1569 for (i = dev_info->capabilities;
1570 i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) {
1571 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
1572 continue;
1573
1574 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) {
1575 ret |= add_mapping(str, cdev_id, qp, params,
1576 ipsec_ctx, NULL, NULL, i);
1577 continue;
1578 }
1579
1580 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
1581 continue;
1582
1583 for (j = dev_info->capabilities;
1584 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) {
1585 if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
1586 continue;
1587
1588 if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
1589 continue;
1590
1591 ret |= add_mapping(str, cdev_id, qp, params,
1592 ipsec_ctx, i, j, NULL);
1593 }
1594 }
1595
1596 return ret;
1597}
1598
1599
1600static int
1601check_cryptodev_mask(uint8_t cdev_id)
1602{
1603 if (enabled_cryptodev_mask & (1 << cdev_id))
1604 return 0;
1605
1606 return -1;
1607}
1608
1609static uint16_t
1610cryptodevs_init(uint16_t req_queue_num)
1611{
1612 struct rte_cryptodev_config dev_conf;
1613 struct rte_cryptodev_qp_conf qp_conf;
1614 uint16_t idx, max_nb_qps, qp, total_nb_qps, i;
1615 int16_t cdev_id;
1616 struct rte_hash_parameters params = { 0 };
1617
1618 const uint64_t mseg_flag = multi_seg_required() ?
1619 RTE_CRYPTODEV_FF_IN_PLACE_SGL : 0;
1620
1621 params.entries = CDEV_MAP_ENTRIES;
1622 params.key_len = sizeof(struct cdev_key);
1623 params.hash_func = rte_jhash;
1624 params.hash_func_init_val = 0;
1625 params.socket_id = rte_socket_id();
1626
1627 params.name = "cdev_map_in";
1628 cdev_map_in = rte_hash_create(¶ms);
1629 if (cdev_map_in == NULL)
1630 rte_panic("Failed to create cdev_map hash table, errno = %d\n",
1631 rte_errno);
1632
1633 params.name = "cdev_map_out";
1634 cdev_map_out = rte_hash_create(¶ms);
1635 if (cdev_map_out == NULL)
1636 rte_panic("Failed to create cdev_map hash table, errno = %d\n",
1637 rte_errno);
1638
1639 printf("lcore/cryptodev/qp mappings:\n");
1640
1641 idx = 0;
1642 total_nb_qps = 0;
1643 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
1644 struct rte_cryptodev_info cdev_info;
1645
1646 if (check_cryptodev_mask((uint8_t)cdev_id))
1647 continue;
1648
1649 rte_cryptodev_info_get(cdev_id, &cdev_info);
1650
1651 if ((mseg_flag & cdev_info.feature_flags) != mseg_flag)
1652 rte_exit(EXIT_FAILURE,
1653 "Device %hd does not support \'%s\' feature\n",
1654 cdev_id,
1655 rte_cryptodev_get_feature_name(mseg_flag));
1656
1657 if (nb_lcore_params > cdev_info.max_nb_queue_pairs)
1658 max_nb_qps = cdev_info.max_nb_queue_pairs;
1659 else
1660 max_nb_qps = nb_lcore_params;
1661
1662 qp = 0;
1663 i = 0;
1664 while (qp < max_nb_qps && i < nb_lcore_params) {
1665 if (add_cdev_mapping(&cdev_info, cdev_id, qp,
1666 &lcore_params[idx]))
1667 qp++;
1668 idx++;
1669 idx = idx % nb_lcore_params;
1670 i++;
1671 }
1672
1673 qp = RTE_MIN(max_nb_qps, RTE_MAX(req_queue_num, qp));
1674 if (qp == 0)
1675 continue;
1676
1677 total_nb_qps += qp;
1678 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
1679 dev_conf.nb_queue_pairs = qp;
1680 dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO;
1681
1682 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
1683 if (dev_max_sess != 0 &&
1684 dev_max_sess < get_nb_crypto_sessions())
1685 rte_exit(EXIT_FAILURE,
1686 "Device does not support at least %u "
1687 "sessions", get_nb_crypto_sessions());
1688
1689 if (rte_cryptodev_configure(cdev_id, &dev_conf))
1690 rte_panic("Failed to initialize cryptodev %u\n",
1691 cdev_id);
1692
1693 qp_conf.nb_descriptors = qp_desc_nb;
1694 qp_conf.mp_session =
1695 socket_ctx[dev_conf.socket_id].session_pool;
1696 qp_conf.mp_session_private =
1697 socket_ctx[dev_conf.socket_id].session_priv_pool;
1698 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++)
1699 if (rte_cryptodev_queue_pair_setup(cdev_id, qp,
1700 &qp_conf, dev_conf.socket_id))
1701 rte_panic("Failed to setup queue %u for "
1702 "cdev_id %u\n", 0, cdev_id);
1703
1704 if (rte_cryptodev_start(cdev_id))
1705 rte_panic("Failed to start cryptodev %u\n",
1706 cdev_id);
1707 }
1708
1709 printf("\n");
1710
1711 return total_nb_qps;
1712}
1713
1714static int
1715check_ptype(int portid)
1716{
1717 int l3_ipv4 = 0, l3_ipv6 = 0, l4_udp = 0, tunnel_esp = 0;
1718 int i, nb_ptypes;
1719 uint32_t mask;
1720
1721 mask = (RTE_PTYPE_L3_MASK | RTE_PTYPE_L4_MASK |
1722 RTE_PTYPE_TUNNEL_MASK);
1723
1724 nb_ptypes = rte_eth_dev_get_supported_ptypes(portid, mask, NULL, 0);
1725 if (nb_ptypes <= 0)
1726 return 0;
1727
1728 uint32_t ptypes[nb_ptypes];
1729
1730 nb_ptypes = rte_eth_dev_get_supported_ptypes(portid, mask, ptypes, nb_ptypes);
1731 for (i = 0; i < nb_ptypes; ++i) {
1732 if (RTE_ETH_IS_IPV4_HDR(ptypes[i]))
1733 l3_ipv4 = 1;
1734 if (RTE_ETH_IS_IPV6_HDR(ptypes[i]))
1735 l3_ipv6 = 1;
1736 if ((ptypes[i] & RTE_PTYPE_TUNNEL_MASK) == RTE_PTYPE_TUNNEL_ESP)
1737 tunnel_esp = 1;
1738 if ((ptypes[i] & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP)
1739 l4_udp = 1;
1740 }
1741
1742 if (l3_ipv4 == 0)
1743 printf("port %d cannot parse RTE_PTYPE_L3_IPV4\n", portid);
1744
1745 if (l3_ipv6 == 0)
1746 printf("port %d cannot parse RTE_PTYPE_L3_IPV6\n", portid);
1747
1748 if (l4_udp == 0)
1749 printf("port %d cannot parse RTE_PTYPE_L4_UDP\n", portid);
1750
1751 if (tunnel_esp == 0)
1752 printf("port %d cannot parse RTE_PTYPE_TUNNEL_ESP\n", portid);
1753
1754 if (l3_ipv4 && l3_ipv6 && l4_udp && tunnel_esp)
1755 return 1;
1756
1757 return 0;
1758
1759}
1760
1761static inline void
1762parse_ptype(struct rte_mbuf *m)
1763{
1764 uint32_t packet_type = RTE_PTYPE_UNKNOWN;
1765 const struct rte_ipv4_hdr *iph4;
1766 const struct rte_ipv6_hdr *iph6;
1767 const struct rte_ether_hdr *eth;
1768 const struct rte_udp_hdr *udp;
1769 uint16_t nat_port, ether_type;
1770 int next_proto = 0;
1771 size_t ext_len = 0;
1772 const uint8_t *p;
1773 uint32_t l3len;
1774
1775 eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
1776 ether_type = eth->ether_type;
1777
1778 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
1779 iph4 = (const struct rte_ipv4_hdr *)(eth + 1);
1780 l3len = ((iph4->version_ihl & RTE_IPV4_HDR_IHL_MASK) *
1781 RTE_IPV4_IHL_MULTIPLIER);
1782
1783 if (l3len == sizeof(struct rte_ipv4_hdr))
1784 packet_type |= RTE_PTYPE_L3_IPV4;
1785 else
1786 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1787
1788 next_proto = iph4->next_proto_id;
1789 p = (const uint8_t *)iph4;
1790 } else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
1791 iph6 = (const struct rte_ipv6_hdr *)(eth + 1);
1792 l3len = sizeof(struct ip6_hdr);
1793
1794
1795 next_proto = iph6->proto;
1796 p = (const uint8_t *)iph6;
1797 while (next_proto != IPPROTO_ESP && l3len < m->data_len &&
1798 (next_proto = rte_ipv6_get_next_ext(p + l3len,
1799 next_proto, &ext_len)) >= 0)
1800 l3len += ext_len;
1801
1802
1803 if (unlikely(l3len + RTE_ETHER_HDR_LEN > m->data_len))
1804 goto exit;
1805
1806 if (l3len == sizeof(struct ip6_hdr))
1807 packet_type |= RTE_PTYPE_L3_IPV6;
1808 else
1809 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1810 }
1811
1812 switch (next_proto) {
1813 case IPPROTO_ESP:
1814 packet_type |= RTE_PTYPE_TUNNEL_ESP;
1815 break;
1816 case IPPROTO_UDP:
1817 if (app_sa_prm.udp_encap == 1) {
1818 udp = (const struct rte_udp_hdr *)(p + l3len);
1819 nat_port = rte_cpu_to_be_16(IPSEC_NAT_T_PORT);
1820 if (udp->src_port == nat_port ||
1821 udp->dst_port == nat_port)
1822 packet_type |=
1823 MBUF_PTYPE_TUNNEL_ESP_IN_UDP;
1824 }
1825 break;
1826 default:
1827 break;
1828 }
1829exit:
1830 m->packet_type |= packet_type;
1831}
1832
1833static uint16_t
1834parse_ptype_cb(uint16_t port __rte_unused, uint16_t queue __rte_unused,
1835 struct rte_mbuf *pkts[], uint16_t nb_pkts,
1836 uint16_t max_pkts __rte_unused,
1837 void *user_param __rte_unused)
1838{
1839 uint32_t i;
1840
1841 if (unlikely(nb_pkts == 0))
1842 return nb_pkts;
1843
1844 rte_prefetch0(rte_pktmbuf_mtod(pkts[0], struct ether_hdr *));
1845 for (i = 0; i < (unsigned int) (nb_pkts - 1); ++i) {
1846 rte_prefetch0(rte_pktmbuf_mtod(pkts[i+1],
1847 struct ether_hdr *));
1848 parse_ptype(pkts[i]);
1849 }
1850 parse_ptype(pkts[i]);
1851
1852 return nb_pkts;
1853}
1854
1855static void
1856port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
1857{
1858 struct rte_eth_dev_info dev_info;
1859 struct rte_eth_txconf *txconf;
1860 uint16_t nb_tx_queue, nb_rx_queue;
1861 uint16_t tx_queueid, rx_queueid, queue, lcore_id;
1862 int32_t ret, socket_id;
1863 struct lcore_conf *qconf;
1864 struct rte_ether_addr ethaddr;
1865 struct rte_eth_conf local_port_conf = port_conf;
1866 int ptype_supported;
1867
1868 ret = rte_eth_dev_info_get(portid, &dev_info);
1869 if (ret != 0)
1870 rte_exit(EXIT_FAILURE,
1871 "Error during getting device (port %u) info: %s\n",
1872 portid, strerror(-ret));
1873
1874
1875 dev_info.rx_offload_capa &= dev_rx_offload;
1876 dev_info.tx_offload_capa &= dev_tx_offload;
1877
1878 printf("Configuring device port %u:\n", portid);
1879
1880 ret = rte_eth_macaddr_get(portid, ðaddr);
1881 if (ret != 0)
1882 rte_exit(EXIT_FAILURE,
1883 "Error getting MAC address (port %u): %s\n",
1884 portid, rte_strerror(-ret));
1885
1886 ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(ðaddr);
1887
1888 rte_ether_addr_copy((struct rte_ether_addr *)ðaddr_tbl[portid].dst,
1889 (struct rte_ether_addr *)(val_eth + portid));
1890 rte_ether_addr_copy((struct rte_ether_addr *)ðaddr_tbl[portid].src,
1891 (struct rte_ether_addr *)(val_eth + portid) + 1);
1892
1893 print_ethaddr("Address: ", ðaddr);
1894 printf("\n");
1895
1896 nb_rx_queue = get_port_nb_rx_queues(portid);
1897 nb_tx_queue = nb_lcores;
1898
1899 if (nb_rx_queue > dev_info.max_rx_queues)
1900 rte_exit(EXIT_FAILURE, "Error: queue %u not available "
1901 "(max rx queue is %u)\n",
1902 nb_rx_queue, dev_info.max_rx_queues);
1903
1904 if (nb_tx_queue > dev_info.max_tx_queues)
1905 rte_exit(EXIT_FAILURE, "Error: queue %u not available "
1906 "(max tx queue is %u)\n",
1907 nb_tx_queue, dev_info.max_tx_queues);
1908
1909 printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n",
1910 nb_rx_queue, nb_tx_queue);
1911
1912 local_port_conf.rxmode.mtu = mtu_size;
1913
1914 if (multi_seg_required()) {
1915 local_port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_SCATTER;
1916 local_port_conf.txmode.offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
1917 }
1918
1919 local_port_conf.rxmode.offloads |= req_rx_offloads;
1920 local_port_conf.txmode.offloads |= req_tx_offloads;
1921
1922
1923 if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
1924 local_port_conf.rxmode.offloads)
1925 rte_exit(EXIT_FAILURE,
1926 "Error: port %u required RX offloads: 0x%" PRIx64
1927 ", available RX offloads: 0x%" PRIx64 "\n",
1928 portid, local_port_conf.rxmode.offloads,
1929 dev_info.rx_offload_capa);
1930
1931 if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) !=
1932 local_port_conf.txmode.offloads)
1933 rte_exit(EXIT_FAILURE,
1934 "Error: port %u required TX offloads: 0x%" PRIx64
1935 ", available TX offloads: 0x%" PRIx64 "\n",
1936 portid, local_port_conf.txmode.offloads,
1937 dev_info.tx_offload_capa);
1938
1939 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)
1940 local_port_conf.txmode.offloads |=
1941 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1942
1943 printf("port %u configuring rx_offloads=0x%" PRIx64
1944 ", tx_offloads=0x%" PRIx64 "\n",
1945 portid, local_port_conf.rxmode.offloads,
1946 local_port_conf.txmode.offloads);
1947
1948 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
1949 dev_info.flow_type_rss_offloads;
1950 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
1951 port_conf.rx_adv_conf.rss_conf.rss_hf) {
1952 printf("Port %u modified RSS hash function based on hardware support,"
1953 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
1954 portid,
1955 port_conf.rx_adv_conf.rss_conf.rss_hf,
1956 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
1957 }
1958
1959 ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue,
1960 &local_port_conf);
1961 if (ret < 0)
1962 rte_exit(EXIT_FAILURE, "Cannot configure device: "
1963 "err=%d, port=%d\n", ret, portid);
1964
1965 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd);
1966 if (ret < 0)
1967 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: "
1968 "err=%d, port=%d\n", ret, portid);
1969
1970
1971 ptype_supported = check_ptype(portid);
1972 if (!ptype_supported)
1973 printf("Port %d: softly parse packet type info\n", portid);
1974
1975
1976 tx_queueid = 0;
1977 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1978 if (rte_lcore_is_enabled(lcore_id) == 0)
1979 continue;
1980
1981 if (numa_on)
1982 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id);
1983 else
1984 socket_id = 0;
1985
1986
1987 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id);
1988
1989 txconf = &dev_info.default_txconf;
1990 txconf->offloads = local_port_conf.txmode.offloads;
1991
1992 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd,
1993 socket_id, txconf);
1994 if (ret < 0)
1995 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: "
1996 "err=%d, port=%d\n", ret, portid);
1997
1998 qconf = &lcore_conf[lcore_id];
1999 qconf->tx_queue_id[portid] = tx_queueid;
2000
2001
2002 qconf->outbound.ipv4_offloads = RTE_MBUF_F_TX_IPV4;
2003 qconf->outbound.ipv6_offloads = RTE_MBUF_F_TX_IPV6;
2004 if (local_port_conf.txmode.offloads & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM)
2005 qconf->outbound.ipv4_offloads |= RTE_MBUF_F_TX_IP_CKSUM;
2006
2007 tx_queueid++;
2008
2009
2010 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) {
2011 struct rte_eth_rxconf rxq_conf;
2012 struct rte_mempool *pool;
2013
2014 if (portid != qconf->rx_queue_list[queue].port_id)
2015 continue;
2016
2017 rx_queueid = qconf->rx_queue_list[queue].queue_id;
2018
2019 printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid,
2020 socket_id);
2021
2022 rxq_conf = dev_info.default_rxconf;
2023 rxq_conf.offloads = local_port_conf.rxmode.offloads;
2024
2025 if (per_port_pool)
2026 pool = socket_ctx[socket_id].mbuf_pool[portid];
2027 else
2028 pool = socket_ctx[socket_id].mbuf_pool[0];
2029
2030 ret = rte_eth_rx_queue_setup(portid, rx_queueid,
2031 nb_rxd, socket_id, &rxq_conf, pool);
2032 if (ret < 0)
2033 rte_exit(EXIT_FAILURE,
2034 "rte_eth_rx_queue_setup: err=%d, "
2035 "port=%d\n", ret, portid);
2036
2037
2038 if (!ptype_supported &&
2039 !rte_eth_add_rx_callback(portid, queue,
2040 parse_ptype_cb, NULL)) {
2041 printf("Failed to add rx callback: port=%d, "
2042 "queue=%d\n", portid, queue);
2043 }
2044
2045
2046 }
2047 }
2048 printf("\n");
2049}
2050
2051static size_t
2052max_session_size(void)
2053{
2054 size_t max_sz, sz;
2055 void *sec_ctx;
2056 int16_t cdev_id, port_id, n;
2057
2058 max_sz = 0;
2059 n = rte_cryptodev_count();
2060 for (cdev_id = 0; cdev_id != n; cdev_id++) {
2061 sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
2062 if (sz > max_sz)
2063 max_sz = sz;
2064
2065
2066
2067
2068
2069
2070 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
2071 if (sec_ctx == NULL)
2072 continue;
2073
2074
2075 sz = rte_security_session_get_size(sec_ctx);
2076 if (sz > max_sz)
2077 max_sz = sz;
2078 }
2079
2080 RTE_ETH_FOREACH_DEV(port_id) {
2081 if ((enabled_port_mask & (1 << port_id)) == 0)
2082 continue;
2083
2084 sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
2085 if (sec_ctx == NULL)
2086 continue;
2087
2088 sz = rte_security_session_get_size(sec_ctx);
2089 if (sz > max_sz)
2090 max_sz = sz;
2091 }
2092
2093 return max_sz;
2094}
2095
2096static void
2097session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
2098{
2099 char mp_name[RTE_MEMPOOL_NAMESIZE];
2100 struct rte_mempool *sess_mp;
2101 uint32_t nb_sess;
2102
2103 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
2104 "sess_mp_%u", socket_id);
2105 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ *
2106 rte_lcore_count());
2107 nb_sess = RTE_MAX(nb_sess, CDEV_MP_CACHE_SZ *
2108 CDEV_MP_CACHE_MULTIPLIER);
2109 sess_mp = rte_cryptodev_sym_session_pool_create(
2110 mp_name, nb_sess, sess_sz, CDEV_MP_CACHE_SZ, 0,
2111 socket_id);
2112 ctx->session_pool = sess_mp;
2113
2114 if (ctx->session_pool == NULL)
2115 rte_exit(EXIT_FAILURE,
2116 "Cannot init session pool on socket %d\n", socket_id);
2117 else
2118 printf("Allocated session pool on socket %d\n", socket_id);
2119}
2120
2121static void
2122session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
2123 size_t sess_sz)
2124{
2125 char mp_name[RTE_MEMPOOL_NAMESIZE];
2126 struct rte_mempool *sess_mp;
2127 uint32_t nb_sess;
2128
2129 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
2130 "sess_mp_priv_%u", socket_id);
2131 nb_sess = (get_nb_crypto_sessions() + CDEV_MP_CACHE_SZ *
2132 rte_lcore_count());
2133 nb_sess = RTE_MAX(nb_sess, CDEV_MP_CACHE_SZ *
2134 CDEV_MP_CACHE_MULTIPLIER);
2135 sess_mp = rte_mempool_create(mp_name,
2136 nb_sess,
2137 sess_sz,
2138 CDEV_MP_CACHE_SZ,
2139 0, NULL, NULL, NULL,
2140 NULL, socket_id,
2141 0);
2142 ctx->session_priv_pool = sess_mp;
2143
2144 if (ctx->session_priv_pool == NULL)
2145 rte_exit(EXIT_FAILURE,
2146 "Cannot init session priv pool on socket %d\n",
2147 socket_id);
2148 else
2149 printf("Allocated session priv pool on socket %d\n",
2150 socket_id);
2151}
2152
2153static void
2154pool_init(struct socket_ctx *ctx, int32_t socket_id, int portid,
2155 uint32_t nb_mbuf)
2156{
2157 char s[64];
2158 int32_t ms;
2159
2160
2161
2162 if (socket_ctx[socket_id].mbuf_pool[portid])
2163 return;
2164
2165 snprintf(s, sizeof(s), "mbuf_pool_%d_%d", socket_id, portid);
2166 ctx->mbuf_pool[portid] = rte_pktmbuf_pool_create(s, nb_mbuf,
2167 MEMPOOL_CACHE_SIZE,
2168 ipsec_metadata_size(),
2169 frame_buf_size,
2170 socket_id);
2171
2172
2173
2174
2175
2176 ms = multi_seg_required();
2177 if (ms != 0 && !ctx->mbuf_pool_indir) {
2178 snprintf(s, sizeof(s), "mbuf_pool_indir_%d", socket_id);
2179 ctx->mbuf_pool_indir = rte_pktmbuf_pool_create(s, nb_mbuf,
2180 MEMPOOL_CACHE_SIZE, 0, 0, socket_id);
2181 }
2182
2183 if (ctx->mbuf_pool[portid] == NULL ||
2184 (ms != 0 && ctx->mbuf_pool_indir == NULL))
2185 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n",
2186 socket_id);
2187 else
2188 printf("Allocated mbuf pool on socket %d\n", socket_id);
2189}
2190
2191static inline int
2192inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md)
2193{
2194 struct ipsec_sa *sa;
2195
2196
2197
2198
2199
2200
2201
2202 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md);
2203
2204 if (sa == NULL) {
2205
2206 return -1;
2207 }
2208
2209
2210 RTE_SET_USED(sa);
2211 return 0;
2212}
2213
2214static int
2215inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type,
2216 void *param, void *ret_param)
2217{
2218 uint64_t md;
2219 struct rte_eth_event_ipsec_desc *event_desc = NULL;
2220 struct rte_security_ctx *ctx = (struct rte_security_ctx *)
2221 rte_eth_dev_get_sec_ctx(port_id);
2222
2223 RTE_SET_USED(param);
2224
2225 if (type != RTE_ETH_EVENT_IPSEC)
2226 return -1;
2227
2228 event_desc = ret_param;
2229 if (event_desc == NULL) {
2230 printf("Event descriptor not set\n");
2231 return -1;
2232 }
2233
2234 md = event_desc->metadata;
2235
2236 if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW)
2237 return inline_ipsec_event_esn_overflow(ctx, md);
2238 else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) {
2239 printf("Invalid IPsec event reported\n");
2240 return -1;
2241 }
2242
2243 return -1;
2244}
2245
2246static int
2247ethdev_reset_event_callback(uint16_t port_id,
2248 enum rte_eth_event_type type,
2249 void *param __rte_unused, void *ret_param __rte_unused)
2250{
2251 printf("Reset Event on port id %d type %d\n", port_id, type);
2252 printf("Force quit application");
2253 force_quit = true;
2254 return 0;
2255}
2256
2257static uint16_t
2258rx_callback(__rte_unused uint16_t port, __rte_unused uint16_t queue,
2259 struct rte_mbuf *pkt[], uint16_t nb_pkts,
2260 __rte_unused uint16_t max_pkts, void *user_param)
2261{
2262 uint64_t tm;
2263 uint32_t i, k;
2264 struct lcore_conf *lc;
2265 struct rte_mbuf *mb;
2266 struct rte_ether_hdr *eth;
2267
2268 lc = user_param;
2269 k = 0;
2270 tm = 0;
2271
2272 for (i = 0; i != nb_pkts; i++) {
2273
2274 mb = pkt[i];
2275 eth = rte_pktmbuf_mtod(mb, struct rte_ether_hdr *);
2276 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
2277
2278 struct rte_ipv4_hdr *iph;
2279
2280 iph = (struct rte_ipv4_hdr *)(eth + 1);
2281 if (rte_ipv4_frag_pkt_is_fragmented(iph)) {
2282
2283 mb->l2_len = sizeof(*eth);
2284 mb->l3_len = sizeof(*iph);
2285 tm = (tm != 0) ? tm : rte_rdtsc();
2286 mb = rte_ipv4_frag_reassemble_packet(
2287 lc->frag.tbl, &lc->frag.dr,
2288 mb, tm, iph);
2289
2290 if (mb != NULL) {
2291
2292 iph = rte_pktmbuf_mtod_offset(mb,
2293 struct rte_ipv4_hdr *,
2294 mb->l2_len);
2295 iph->hdr_checksum = 0;
2296 iph->hdr_checksum = rte_ipv4_cksum(iph);
2297 }
2298 }
2299 } else if (eth->ether_type ==
2300 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
2301
2302 struct rte_ipv6_hdr *iph;
2303 struct rte_ipv6_fragment_ext *fh;
2304
2305 iph = (struct rte_ipv6_hdr *)(eth + 1);
2306 fh = rte_ipv6_frag_get_ipv6_fragment_header(iph);
2307 if (fh != NULL) {
2308 mb->l2_len = sizeof(*eth);
2309 mb->l3_len = (uintptr_t)fh - (uintptr_t)iph +
2310 sizeof(*fh);
2311 tm = (tm != 0) ? tm : rte_rdtsc();
2312 mb = rte_ipv6_frag_reassemble_packet(
2313 lc->frag.tbl, &lc->frag.dr,
2314 mb, tm, iph, fh);
2315 if (mb != NULL)
2316
2317 mb->l3_len = mb->l3_len - sizeof(*fh);
2318 }
2319 }
2320
2321 pkt[k] = mb;
2322 k += (mb != NULL);
2323 }
2324
2325
2326 if (tm != 0)
2327 rte_ip_frag_free_death_row(&lc->frag.dr, 0);
2328
2329 return k;
2330}
2331
2332
2333static int
2334reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid)
2335{
2336 int32_t sid;
2337 uint32_t i;
2338 uint64_t frag_cycles;
2339 const struct lcore_rx_queue *rxq;
2340 const struct rte_eth_rxtx_callback *cb;
2341
2342
2343 sid = rte_lcore_to_socket_id(cid);
2344 frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) /
2345 NS_PER_S * frag_ttl_ns;
2346
2347 lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz,
2348 FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid);
2349 if (lc->frag.tbl == NULL) {
2350 printf("%s(%u): failed to create fragment table of size: %u, "
2351 "error code: %d\n",
2352 __func__, cid, frag_tbl_sz, rte_errno);
2353 return -ENOMEM;
2354 }
2355
2356
2357 for (i = 0; i != lc->nb_rx_queue; i++) {
2358
2359 rxq = lc->rx_queue_list + i;
2360 cb = rte_eth_add_rx_callback(rxq->port_id, rxq->queue_id,
2361 rx_callback, lc);
2362 if (cb == NULL) {
2363 printf("%s(%u): failed to install RX callback for "
2364 "portid=%u, queueid=%u, error code: %d\n",
2365 __func__, cid,
2366 rxq->port_id, rxq->queue_id, rte_errno);
2367 return -ENOMEM;
2368 }
2369 }
2370
2371 return 0;
2372}
2373
2374static int
2375reassemble_init(void)
2376{
2377 int32_t rc;
2378 uint32_t i, lc;
2379
2380 rc = 0;
2381 for (i = 0; i != nb_lcore_params; i++) {
2382 lc = lcore_params[i].lcore_id;
2383 rc = reassemble_lcore_init(lcore_conf + lc, lc);
2384 if (rc != 0)
2385 break;
2386 }
2387
2388 return rc;
2389}
2390
2391static void
2392create_default_ipsec_flow(uint16_t port_id, uint64_t rx_offloads)
2393{
2394 struct rte_flow_action action[2];
2395 struct rte_flow_item pattern[2];
2396 struct rte_flow_attr attr = {0};
2397 struct rte_flow_error err;
2398 struct rte_flow *flow;
2399 int ret;
2400
2401 if (!(rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY))
2402 return;
2403
2404
2405
2406 pattern[0].type = RTE_FLOW_ITEM_TYPE_ESP;
2407 pattern[0].spec = NULL;
2408 pattern[0].mask = NULL;
2409 pattern[0].last = NULL;
2410 pattern[1].type = RTE_FLOW_ITEM_TYPE_END;
2411
2412 action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
2413 action[0].conf = NULL;
2414 action[1].type = RTE_FLOW_ACTION_TYPE_END;
2415 action[1].conf = NULL;
2416
2417 attr.ingress = 1;
2418
2419 ret = rte_flow_validate(port_id, &attr, pattern, action, &err);
2420 if (ret)
2421 return;
2422
2423 flow = rte_flow_create(port_id, &attr, pattern, action, &err);
2424 if (flow == NULL)
2425 return;
2426
2427 flow_info_tbl[port_id].rx_def_flow = flow;
2428 RTE_LOG(INFO, IPSEC,
2429 "Created default flow enabling SECURITY for all ESP traffic on port %d\n",
2430 port_id);
2431}
2432
2433static void
2434signal_handler(int signum)
2435{
2436 if (signum == SIGINT || signum == SIGTERM) {
2437 printf("\n\nSignal %d received, preparing to exit...\n",
2438 signum);
2439 force_quit = true;
2440 }
2441}
2442
2443static void
2444ev_mode_sess_verify(struct ipsec_sa *sa, int nb_sa)
2445{
2446 struct rte_ipsec_session *ips;
2447 int32_t i;
2448
2449 if (!sa || !nb_sa)
2450 return;
2451
2452 for (i = 0; i < nb_sa; i++) {
2453 ips = ipsec_get_primary_session(&sa[i]);
2454 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
2455 rte_exit(EXIT_FAILURE, "Event mode supports only "
2456 "inline protocol sessions\n");
2457 }
2458
2459}
2460
2461static int32_t
2462check_event_mode_params(struct eh_conf *eh_conf)
2463{
2464 struct eventmode_conf *em_conf = NULL;
2465 struct lcore_params *params;
2466 uint16_t portid;
2467
2468 if (!eh_conf || !eh_conf->mode_params)
2469 return -EINVAL;
2470
2471
2472 em_conf = eh_conf->mode_params;
2473
2474 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_POLL &&
2475 em_conf->ext_params.sched_type != SCHED_TYPE_NOT_SET) {
2476 printf("error: option --event-schedule-type applies only to "
2477 "event mode\n");
2478 return -EINVAL;
2479 }
2480
2481 if (eh_conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
2482 return 0;
2483
2484
2485 if (em_conf->ext_params.sched_type == SCHED_TYPE_NOT_SET)
2486 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED;
2487
2488
2489
2490
2491
2492
2493 ev_mode_sess_verify(sa_in, nb_sa_in);
2494 ev_mode_sess_verify(sa_out, nb_sa_out);
2495
2496
2497
2498 if (nb_lcore_params > 0) {
2499 printf("error: option --config applies only to poll mode\n");
2500 return -EINVAL;
2501 }
2502
2503
2504
2505
2506
2507 lcore_params = lcore_params_array;
2508 RTE_ETH_FOREACH_DEV(portid) {
2509 if ((enabled_port_mask & (1 << portid)) == 0)
2510 continue;
2511
2512 params = &lcore_params[nb_lcore_params++];
2513 params->port_id = portid;
2514 params->queue_id = 0;
2515 params->lcore_id = rte_get_next_lcore(0, 0, 1);
2516 }
2517
2518 return 0;
2519}
2520
2521static int
2522one_session_free(struct rte_ipsec_session *ips)
2523{
2524 int32_t ret = 0;
2525
2526 if (ips->type == RTE_SECURITY_ACTION_TYPE_NONE ||
2527 ips->type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) {
2528
2529 if (ips->crypto.ses == NULL)
2530 return 0;
2531
2532 ret = rte_cryptodev_sym_session_clear(ips->crypto.dev_id,
2533 ips->crypto.ses);
2534 if (ret)
2535 return ret;
2536
2537 ret = rte_cryptodev_sym_session_free(ips->crypto.ses);
2538 } else {
2539
2540 if (ips->security.ctx == NULL || ips->security.ses == NULL)
2541 return 0;
2542
2543 ret = rte_security_session_destroy(ips->security.ctx,
2544 ips->security.ses);
2545 }
2546
2547 return ret;
2548}
2549
2550static void
2551sessions_free(struct sa_ctx *sa_ctx)
2552{
2553 struct rte_ipsec_session *ips;
2554 struct ipsec_sa *sa;
2555 int32_t ret;
2556 uint32_t i;
2557
2558 if (!sa_ctx)
2559 return;
2560
2561 for (i = 0; i < sa_ctx->nb_sa; i++) {
2562
2563 sa = &sa_ctx->sa[i];
2564 if (!sa->spi)
2565 continue;
2566
2567 ips = ipsec_get_primary_session(sa);
2568 ret = one_session_free(ips);
2569 if (ret)
2570 RTE_LOG(ERR, IPSEC, "Failed to destroy security "
2571 "session type %d, spi %d\n",
2572 ips->type, sa->spi);
2573 }
2574}
2575
2576static uint32_t
2577calculate_nb_mbufs(uint16_t nb_ports, uint16_t nb_crypto_qp, uint32_t nb_rxq,
2578 uint32_t nb_txq)
2579{
2580 return RTE_MAX((nb_rxq * nb_rxd +
2581 nb_ports * nb_lcores * MAX_PKT_BURST +
2582 nb_ports * nb_txq * nb_txd +
2583 nb_lcores * MEMPOOL_CACHE_SIZE +
2584 nb_crypto_qp * qp_desc_nb +
2585 nb_lcores * frag_tbl_sz *
2586 FRAG_TBL_BUCKET_ENTRIES),
2587 8192U);
2588}
2589
2590
2591static int
2592handle_telemetry_cmd_ipsec_secgw_stats(const char *cmd __rte_unused,
2593 const char *params, struct rte_tel_data *data)
2594{
2595 uint64_t total_pkts_dropped = 0, total_pkts_tx = 0, total_pkts_rx = 0;
2596 unsigned int coreid;
2597
2598 rte_tel_data_start_dict(data);
2599
2600 if (params) {
2601 coreid = (uint32_t)atoi(params);
2602 if (rte_lcore_is_enabled(coreid) == 0)
2603 return -EINVAL;
2604
2605 total_pkts_dropped = core_statistics[coreid].dropped;
2606 total_pkts_tx = core_statistics[coreid].tx;
2607 total_pkts_rx = core_statistics[coreid].rx;
2608
2609 } else {
2610 for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++) {
2611
2612
2613 if (rte_lcore_is_enabled(coreid) == 0)
2614 continue;
2615
2616 total_pkts_dropped += core_statistics[coreid].dropped;
2617 total_pkts_tx += core_statistics[coreid].tx;
2618 total_pkts_rx += core_statistics[coreid].rx;
2619 }
2620 }
2621
2622
2623 rte_tel_data_add_dict_u64(data, "packets received",
2624 total_pkts_rx);
2625
2626 rte_tel_data_add_dict_u64(data, "packets transmitted",
2627 total_pkts_tx);
2628
2629 rte_tel_data_add_dict_u64(data, "packets dropped",
2630 total_pkts_dropped);
2631
2632
2633 return 0;
2634}
2635
2636static void
2637update_lcore_statistics(struct ipsec_core_statistics *total, uint32_t coreid)
2638{
2639 struct ipsec_core_statistics *lcore_stats;
2640
2641
2642 if (rte_lcore_is_enabled(coreid) == 0)
2643 return;
2644
2645 lcore_stats = &core_statistics[coreid];
2646
2647 total->rx = lcore_stats->rx;
2648 total->dropped = lcore_stats->dropped;
2649 total->tx = lcore_stats->tx;
2650
2651
2652 total->outbound.spd6.protect += lcore_stats->outbound.spd6.protect;
2653 total->outbound.spd6.bypass += lcore_stats->outbound.spd6.bypass;
2654 total->outbound.spd6.discard += lcore_stats->outbound.spd6.discard;
2655
2656 total->outbound.spd4.protect += lcore_stats->outbound.spd4.protect;
2657 total->outbound.spd4.bypass += lcore_stats->outbound.spd4.bypass;
2658 total->outbound.spd4.discard += lcore_stats->outbound.spd4.discard;
2659
2660 total->outbound.sad.miss += lcore_stats->outbound.sad.miss;
2661
2662
2663 total->inbound.spd6.protect += lcore_stats->inbound.spd6.protect;
2664 total->inbound.spd6.bypass += lcore_stats->inbound.spd6.bypass;
2665 total->inbound.spd6.discard += lcore_stats->inbound.spd6.discard;
2666
2667 total->inbound.spd4.protect += lcore_stats->inbound.spd4.protect;
2668 total->inbound.spd4.bypass += lcore_stats->inbound.spd4.bypass;
2669 total->inbound.spd4.discard += lcore_stats->inbound.spd4.discard;
2670
2671 total->inbound.sad.miss += lcore_stats->inbound.sad.miss;
2672
2673
2674
2675 total->lpm4.miss += lcore_stats->lpm4.miss;
2676 total->lpm6.miss += lcore_stats->lpm6.miss;
2677}
2678
2679static void
2680update_statistics(struct ipsec_core_statistics *total, uint32_t coreid)
2681{
2682 memset(total, 0, sizeof(*total));
2683
2684 if (coreid != UINT32_MAX) {
2685 update_lcore_statistics(total, coreid);
2686 } else {
2687 for (coreid = 0; coreid < RTE_MAX_LCORE; coreid++)
2688 update_lcore_statistics(total, coreid);
2689 }
2690}
2691
2692static int
2693handle_telemetry_cmd_ipsec_secgw_stats_outbound(const char *cmd __rte_unused,
2694 const char *params, struct rte_tel_data *data)
2695{
2696 struct ipsec_core_statistics total_stats;
2697
2698 struct rte_tel_data *spd4_data = rte_tel_data_alloc();
2699 struct rte_tel_data *spd6_data = rte_tel_data_alloc();
2700 struct rte_tel_data *sad_data = rte_tel_data_alloc();
2701 unsigned int coreid = UINT32_MAX;
2702 int rc = 0;
2703
2704
2705 if (!spd4_data || !spd6_data || !sad_data) {
2706 rc = -ENOMEM;
2707 goto exit;
2708 }
2709
2710
2711 rte_tel_data_start_dict(data);
2712
2713 rte_tel_data_start_dict(spd4_data);
2714 rte_tel_data_start_dict(spd6_data);
2715 rte_tel_data_start_dict(sad_data);
2716
2717 if (params) {
2718 coreid = (uint32_t)atoi(params);
2719 if (rte_lcore_is_enabled(coreid) == 0) {
2720 rc = -EINVAL;
2721 goto exit;
2722 }
2723 }
2724
2725 update_statistics(&total_stats, coreid);
2726
2727
2728
2729 rte_tel_data_add_dict_u64(spd4_data, "protect",
2730 total_stats.outbound.spd4.protect);
2731 rte_tel_data_add_dict_u64(spd4_data, "bypass",
2732 total_stats.outbound.spd4.bypass);
2733 rte_tel_data_add_dict_u64(spd4_data, "discard",
2734 total_stats.outbound.spd4.discard);
2735
2736 rte_tel_data_add_dict_container(data, "spd4", spd4_data, 0);
2737
2738
2739
2740 rte_tel_data_add_dict_u64(spd6_data, "protect",
2741 total_stats.outbound.spd6.protect);
2742 rte_tel_data_add_dict_u64(spd6_data, "bypass",
2743 total_stats.outbound.spd6.bypass);
2744 rte_tel_data_add_dict_u64(spd6_data, "discard",
2745 total_stats.outbound.spd6.discard);
2746
2747 rte_tel_data_add_dict_container(data, "spd6", spd6_data, 0);
2748
2749
2750
2751 rte_tel_data_add_dict_u64(sad_data, "miss",
2752 total_stats.outbound.sad.miss);
2753
2754 rte_tel_data_add_dict_container(data, "sad", sad_data, 0);
2755
2756exit:
2757 if (rc) {
2758 rte_tel_data_free(spd4_data);
2759 rte_tel_data_free(spd6_data);
2760 rte_tel_data_free(sad_data);
2761 }
2762 return rc;
2763}
2764
2765static int
2766handle_telemetry_cmd_ipsec_secgw_stats_inbound(const char *cmd __rte_unused,
2767 const char *params, struct rte_tel_data *data)
2768{
2769 struct ipsec_core_statistics total_stats;
2770
2771 struct rte_tel_data *spd4_data = rte_tel_data_alloc();
2772 struct rte_tel_data *spd6_data = rte_tel_data_alloc();
2773 struct rte_tel_data *sad_data = rte_tel_data_alloc();
2774 unsigned int coreid = UINT32_MAX;
2775 int rc = 0;
2776
2777
2778 if (!spd4_data || !spd6_data || !sad_data) {
2779 rc = -ENOMEM;
2780 goto exit;
2781 }
2782
2783
2784 rte_tel_data_start_dict(data);
2785 rte_tel_data_start_dict(spd4_data);
2786 rte_tel_data_start_dict(spd6_data);
2787 rte_tel_data_start_dict(sad_data);
2788
2789
2790
2791 if (params) {
2792 coreid = (uint32_t)atoi(params);
2793 if (rte_lcore_is_enabled(coreid) == 0) {
2794 rc = -EINVAL;
2795 goto exit;
2796 }
2797 }
2798
2799 update_statistics(&total_stats, coreid);
2800
2801
2802
2803 rte_tel_data_add_dict_u64(sad_data, "miss",
2804 total_stats.inbound.sad.miss);
2805
2806 rte_tel_data_add_dict_container(data, "sad", sad_data, 0);
2807
2808
2809
2810 rte_tel_data_add_dict_u64(spd4_data, "protect",
2811 total_stats.inbound.spd4.protect);
2812 rte_tel_data_add_dict_u64(spd4_data, "bypass",
2813 total_stats.inbound.spd4.bypass);
2814 rte_tel_data_add_dict_u64(spd4_data, "discard",
2815 total_stats.inbound.spd4.discard);
2816
2817 rte_tel_data_add_dict_container(data, "spd4", spd4_data, 0);
2818
2819
2820
2821 rte_tel_data_add_dict_u64(spd6_data, "protect",
2822 total_stats.inbound.spd6.protect);
2823 rte_tel_data_add_dict_u64(spd6_data, "bypass",
2824 total_stats.inbound.spd6.bypass);
2825 rte_tel_data_add_dict_u64(spd6_data, "discard",
2826 total_stats.inbound.spd6.discard);
2827
2828 rte_tel_data_add_dict_container(data, "spd6", spd6_data, 0);
2829
2830exit:
2831 if (rc) {
2832 rte_tel_data_free(spd4_data);
2833 rte_tel_data_free(spd6_data);
2834 rte_tel_data_free(sad_data);
2835 }
2836 return rc;
2837}
2838
2839static int
2840handle_telemetry_cmd_ipsec_secgw_stats_routing(const char *cmd __rte_unused,
2841 const char *params, struct rte_tel_data *data)
2842{
2843 struct ipsec_core_statistics total_stats;
2844
2845 struct rte_tel_data *lpm4_data = rte_tel_data_alloc();
2846 struct rte_tel_data *lpm6_data = rte_tel_data_alloc();
2847 unsigned int coreid = UINT32_MAX;
2848 int rc = 0;
2849
2850
2851 if (!lpm4_data || !lpm6_data) {
2852 rc = -ENOMEM;
2853 goto exit;
2854 }
2855
2856
2857 rte_tel_data_start_dict(data);
2858 rte_tel_data_start_dict(lpm4_data);
2859 rte_tel_data_start_dict(lpm6_data);
2860
2861
2862 if (params) {
2863 coreid = (uint32_t)atoi(params);
2864 if (rte_lcore_is_enabled(coreid) == 0) {
2865 rc = -EINVAL;
2866 goto exit;
2867 }
2868 }
2869
2870 update_statistics(&total_stats, coreid);
2871
2872
2873 rte_tel_data_add_dict_u64(lpm4_data, "miss",
2874 total_stats.lpm4.miss);
2875
2876 rte_tel_data_add_dict_container(data, "IPv4 LPM", lpm4_data, 0);
2877
2878
2879 rte_tel_data_add_dict_u64(lpm6_data, "miss",
2880 total_stats.lpm6.miss);
2881
2882 rte_tel_data_add_dict_container(data, "IPv6 LPM", lpm6_data, 0);
2883
2884exit:
2885 if (rc) {
2886 rte_tel_data_free(lpm4_data);
2887 rte_tel_data_free(lpm6_data);
2888 }
2889 return rc;
2890}
2891
2892static void
2893ipsec_secgw_telemetry_init(void)
2894{
2895 rte_telemetry_register_cmd("/examples/ipsec-secgw/stats",
2896 handle_telemetry_cmd_ipsec_secgw_stats,
2897 "Returns global stats. "
2898 "Optional Parameters: int <logical core id>");
2899
2900 rte_telemetry_register_cmd("/examples/ipsec-secgw/stats/outbound",
2901 handle_telemetry_cmd_ipsec_secgw_stats_outbound,
2902 "Returns outbound global stats. "
2903 "Optional Parameters: int <logical core id>");
2904
2905 rte_telemetry_register_cmd("/examples/ipsec-secgw/stats/inbound",
2906 handle_telemetry_cmd_ipsec_secgw_stats_inbound,
2907 "Returns inbound global stats. "
2908 "Optional Parameters: int <logical core id>");
2909
2910 rte_telemetry_register_cmd("/examples/ipsec-secgw/stats/routing",
2911 handle_telemetry_cmd_ipsec_secgw_stats_routing,
2912 "Returns routing stats. "
2913 "Optional Parameters: int <logical core id>");
2914}
2915
2916int32_t
2917main(int32_t argc, char **argv)
2918{
2919 int32_t ret;
2920 uint32_t lcore_id, nb_txq, nb_rxq = 0;
2921 uint32_t cdev_id;
2922 uint32_t i;
2923 uint8_t socket_id;
2924 uint16_t portid, nb_crypto_qp, nb_ports = 0;
2925 uint64_t req_rx_offloads[RTE_MAX_ETHPORTS];
2926 uint64_t req_tx_offloads[RTE_MAX_ETHPORTS];
2927 struct eh_conf *eh_conf = NULL;
2928 size_t sess_sz;
2929
2930 nb_bufs_in_pool = 0;
2931
2932
2933 ret = rte_eal_init(argc, argv);
2934 if (ret < 0)
2935 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
2936 argc -= ret;
2937 argv += ret;
2938
2939 force_quit = false;
2940 signal(SIGINT, signal_handler);
2941 signal(SIGTERM, signal_handler);
2942
2943
2944 eh_conf = eh_conf_init();
2945 if (eh_conf == NULL)
2946 rte_exit(EXIT_FAILURE, "Failed to init event helper config");
2947
2948
2949 ret = parse_args(argc, argv, eh_conf);
2950 if (ret < 0)
2951 rte_exit(EXIT_FAILURE, "Invalid parameters\n");
2952
2953 ipsec_secgw_telemetry_init();
2954
2955
2956 if (parse_cfg_file(cfgfile) < 0) {
2957 printf("parsing file \"%s\" failed\n",
2958 optarg);
2959 print_usage(argv[0]);
2960 return -1;
2961 }
2962
2963 if ((unprotected_port_mask & enabled_port_mask) !=
2964 unprotected_port_mask)
2965 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
2966 unprotected_port_mask);
2967
2968 if (unprotected_port_mask && !nb_sa_in)
2969 rte_exit(EXIT_FAILURE, "Cannot use unprotected portmask without configured SA inbound\n");
2970
2971 if (check_poll_mode_params(eh_conf) < 0)
2972 rte_exit(EXIT_FAILURE, "check_poll_mode_params failed\n");
2973
2974 if (check_event_mode_params(eh_conf) < 0)
2975 rte_exit(EXIT_FAILURE, "check_event_mode_params failed\n");
2976
2977 ret = init_lcore_rx_queues();
2978 if (ret < 0)
2979 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
2980
2981 nb_lcores = rte_lcore_count();
2982
2983 sess_sz = max_session_size();
2984
2985
2986
2987
2988
2989 if (eh_conf->mode == EH_PKT_TRANSFER_MODE_EVENT)
2990 nb_crypto_qp = rte_eth_dev_count_avail();
2991 else
2992 nb_crypto_qp = 0;
2993
2994 nb_crypto_qp = cryptodevs_init(nb_crypto_qp);
2995
2996 if (nb_bufs_in_pool == 0) {
2997 RTE_ETH_FOREACH_DEV(portid) {
2998 if ((enabled_port_mask & (1 << portid)) == 0)
2999 continue;
3000 nb_ports++;
3001 nb_rxq += get_port_nb_rx_queues(portid);
3002 }
3003
3004 nb_txq = nb_lcores;
3005
3006 nb_bufs_in_pool = calculate_nb_mbufs(nb_ports, nb_crypto_qp,
3007 nb_rxq, nb_txq);
3008 }
3009
3010 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
3011 if (rte_lcore_is_enabled(lcore_id) == 0)
3012 continue;
3013
3014 if (numa_on)
3015 socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id);
3016 else
3017 socket_id = 0;
3018
3019 if (per_port_pool) {
3020 RTE_ETH_FOREACH_DEV(portid) {
3021 if ((enabled_port_mask & (1 << portid)) == 0)
3022 continue;
3023
3024 pool_init(&socket_ctx[socket_id], socket_id,
3025 portid, nb_bufs_in_pool);
3026 }
3027 } else {
3028 pool_init(&socket_ctx[socket_id], socket_id, 0,
3029 nb_bufs_in_pool);
3030 }
3031
3032 if (socket_ctx[socket_id].session_pool)
3033 continue;
3034
3035 session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
3036 session_priv_pool_init(&socket_ctx[socket_id], socket_id,
3037 sess_sz);
3038 }
3039 printf("Number of mbufs in packet pool %d\n", nb_bufs_in_pool);
3040
3041 RTE_ETH_FOREACH_DEV(portid) {
3042 if ((enabled_port_mask & (1 << portid)) == 0)
3043 continue;
3044
3045 sa_check_offloads(portid, &req_rx_offloads[portid],
3046 &req_tx_offloads[portid]);
3047 port_init(portid, req_rx_offloads[portid],
3048 req_tx_offloads[portid]);
3049 }
3050
3051
3052
3053
3054
3055
3056 eh_conf->eth_portmask = enabled_port_mask;
3057
3058
3059 ret = eh_devs_init(eh_conf);
3060 if (ret < 0)
3061 rte_exit(EXIT_FAILURE, "eh_devs_init failed, err=%d\n", ret);
3062
3063
3064 RTE_ETH_FOREACH_DEV(portid) {
3065 if ((enabled_port_mask & (1 << portid)) == 0)
3066 continue;
3067
3068 ret = rte_eth_dev_start(portid);
3069 if (ret < 0)
3070 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
3071 "err=%d, port=%d\n", ret, portid);
3072
3073
3074 create_default_ipsec_flow(portid, req_rx_offloads[portid]);
3075
3076
3077
3078
3079
3080
3081
3082 if (promiscuous_on) {
3083 ret = rte_eth_promiscuous_enable(portid);
3084 if (ret != 0)
3085 rte_exit(EXIT_FAILURE,
3086 "rte_eth_promiscuous_enable: err=%s, port=%d\n",
3087 rte_strerror(-ret), portid);
3088 }
3089
3090 rte_eth_dev_callback_register(portid, RTE_ETH_EVENT_INTR_RESET,
3091 ethdev_reset_event_callback, NULL);
3092
3093 rte_eth_dev_callback_register(portid,
3094 RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
3095 }
3096
3097
3098 if (frag_tbl_sz != 0) {
3099 ret = reassemble_init();
3100 if (ret != 0)
3101 rte_exit(EXIT_FAILURE, "failed at reassemble init");
3102 }
3103
3104
3105 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
3106 socket_id = rte_socket_id_by_idx(i);
3107 if ((socket_ctx[socket_id].session_pool != NULL) &&
3108 (socket_ctx[socket_id].sa_in == NULL) &&
3109 (socket_ctx[socket_id].sa_out == NULL)) {
3110 sa_init(&socket_ctx[socket_id], socket_id, lcore_conf);
3111 sp4_init(&socket_ctx[socket_id], socket_id);
3112 sp6_init(&socket_ctx[socket_id], socket_id);
3113 rt_init(&socket_ctx[socket_id], socket_id);
3114 }
3115 }
3116
3117 flow_init();
3118
3119
3120
3121
3122 if (!rte_security_dynfield_is_registered())
3123 goto skip_sec_ctx;
3124
3125 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
3126 for (i = 0; i < lcore_conf[lcore_id].nb_rx_queue; i++) {
3127 portid = lcore_conf[lcore_id].rx_queue_list[i].port_id;
3128 lcore_conf[lcore_id].rx_queue_list[i].sec_ctx =
3129 rte_eth_dev_get_sec_ctx(portid);
3130 }
3131 }
3132skip_sec_ctx:
3133
3134 check_all_ports_link_status(enabled_port_mask);
3135
3136 if (stats_interval > 0)
3137 rte_eal_alarm_set(stats_interval * US_PER_S,
3138 print_stats_cb, NULL);
3139 else
3140 RTE_LOG(INFO, IPSEC, "Stats display disabled\n");
3141
3142
3143 rte_eal_mp_remote_launch(ipsec_launch_one_lcore, eh_conf, CALL_MAIN);
3144 RTE_LCORE_FOREACH_WORKER(lcore_id) {
3145 if (rte_eal_wait_lcore(lcore_id) < 0)
3146 return -1;
3147 }
3148
3149
3150 ret = eh_devs_uninit(eh_conf);
3151 if (ret < 0)
3152 rte_exit(EXIT_FAILURE, "eh_devs_uninit failed, err=%d\n", ret);
3153
3154
3155 eh_conf_uninit(eh_conf);
3156
3157
3158 for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
3159 socket_id = rte_socket_id_by_idx(i);
3160 sessions_free(socket_ctx[socket_id].sa_in);
3161 sessions_free(socket_ctx[socket_id].sa_out);
3162 }
3163
3164 for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
3165 printf("Closing cryptodev %d...", cdev_id);
3166 rte_cryptodev_stop(cdev_id);
3167 rte_cryptodev_close(cdev_id);
3168 printf(" Done\n");
3169 }
3170
3171 flow_print_counters();
3172
3173 RTE_ETH_FOREACH_DEV(portid) {
3174 if ((enabled_port_mask & (1 << portid)) == 0)
3175 continue;
3176
3177 printf("Closing port %d...", portid);
3178 if (flow_info_tbl[portid].rx_def_flow) {
3179 struct rte_flow_error err;
3180
3181 ret = rte_flow_destroy(portid,
3182 flow_info_tbl[portid].rx_def_flow, &err);
3183 if (ret)
3184 RTE_LOG(ERR, IPSEC, "Failed to destroy flow "
3185 " for port %u, err msg: %s\n", portid,
3186 err.message);
3187 }
3188 ret = rte_eth_dev_stop(portid);
3189 if (ret != 0)
3190 RTE_LOG(ERR, IPSEC,
3191 "rte_eth_dev_stop: err=%s, port=%u\n",
3192 rte_strerror(-ret), portid);
3193
3194 rte_eth_dev_close(portid);
3195 printf(" Done\n");
3196 }
3197
3198
3199 rte_eal_cleanup();
3200 printf("Bye...\n");
3201
3202 return 0;
3203}
3204
