1
2
3
4
5#include <arpa/inet.h>
6#include <sys/socket.h>
7
8#include <rte_ethdev.h>
9#include <rte_ether.h>
10#include <rte_graph.h>
11#include <rte_graph_worker.h>
12#include <rte_ip.h>
13#include <rte_lpm.h>
14
15#include "rte_node_ip4_api.h"
16
17#include "node_private.h"
18
19#define IPV4_L3FWD_LPM_MAX_RULES 1024
20#define IPV4_L3FWD_LPM_NUMBER_TBL8S (1 << 8)
21
22
23struct ip4_lookup_node_main {
24 struct rte_lpm *lpm_tbl[RTE_MAX_NUMA_NODES];
25};
26
27struct ip4_lookup_node_ctx {
28
29 struct rte_lpm *lpm;
30
31 int mbuf_priv1_off;
32};
33
34int node_mbuf_priv1_dynfield_offset = -1;
35
36static struct ip4_lookup_node_main ip4_lookup_nm;
37
38#define IP4_LOOKUP_NODE_LPM(ctx) \
39 (((struct ip4_lookup_node_ctx *)ctx)->lpm)
40
41#define IP4_LOOKUP_NODE_PRIV1_OFF(ctx) \
42 (((struct ip4_lookup_node_ctx *)ctx)->mbuf_priv1_off)
43
44#if defined(__ARM_NEON)
45#include "ip4_lookup_neon.h"
46#elif defined(RTE_ARCH_X86)
47#include "ip4_lookup_sse.h"
48#endif
49
50static uint16_t
51ip4_lookup_node_process_scalar(struct rte_graph *graph, struct rte_node *node,
52 void **objs, uint16_t nb_objs)
53{
54 struct rte_lpm *lpm = IP4_LOOKUP_NODE_LPM(node->ctx);
55 const int dyn = IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx);
56 struct rte_ipv4_hdr *ipv4_hdr;
57 void **to_next, **from;
58 uint16_t last_spec = 0;
59 struct rte_mbuf *mbuf;
60 rte_edge_t next_index;
61 uint16_t held = 0;
62 uint32_t drop_nh;
63 int i, rc;
64
65
66 next_index = RTE_NODE_IP4_LOOKUP_NEXT_REWRITE;
67
68 drop_nh = ((uint32_t)RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP) << 16;
69 from = objs;
70
71
72 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs);
73 for (i = 0; i < nb_objs; i++) {
74 uint32_t next_hop;
75 uint16_t next;
76
77 mbuf = (struct rte_mbuf *)objs[i];
78
79
80 ipv4_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *,
81 sizeof(struct rte_ether_hdr));
82
83 node_mbuf_priv1(mbuf, dyn)->cksum = ipv4_hdr->hdr_checksum;
84 node_mbuf_priv1(mbuf, dyn)->ttl = ipv4_hdr->time_to_live;
85
86 rc = rte_lpm_lookup(lpm, rte_be_to_cpu_32(ipv4_hdr->dst_addr),
87 &next_hop);
88 next_hop = (rc == 0) ? next_hop : drop_nh;
89
90 node_mbuf_priv1(mbuf, dyn)->nh = (uint16_t)next_hop;
91 next_hop = next_hop >> 16;
92 next = (uint16_t)next_hop;
93
94 if (unlikely(next_index != next)) {
95
96 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
97 from += last_spec;
98 to_next += last_spec;
99 held += last_spec;
100 last_spec = 0;
101
102 rte_node_enqueue_x1(graph, node, next, from[0]);
103 from += 1;
104 } else {
105 last_spec += 1;
106 }
107 }
108
109
110 if (likely(last_spec == nb_objs)) {
111 rte_node_next_stream_move(graph, node, next_index);
112 return nb_objs;
113 }
114 held += last_spec;
115 rte_memcpy(to_next, from, last_spec * sizeof(from[0]));
116 rte_node_next_stream_put(graph, node, next_index, held);
117
118 return nb_objs;
119}
120
121int
122rte_node_ip4_route_add(uint32_t ip, uint8_t depth, uint16_t next_hop,
123 enum rte_node_ip4_lookup_next next_node)
124{
125 char abuf[INET6_ADDRSTRLEN];
126 struct in_addr in;
127 uint8_t socket;
128 uint32_t val;
129 int ret;
130
131 in.s_addr = htonl(ip);
132 inet_ntop(AF_INET, &in, abuf, sizeof(abuf));
133
134 val = ((next_node << 16) | next_hop) & ((1ull << 24) - 1);
135 node_dbg("ip4_lookup", "LPM: Adding route %s / %d nh (0x%x)", abuf,
136 depth, val);
137
138 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
139 if (!ip4_lookup_nm.lpm_tbl[socket])
140 continue;
141
142 ret = rte_lpm_add(ip4_lookup_nm.lpm_tbl[socket],
143 ip, depth, val);
144 if (ret < 0) {
145 node_err("ip4_lookup",
146 "Unable to add entry %s / %d nh (%x) to LPM table on sock %d, rc=%d\n",
147 abuf, depth, val, socket, ret);
148 return ret;
149 }
150 }
151
152 return 0;
153}
154
155static int
156setup_lpm(struct ip4_lookup_node_main *nm, int socket)
157{
158 struct rte_lpm_config config_ipv4;
159 char s[RTE_LPM_NAMESIZE];
160
161
162 if (nm->lpm_tbl[socket])
163 return 0;
164
165
166 config_ipv4.max_rules = IPV4_L3FWD_LPM_MAX_RULES;
167 config_ipv4.number_tbl8s = IPV4_L3FWD_LPM_NUMBER_TBL8S;
168 config_ipv4.flags = 0;
169 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socket);
170 nm->lpm_tbl[socket] = rte_lpm_create(s, socket, &config_ipv4);
171 if (nm->lpm_tbl[socket] == NULL)
172 return -rte_errno;
173
174 return 0;
175}
176
177static int
178ip4_lookup_node_init(const struct rte_graph *graph, struct rte_node *node)
179{
180 uint16_t socket, lcore_id;
181 static uint8_t init_once;
182 int rc;
183
184 RTE_SET_USED(graph);
185 RTE_BUILD_BUG_ON(sizeof(struct ip4_lookup_node_ctx) > RTE_NODE_CTX_SZ);
186
187 if (!init_once) {
188 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register(
189 &node_mbuf_priv1_dynfield_desc);
190 if (node_mbuf_priv1_dynfield_offset < 0)
191 return -rte_errno;
192
193
194 RTE_LCORE_FOREACH(lcore_id)
195 {
196 socket = rte_lcore_to_socket_id(lcore_id);
197 rc = setup_lpm(&ip4_lookup_nm, socket);
198 if (rc) {
199 node_err("ip4_lookup",
200 "Failed to setup lpm tbl for sock %u, rc=%d",
201 socket, rc);
202 return rc;
203 }
204 }
205 init_once = 1;
206 }
207
208
209 IP4_LOOKUP_NODE_LPM(node->ctx) = ip4_lookup_nm.lpm_tbl[graph->socket];
210 IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset;
211
212#if defined(__ARM_NEON) || defined(RTE_ARCH_X86)
213 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
214 node->process = ip4_lookup_node_process_vec;
215#endif
216
217 node_dbg("ip4_lookup", "Initialized ip4_lookup node");
218
219 return 0;
220}
221
222static struct rte_node_register ip4_lookup_node = {
223 .process = ip4_lookup_node_process_scalar,
224 .name = "ip4_lookup",
225
226 .init = ip4_lookup_node_init,
227
228 .nb_edges = RTE_NODE_IP4_LOOKUP_NEXT_MAX,
229 .next_nodes = {
230 [RTE_NODE_IP4_LOOKUP_NEXT_REWRITE] = "ip4_rewrite",
231 [RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP] = "pkt_drop",
232 },
233};
234
235RTE_NODE_REGISTER(ip4_lookup_node);
236
