1
2
3
4
5#include <sys/queue.h>
6#include <stdio.h>
7#include <errno.h>
8#include <stdint.h>
9#include <string.h>
10#include <assert.h>
11
12#include <rte_malloc.h>
13#include <rte_tailq.h>
14#include "base/i40e_prototype.h"
15#include "i40e_logs.h"
16#include "i40e_ethdev.h"
17#include "i40e_hash.h"
18
19#ifndef BIT
20#define BIT(n) (1UL << (n))
21#endif
22
23#ifndef BIT_ULL
24#define BIT_ULL(n) (1ULL << (n))
25#endif
26
27
28#define I40E_HASH_HDR_ETH 0x01ULL
29#define I40E_HASH_HDR_IPV4 0x10ULL
30#define I40E_HASH_HDR_IPV6 0x20ULL
31#define I40E_HASH_HDR_TCP 0x100ULL
32#define I40E_HASH_HDR_UDP 0x200ULL
33#define I40E_HASH_HDR_SCTP 0x400ULL
34#define I40E_HASH_HDR_ESP 0x10000ULL
35#define I40E_HASH_HDR_L2TPV3 0x20000ULL
36#define I40E_HASH_HDR_AH 0x40000ULL
37#define I40E_HASH_HDR_GTPC 0x100000ULL
38#define I40E_HASH_HDR_GTPU 0x200000ULL
39
40#define I40E_HASH_HDR_INNER_SHIFT 32
41#define I40E_HASH_HDR_IPV4_INNER (I40E_HASH_HDR_IPV4 << \
42 I40E_HASH_HDR_INNER_SHIFT)
43#define I40E_HASH_HDR_IPV6_INNER (I40E_HASH_HDR_IPV6 << \
44 I40E_HASH_HDR_INNER_SHIFT)
45
46
47#define I40E_PHINT_ETH I40E_HASH_HDR_ETH
48
49
50#define I40E_PHINT_IPV4 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV4)
51#define I40E_PHINT_IPV4_TCP (I40E_PHINT_IPV4 | I40E_HASH_HDR_TCP)
52#define I40E_PHINT_IPV4_UDP (I40E_PHINT_IPV4 | I40E_HASH_HDR_UDP)
53#define I40E_PHINT_IPV4_SCTP (I40E_PHINT_IPV4 | I40E_HASH_HDR_SCTP)
54
55
56#define I40E_PHINT_IPV6 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV6)
57#define I40E_PHINT_IPV6_TCP (I40E_PHINT_IPV6 | I40E_HASH_HDR_TCP)
58#define I40E_PHINT_IPV6_UDP (I40E_PHINT_IPV6 | I40E_HASH_HDR_UDP)
59#define I40E_PHINT_IPV6_SCTP (I40E_PHINT_IPV6 | I40E_HASH_HDR_SCTP)
60
61
62#define I40E_PHINT_IPV4_ESP (I40E_PHINT_IPV4 | I40E_HASH_HDR_ESP)
63#define I40E_PHINT_IPV6_ESP (I40E_PHINT_IPV6 | I40E_HASH_HDR_ESP)
64#define I40E_PHINT_IPV4_UDP_ESP (I40E_PHINT_IPV4_UDP | \
65 I40E_HASH_HDR_ESP)
66#define I40E_PHINT_IPV6_UDP_ESP (I40E_PHINT_IPV6_UDP | \
67 I40E_HASH_HDR_ESP)
68
69
70#define I40E_PHINT_IPV4_GTPC (I40E_PHINT_IPV4_UDP | \
71 I40E_HASH_HDR_GTPC)
72#define I40E_PHINT_IPV6_GTPC (I40E_PHINT_IPV6_UDP | \
73 I40E_HASH_HDR_GTPC)
74
75
76#define I40E_PHINT_IPV4_GTPU (I40E_PHINT_IPV4_UDP | \
77 I40E_HASH_HDR_GTPU)
78#define I40E_PHINT_IPV4_GTPU_IPV4 (I40E_PHINT_IPV4_GTPU | \
79 I40E_HASH_HDR_IPV4_INNER)
80#define I40E_PHINT_IPV4_GTPU_IPV6 (I40E_PHINT_IPV4_GTPU | \
81 I40E_HASH_HDR_IPV6_INNER)
82#define I40E_PHINT_IPV6_GTPU (I40E_PHINT_IPV6_UDP | \
83 I40E_HASH_HDR_GTPU)
84#define I40E_PHINT_IPV6_GTPU_IPV4 (I40E_PHINT_IPV6_GTPU | \
85 I40E_HASH_HDR_IPV4_INNER)
86#define I40E_PHINT_IPV6_GTPU_IPV6 (I40E_PHINT_IPV6_GTPU | \
87 I40E_HASH_HDR_IPV6_INNER)
88
89
90#define I40E_PHINT_IPV4_L2TPV3 (I40E_PHINT_IPV4 | I40E_HASH_HDR_L2TPV3)
91#define I40E_PHINT_IPV6_L2TPV3 (I40E_PHINT_IPV6 | I40E_HASH_HDR_L2TPV3)
92
93
94#define I40E_PHINT_IPV4_AH (I40E_PHINT_IPV4 | I40E_HASH_HDR_AH)
95#define I40E_PHINT_IPV6_AH (I40E_PHINT_IPV6 | I40E_HASH_HDR_AH)
96
97
98struct i40e_hash_map_rss_inset {
99 uint64_t rss_type;
100 uint64_t inset;
101};
102
103const struct i40e_hash_map_rss_inset i40e_hash_rss_inset[] = {
104
105 { ETH_RSS_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
106 { ETH_RSS_FRAG_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
107
108 { ETH_RSS_NONFRAG_IPV4_OTHER,
109 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
110
111 { ETH_RSS_NONFRAG_IPV4_TCP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
112 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
113
114 { ETH_RSS_NONFRAG_IPV4_UDP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
115 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
116
117 { ETH_RSS_NONFRAG_IPV4_SCTP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
118 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
119
120
121 { ETH_RSS_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
122 { ETH_RSS_FRAG_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
123
124 { ETH_RSS_NONFRAG_IPV6_OTHER,
125 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
126
127 { ETH_RSS_NONFRAG_IPV6_TCP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
128 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
129
130 { ETH_RSS_NONFRAG_IPV6_UDP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
131 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
132
133 { ETH_RSS_NONFRAG_IPV6_SCTP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
134 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
135
136
137 { ETH_RSS_PORT, I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
138
139
140 { ETH_RSS_L2_PAYLOAD, I40E_INSET_LAST_ETHER_TYPE },
141 { ETH_RSS_ETH, I40E_INSET_DMAC | I40E_INSET_SMAC },
142
143
144 { ETH_RSS_S_VLAN, I40E_INSET_VLAN_OUTER },
145 { ETH_RSS_C_VLAN, I40E_INSET_VLAN_INNER },
146};
147
148#define I40E_HASH_VOID_NEXT_ALLOW BIT_ULL(RTE_FLOW_ITEM_TYPE_ETH)
149
150#define I40E_HASH_ETH_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
151 BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6) | \
152 BIT_ULL(RTE_FLOW_ITEM_TYPE_VLAN))
153
154#define I40E_HASH_IP_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_TCP) | \
155 BIT_ULL(RTE_FLOW_ITEM_TYPE_UDP) | \
156 BIT_ULL(RTE_FLOW_ITEM_TYPE_SCTP) | \
157 BIT_ULL(RTE_FLOW_ITEM_TYPE_ESP) | \
158 BIT_ULL(RTE_FLOW_ITEM_TYPE_L2TPV3OIP) |\
159 BIT_ULL(RTE_FLOW_ITEM_TYPE_AH))
160
161#define I40E_HASH_UDP_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPU) | \
162 BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPC))
163
164#define I40E_HASH_GTPU_NEXT_ALLOW (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
165 BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6))
166
167static const uint64_t pattern_next_allow_items[] = {
168 [RTE_FLOW_ITEM_TYPE_VOID] = I40E_HASH_VOID_NEXT_ALLOW,
169 [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_ETH_NEXT_ALLOW,
170 [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_IP_NEXT_ALLOW,
171 [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_IP_NEXT_ALLOW,
172 [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_UDP_NEXT_ALLOW,
173 [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_GTPU_NEXT_ALLOW,
174};
175
176static const uint64_t pattern_item_header[] = {
177 [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_HDR_ETH,
178 [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_HDR_IPV4,
179 [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_HDR_IPV6,
180 [RTE_FLOW_ITEM_TYPE_TCP] = I40E_HASH_HDR_TCP,
181 [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_HDR_UDP,
182 [RTE_FLOW_ITEM_TYPE_SCTP] = I40E_HASH_HDR_SCTP,
183 [RTE_FLOW_ITEM_TYPE_ESP] = I40E_HASH_HDR_ESP,
184 [RTE_FLOW_ITEM_TYPE_GTPC] = I40E_HASH_HDR_GTPC,
185 [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_HDR_GTPU,
186 [RTE_FLOW_ITEM_TYPE_L2TPV3OIP] = I40E_HASH_HDR_L2TPV3,
187 [RTE_FLOW_ITEM_TYPE_AH] = I40E_HASH_HDR_AH,
188};
189
190
191struct i40e_hash_match_pattern {
192 uint64_t pattern_type;
193 uint64_t rss_mask;
194 bool custom_pctype_flag;
195 uint8_t pctype;
196};
197
198#define I40E_HASH_MAP_PATTERN(pattern, rss_mask, pctype) { \
199 pattern, rss_mask, false, pctype }
200
201#define I40E_HASH_MAP_CUS_PATTERN(pattern, rss_mask, cus_pctype) { \
202 pattern, rss_mask, true, cus_pctype }
203
204#define I40E_HASH_L2_RSS_MASK (ETH_RSS_ETH | ETH_RSS_L2_SRC_ONLY | \
205 ETH_RSS_L2_DST_ONLY)
206
207#define I40E_HASH_L23_RSS_MASK (I40E_HASH_L2_RSS_MASK | \
208 ETH_RSS_VLAN | \
209 ETH_RSS_L3_SRC_ONLY | \
210 ETH_RSS_L3_DST_ONLY)
211
212#define I40E_HASH_IPV4_L23_RSS_MASK (ETH_RSS_IPV4 | I40E_HASH_L23_RSS_MASK)
213#define I40E_HASH_IPV6_L23_RSS_MASK (ETH_RSS_IPV6 | I40E_HASH_L23_RSS_MASK)
214
215#define I40E_HASH_L234_RSS_MASK (I40E_HASH_L23_RSS_MASK | \
216 ETH_RSS_PORT | ETH_RSS_L4_SRC_ONLY | \
217 ETH_RSS_L4_DST_ONLY)
218
219#define I40E_HASH_IPV4_L234_RSS_MASK (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV4)
220#define I40E_HASH_IPV6_L234_RSS_MASK (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV6)
221
222#define I40E_HASH_L4_TYPES (ETH_RSS_NONFRAG_IPV4_TCP | \
223 ETH_RSS_NONFRAG_IPV4_UDP | \
224 ETH_RSS_NONFRAG_IPV4_SCTP | \
225 ETH_RSS_NONFRAG_IPV6_TCP | \
226 ETH_RSS_NONFRAG_IPV6_UDP | \
227 ETH_RSS_NONFRAG_IPV6_SCTP)
228
229
230
231
232static const struct i40e_hash_match_pattern match_patterns[] = {
233
234 I40E_HASH_MAP_PATTERN(I40E_PHINT_ETH,
235 ETH_RSS_L2_PAYLOAD | I40E_HASH_L2_RSS_MASK,
236 I40E_FILTER_PCTYPE_L2_PAYLOAD),
237
238
239 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
240 ETH_RSS_FRAG_IPV4 | I40E_HASH_IPV4_L23_RSS_MASK,
241 I40E_FILTER_PCTYPE_FRAG_IPV4),
242
243 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
244 ETH_RSS_NONFRAG_IPV4_OTHER |
245 I40E_HASH_IPV4_L23_RSS_MASK,
246 I40E_FILTER_PCTYPE_NONF_IPV4_OTHER),
247
248 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_TCP,
249 ETH_RSS_NONFRAG_IPV4_TCP |
250 I40E_HASH_IPV4_L234_RSS_MASK,
251 I40E_FILTER_PCTYPE_NONF_IPV4_TCP),
252
253 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_UDP,
254 ETH_RSS_NONFRAG_IPV4_UDP |
255 I40E_HASH_IPV4_L234_RSS_MASK,
256 I40E_FILTER_PCTYPE_NONF_IPV4_UDP),
257
258 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_SCTP,
259 ETH_RSS_NONFRAG_IPV4_SCTP |
260 I40E_HASH_IPV4_L234_RSS_MASK,
261 I40E_FILTER_PCTYPE_NONF_IPV4_SCTP),
262
263
264 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
265 ETH_RSS_FRAG_IPV6 | I40E_HASH_IPV6_L23_RSS_MASK,
266 I40E_FILTER_PCTYPE_FRAG_IPV6),
267
268 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
269 ETH_RSS_NONFRAG_IPV6_OTHER |
270 I40E_HASH_IPV6_L23_RSS_MASK,
271 I40E_FILTER_PCTYPE_NONF_IPV6_OTHER),
272
273 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_TCP,
274 ETH_RSS_NONFRAG_IPV6_TCP |
275 I40E_HASH_IPV6_L234_RSS_MASK,
276 I40E_FILTER_PCTYPE_NONF_IPV6_TCP),
277
278 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_UDP,
279 ETH_RSS_NONFRAG_IPV6_UDP |
280 I40E_HASH_IPV6_L234_RSS_MASK,
281 I40E_FILTER_PCTYPE_NONF_IPV6_UDP),
282
283 I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_SCTP,
284 ETH_RSS_NONFRAG_IPV6_SCTP |
285 I40E_HASH_IPV6_L234_RSS_MASK,
286 I40E_FILTER_PCTYPE_NONF_IPV6_SCTP),
287
288
289 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_ESP,
290 ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4),
291 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_ESP,
292 ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6),
293 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_UDP_ESP,
294 ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4_UDP),
295 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_UDP_ESP,
296 ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6_UDP),
297
298
299 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPC,
300 I40E_HASH_IPV4_L234_RSS_MASK,
301 I40E_CUSTOMIZED_GTPC),
302 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPC,
303 I40E_HASH_IPV6_L234_RSS_MASK,
304 I40E_CUSTOMIZED_GTPC),
305
306
307 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU,
308 I40E_HASH_IPV4_L234_RSS_MASK,
309 I40E_CUSTOMIZED_GTPU),
310 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV4,
311 ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
312 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV6,
313 ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
314 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU,
315 I40E_HASH_IPV6_L234_RSS_MASK,
316 I40E_CUSTOMIZED_GTPU),
317 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV4,
318 ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
319 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV6,
320 ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
321
322
323 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_L2TPV3,
324 ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV4_L2TPV3),
325 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_L2TPV3,
326 ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV6_L2TPV3),
327
328
329 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_AH, ETH_RSS_AH,
330 I40E_CUSTOMIZED_AH_IPV4),
331 I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_AH, ETH_RSS_AH,
332 I40E_CUSTOMIZED_AH_IPV6),
333};
334
335static int
336i40e_hash_get_pattern_type(const struct rte_flow_item pattern[],
337 uint64_t *pattern_types,
338 struct rte_flow_error *error)
339{
340 const char *message = "Pattern not supported";
341 enum rte_flow_item_type prev_item_type = RTE_FLOW_ITEM_TYPE_VOID;
342 enum rte_flow_item_type last_item_type = prev_item_type;
343 uint64_t item_hdr, pattern_hdrs = 0;
344 bool inner_flag = false;
345 int vlan_count = 0;
346
347 for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
348 if (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
349 continue;
350
351 if (pattern->mask || pattern->spec || pattern->last) {
352 message = "Header info should not be specified";
353 goto not_sup;
354 }
355
356
357 if (prev_item_type >= (enum rte_flow_item_type)
358 RTE_DIM(pattern_next_allow_items) ||
359 !(pattern_next_allow_items[prev_item_type] &
360 BIT_ULL(pattern->type)))
361 goto not_sup;
362
363
364
365
366
367 last_item_type = pattern->type;
368 if (last_item_type == RTE_FLOW_ITEM_TYPE_VLAN) {
369 if (vlan_count >= 2)
370 goto not_sup;
371 vlan_count++;
372 continue;
373 }
374
375 prev_item_type = last_item_type;
376 assert(last_item_type < (enum rte_flow_item_type)
377 RTE_DIM(pattern_item_header));
378 item_hdr = pattern_item_header[last_item_type];
379 assert(item_hdr);
380
381 if (inner_flag) {
382 item_hdr <<= I40E_HASH_HDR_INNER_SHIFT;
383
384
385 if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU)
386 goto not_sup;
387 } else {
388 if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU) {
389 inner_flag = true;
390 vlan_count = 0;
391 }
392 }
393
394 if (item_hdr & pattern_hdrs)
395 goto not_sup;
396
397 pattern_hdrs |= item_hdr;
398 }
399
400 if (pattern_hdrs && last_item_type != RTE_FLOW_ITEM_TYPE_VLAN) {
401 *pattern_types = pattern_hdrs;
402 return 0;
403 }
404
405not_sup:
406 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
407 pattern, message);
408}
409
410static uint64_t
411i40e_hash_get_x722_ext_pctypes(uint8_t match_pctype)
412{
413 uint64_t pctypes = 0;
414
415 switch (match_pctype) {
416 case I40E_FILTER_PCTYPE_NONF_IPV4_TCP:
417 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
418 break;
419
420 case I40E_FILTER_PCTYPE_NONF_IPV4_UDP:
421 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
422 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
423 break;
424
425 case I40E_FILTER_PCTYPE_NONF_IPV6_TCP:
426 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
427 break;
428
429 case I40E_FILTER_PCTYPE_NONF_IPV6_UDP:
430 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
431 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
432 break;
433 }
434
435 return pctypes;
436}
437
438static int
439i40e_hash_translate_gtp_inset(struct i40e_rte_flow_rss_conf *rss_conf,
440 struct rte_flow_error *error)
441{
442 if (rss_conf->inset &
443 (I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC |
444 I40E_INSET_DST_PORT | I40E_INSET_SRC_PORT))
445 return rte_flow_error_set(error, ENOTSUP,
446 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
447 NULL,
448 "Only support external destination IP");
449
450 if (rss_conf->inset & I40E_INSET_IPV4_DST)
451 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV4_DST) |
452 I40E_INSET_TUNNEL_IPV4_DST;
453
454 if (rss_conf->inset & I40E_INSET_IPV6_DST)
455 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV6_DST) |
456 I40E_INSET_TUNNEL_IPV6_DST;
457
458 return 0;
459}
460
461static int
462i40e_hash_get_pctypes(const struct rte_eth_dev *dev,
463 const struct i40e_hash_match_pattern *match,
464 struct i40e_rte_flow_rss_conf *rss_conf,
465 struct rte_flow_error *error)
466{
467 if (match->custom_pctype_flag) {
468 struct i40e_pf *pf;
469 struct i40e_customized_pctype *custom_type;
470
471 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
472 custom_type = i40e_find_customized_pctype(pf, match->pctype);
473 if (!custom_type || !custom_type->valid)
474 return rte_flow_error_set(error, ENOTSUP,
475 RTE_FLOW_ERROR_TYPE_ITEM,
476 NULL, "PCTYPE not supported");
477
478 rss_conf->config_pctypes |= BIT_ULL(custom_type->pctype);
479
480 if (match->pctype == I40E_CUSTOMIZED_GTPU ||
481 match->pctype == I40E_CUSTOMIZED_GTPC)
482 return i40e_hash_translate_gtp_inset(rss_conf, error);
483 } else {
484 struct i40e_hw *hw =
485 I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
486 uint64_t types;
487
488 rss_conf->config_pctypes |= BIT_ULL(match->pctype);
489 if (hw->mac.type == I40E_MAC_X722) {
490 types = i40e_hash_get_x722_ext_pctypes(match->pctype);
491 rss_conf->config_pctypes |= types;
492 }
493 }
494
495 return 0;
496}
497
498static int
499i40e_hash_get_pattern_pctypes(const struct rte_eth_dev *dev,
500 const struct rte_flow_item pattern[],
501 const struct rte_flow_action_rss *rss_act,
502 struct i40e_rte_flow_rss_conf *rss_conf,
503 struct rte_flow_error *error)
504{
505 uint64_t pattern_types = 0;
506 bool match_flag = false;
507 int i, ret;
508
509 ret = i40e_hash_get_pattern_type(pattern, &pattern_types, error);
510 if (ret)
511 return ret;
512
513 for (i = 0; i < (int)RTE_DIM(match_patterns); i++) {
514 const struct i40e_hash_match_pattern *match =
515 &match_patterns[i];
516
517
518
519
520
521
522 if (pattern_types != match->pattern_type) {
523 if (match_flag)
524 break;
525 continue;
526 }
527 match_flag = true;
528
529
530 if (!(rss_act->types & ~match->rss_mask)) {
531 ret = i40e_hash_get_pctypes(dev, match,
532 rss_conf, error);
533 if (ret)
534 return ret;
535 }
536 }
537
538 if (rss_conf->config_pctypes)
539 return 0;
540
541 if (match_flag)
542 return rte_flow_error_set(error, ENOTSUP,
543 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
544 NULL, "RSS types not supported");
545
546 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
547 NULL, "Pattern not supported");
548}
549
550static uint64_t
551i40e_hash_get_inset(uint64_t rss_types)
552{
553 uint64_t mask, inset = 0;
554 int i;
555
556 for (i = 0; i < (int)RTE_DIM(i40e_hash_rss_inset); i++) {
557 if (rss_types & i40e_hash_rss_inset[i].rss_type)
558 inset |= i40e_hash_rss_inset[i].inset;
559 }
560
561 if (!inset)
562 return 0;
563
564
565
566
567 mask = rss_types & (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY);
568 if (mask == ETH_RSS_L2_SRC_ONLY)
569 inset &= ~I40E_INSET_DMAC;
570 else if (mask == ETH_RSS_L2_DST_ONLY)
571 inset &= ~I40E_INSET_SMAC;
572
573 mask = rss_types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
574 if (mask == ETH_RSS_L3_SRC_ONLY)
575 inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST);
576 else if (mask == ETH_RSS_L3_DST_ONLY)
577 inset &= ~(I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
578
579 mask = rss_types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
580 if (mask == ETH_RSS_L4_SRC_ONLY)
581 inset &= ~I40E_INSET_DST_PORT;
582 else if (mask == ETH_RSS_L4_DST_ONLY)
583 inset &= ~I40E_INSET_SRC_PORT;
584
585 if (rss_types & I40E_HASH_L4_TYPES) {
586 uint64_t l3_mask = rss_types &
587 (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
588 uint64_t l4_mask = rss_types &
589 (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
590
591 if (l3_mask && !l4_mask)
592 inset &= ~(I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT);
593 else if (!l3_mask && l4_mask)
594 inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST |
595 I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
596 }
597
598 return inset;
599}
600
601static int
602i40e_hash_config_func(struct i40e_hw *hw, enum rte_eth_hash_function func)
603{
604 struct i40e_pf *pf;
605 uint32_t reg;
606 uint8_t symmetric = 0;
607
608 reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
609
610 if (func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
611 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK))
612 goto set_symmetric;
613
614 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
615 } else {
616 if (func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
617 symmetric = 1;
618
619 if (reg & I40E_GLQF_CTL_HTOEP_MASK)
620 goto set_symmetric;
621
622 reg |= I40E_GLQF_CTL_HTOEP_MASK;
623 }
624
625 pf = &((struct i40e_adapter *)hw->back)->pf;
626 if (pf->support_multi_driver) {
627 PMD_DRV_LOG(ERR,
628 "Modify hash function is not permitted when multi-driver enabled");
629 return -EPERM;
630 }
631
632 PMD_DRV_LOG(INFO, "NIC hash function is setting to %d", func);
633 i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg);
634 I40E_WRITE_FLUSH(hw);
635
636set_symmetric:
637 i40e_set_symmetric_hash_enable_per_port(hw, symmetric);
638 return 0;
639}
640
641static int
642i40e_hash_config_pctype_symmetric(struct i40e_hw *hw,
643 uint32_t pctype,
644 bool symmetric)
645{
646 struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
647 uint32_t reg;
648
649
650 if (hw->mac.type == I40E_MAC_X722)
651 pctype = i40e_read_rx_ctl(hw, I40E_GLQF_FD_PCTYPES(pctype));
652
653 reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
654 if (symmetric) {
655 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
656 return 0;
657 reg |= I40E_GLQF_HSYM_SYMH_ENA_MASK;
658 } else {
659 if (!(reg & I40E_GLQF_HSYM_SYMH_ENA_MASK))
660 return 0;
661 reg &= ~I40E_GLQF_HSYM_SYMH_ENA_MASK;
662 }
663
664 if (pf->support_multi_driver) {
665 PMD_DRV_LOG(ERR,
666 "Enable/Disable symmetric hash is not permitted when multi-driver enabled");
667 return -EPERM;
668 }
669
670 i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg);
671 I40E_WRITE_FLUSH(hw);
672 return 0;
673}
674
675static void
676i40e_hash_enable_pctype(struct i40e_hw *hw,
677 uint32_t pctype, bool enable)
678{
679 uint32_t reg, reg_val, mask;
680
681 if (pctype < 32) {
682 mask = BIT(pctype);
683 reg = I40E_PFQF_HENA(0);
684 } else {
685 mask = BIT(pctype - 32);
686 reg = I40E_PFQF_HENA(1);
687 }
688
689 reg_val = i40e_read_rx_ctl(hw, reg);
690
691 if (enable) {
692 if (reg_val & mask)
693 return;
694
695 reg_val |= mask;
696 } else {
697 if (!(reg_val & mask))
698 return;
699
700 reg_val &= ~mask;
701 }
702
703 i40e_write_rx_ctl(hw, reg, reg_val);
704 I40E_WRITE_FLUSH(hw);
705}
706
707static int
708i40e_hash_config_pctype(struct i40e_hw *hw,
709 struct i40e_rte_flow_rss_conf *rss_conf,
710 uint32_t pctype)
711{
712 uint64_t rss_types = rss_conf->conf.types;
713 int ret;
714
715 if (rss_types == 0) {
716 i40e_hash_enable_pctype(hw, pctype, false);
717 return 0;
718 }
719
720 if (rss_conf->inset) {
721 ret = i40e_set_hash_inset(hw, rss_conf->inset, pctype, false);
722 if (ret)
723 return ret;
724 }
725
726 i40e_hash_enable_pctype(hw, pctype, true);
727 return 0;
728}
729
730static int
731i40e_hash_config_region(struct i40e_pf *pf,
732 const struct i40e_rte_flow_rss_conf *rss_conf)
733{
734 struct i40e_hw *hw = &pf->adapter->hw;
735 struct rte_eth_dev *dev = pf->adapter->eth_dev;
736 struct i40e_queue_region_info *regions = pf->queue_region.region;
737 uint32_t num = pf->queue_region.queue_region_number;
738 uint32_t i, region_id_mask = 0;
739
740
741 RTE_BUILD_BUG_ON(I40E_REGION_MAX_INDEX > 31);
742
743
744 for (i = 0; i < num; i++) {
745 if (rss_conf->region_queue_start ==
746 regions[i].queue_start_index &&
747 rss_conf->region_queue_num == regions[i].queue_num) {
748 uint32_t j;
749
750 for (j = 0; j < regions[i].user_priority_num; j++) {
751 if (regions[i].user_priority[j] ==
752 rss_conf->region_priority)
753 return 0;
754 }
755
756 if (j >= I40E_MAX_USER_PRIORITY) {
757 PMD_DRV_LOG(ERR,
758 "Priority number exceed the maximum %d",
759 I40E_MAX_USER_PRIORITY);
760 return -ENOSPC;
761 }
762
763 regions[i].user_priority[j] = rss_conf->region_priority;
764 regions[i].user_priority_num++;
765 return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
766 }
767
768 region_id_mask |= BIT(regions[i].region_id);
769 }
770
771 if (num > I40E_REGION_MAX_INDEX) {
772 PMD_DRV_LOG(ERR, "Queue region resource used up");
773 return -ENOSPC;
774 }
775
776
777
778 pf->queue_region.queue_region_number++;
779 memset(®ions[num], 0, sizeof(regions[0]));
780
781 regions[num].region_id = rte_bsf32(~region_id_mask);
782 regions[num].queue_num = rss_conf->region_queue_num;
783 regions[num].queue_start_index = rss_conf->region_queue_start;
784 regions[num].user_priority[0] = rss_conf->region_priority;
785 regions[num].user_priority_num = 1;
786
787 return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
788}
789
790static int
791i40e_hash_config(struct i40e_pf *pf,
792 struct i40e_rte_flow_rss_conf *rss_conf)
793{
794 struct rte_flow_action_rss *rss_info = &rss_conf->conf;
795 struct i40e_hw *hw = &pf->adapter->hw;
796 uint64_t pctypes;
797 int ret;
798
799 if (rss_info->func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
800 ret = i40e_hash_config_func(hw, rss_info->func);
801 if (ret)
802 return ret;
803
804 if (rss_info->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
805 rss_conf->misc_reset_flags |=
806 I40E_HASH_FLOW_RESET_FLAG_FUNC;
807 }
808
809 if (rss_conf->region_queue_num > 0) {
810 ret = i40e_hash_config_region(pf, rss_conf);
811 if (ret)
812 return ret;
813
814 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
815 }
816
817 if (rss_info->key_len > 0) {
818 ret = i40e_set_rss_key(pf->main_vsi, rss_conf->key,
819 rss_info->key_len);
820 if (ret)
821 return ret;
822
823 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_KEY;
824 }
825
826
827 if (rss_info->queue_num > 0) {
828 uint8_t lut[ETH_RSS_RETA_SIZE_512];
829 uint32_t i, j = 0;
830
831 for (i = 0; i < hw->func_caps.rss_table_size; i++) {
832 lut[i] = (uint8_t)rss_info->queue[j];
833 j = (j == rss_info->queue_num - 1) ? 0 : (j + 1);
834 }
835
836 ret = i40e_set_rss_lut(pf->main_vsi, lut, (uint16_t)i);
837 if (ret)
838 return ret;
839
840 pf->hash_enabled_queues = 0;
841 for (i = 0; i < rss_info->queue_num; i++)
842 pf->hash_enabled_queues |= BIT_ULL(lut[i]);
843
844 pf->adapter->rss_reta_updated = 0;
845 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_QUEUE;
846 }
847
848
849
850
851 pctypes = rss_conf->config_pctypes;
852 if (!pctypes)
853 return 0;
854
855
856
857
858 if (!pf->hash_filter_enabled) {
859 i40e_pf_disable_rss(pf);
860 pf->hash_filter_enabled = true;
861 }
862
863 do {
864 uint32_t idx = rte_bsf64(pctypes);
865 uint64_t bit = BIT_ULL(idx);
866
867 if (rss_conf->symmetric_enable) {
868 ret = i40e_hash_config_pctype_symmetric(hw, idx, true);
869 if (ret)
870 return ret;
871
872 rss_conf->reset_symmetric_pctypes |= bit;
873 }
874
875 ret = i40e_hash_config_pctype(hw, rss_conf, idx);
876 if (ret)
877 return ret;
878
879 rss_conf->reset_config_pctypes |= bit;
880 pctypes &= ~bit;
881 } while (pctypes);
882
883 return 0;
884}
885
886static void
887i40e_hash_parse_key(const struct rte_flow_action_rss *rss_act,
888 struct i40e_rte_flow_rss_conf *rss_conf)
889{
890 const uint8_t *key = rss_act->key;
891
892 if (!key || rss_act->key_len != sizeof(rss_conf->key)) {
893 const uint32_t rss_key_default[] = {0x6b793944,
894 0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
895 0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
896 0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
897
898 if (rss_act->key_len != sizeof(rss_conf->key))
899 PMD_DRV_LOG(WARNING,
900 "RSS key length invalid, must be %u bytes, now set key to default",
901 (uint32_t)sizeof(rss_conf->key));
902
903 memcpy(rss_conf->key, rss_key_default, sizeof(rss_conf->key));
904 } else {
905 memcpy(rss_conf->key, key, sizeof(rss_conf->key));
906 }
907
908 rss_conf->conf.key = rss_conf->key;
909 rss_conf->conf.key_len = sizeof(rss_conf->key);
910}
911
912static int
913i40e_hash_parse_queues(const struct rte_eth_dev *dev,
914 const struct rte_flow_action_rss *rss_act,
915 struct i40e_rte_flow_rss_conf *rss_conf,
916 struct rte_flow_error *error)
917{
918 struct i40e_pf *pf;
919 struct i40e_hw *hw;
920 uint16_t i;
921 int max_queue;
922
923 hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
924 if (!rss_act->queue_num ||
925 rss_act->queue_num > hw->func_caps.rss_table_size)
926 return rte_flow_error_set(error, EINVAL,
927 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
928 NULL, "Invalid RSS queue number");
929
930 if (rss_act->key_len)
931 PMD_DRV_LOG(WARNING,
932 "RSS key is ignored when queues specified");
933
934 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
935 if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
936 max_queue = i40e_pf_calc_configured_queues_num(pf);
937 else
938 max_queue = pf->dev_data->nb_rx_queues;
939
940 max_queue = RTE_MIN(max_queue, I40E_MAX_Q_PER_TC);
941
942 for (i = 0; i < rss_act->queue_num; i++) {
943 if ((int)rss_act->queue[i] >= max_queue)
944 break;
945 }
946
947 if (i < rss_act->queue_num)
948 return rte_flow_error_set(error, EINVAL,
949 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
950 NULL, "Invalid RSS queues");
951
952 memcpy(rss_conf->queue, rss_act->queue,
953 rss_act->queue_num * sizeof(rss_conf->queue[0]));
954 rss_conf->conf.queue = rss_conf->queue;
955 rss_conf->conf.queue_num = rss_act->queue_num;
956 return 0;
957}
958
959static int
960i40e_hash_parse_queue_region(const struct rte_eth_dev *dev,
961 const struct rte_flow_item pattern[],
962 const struct rte_flow_action_rss *rss_act,
963 struct i40e_rte_flow_rss_conf *rss_conf,
964 struct rte_flow_error *error)
965{
966 struct i40e_pf *pf;
967 const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
968 uint64_t hash_queues;
969 uint32_t i;
970
971 if (pattern[1].type != RTE_FLOW_ITEM_TYPE_END)
972 return rte_flow_error_set(error, ENOTSUP,
973 RTE_FLOW_ERROR_TYPE_ITEM_NUM,
974 &pattern[1],
975 "Pattern not supported.");
976
977 vlan_spec = pattern->spec;
978 vlan_mask = pattern->mask;
979 if (!vlan_spec || !vlan_mask ||
980 (rte_be_to_cpu_16(vlan_mask->tci) >> 13) != 7)
981 return rte_flow_error_set(error, EINVAL,
982 RTE_FLOW_ERROR_TYPE_ITEM, pattern,
983 "Pattern error.");
984
985 if (!rss_act->queue)
986 return rte_flow_error_set(error, EINVAL,
987 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
988 NULL, "Queues not specified");
989
990 if (rss_act->key_len)
991 PMD_DRV_LOG(WARNING,
992 "RSS key is ignored when configure queue region");
993
994
995 RTE_BUILD_BUG_ON(I40E_MAX_Q_PER_TC > 64);
996
997 if (!rss_act->queue_num ||
998 !rte_is_power_of_2(rss_act->queue_num) ||
999 rss_act->queue_num + rss_act->queue[0] > I40E_MAX_Q_PER_TC)
1000 return rte_flow_error_set(error, EINVAL,
1001 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1002 NULL, "Queue number error");
1003
1004 for (i = 1; i < rss_act->queue_num; i++) {
1005 if (rss_act->queue[i - 1] + 1 != rss_act->queue[i])
1006 break;
1007 }
1008
1009 if (i < rss_act->queue_num)
1010 return rte_flow_error_set(error, EINVAL,
1011 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1012 NULL,
1013 "Queues must be incremented continuously");
1014
1015
1016 hash_queues = (BIT_ULL(rss_act->queue[0] + rss_act->queue_num) - 1) &
1017 ~(BIT_ULL(rss_act->queue[0]) - 1);
1018
1019 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1020 if (hash_queues & ~pf->hash_enabled_queues)
1021 return rte_flow_error_set(error, EINVAL,
1022 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1023 NULL, "Some queues are not in LUT");
1024
1025 rss_conf->region_queue_num = (uint8_t)rss_act->queue_num;
1026 rss_conf->region_queue_start = rss_act->queue[0];
1027 rss_conf->region_priority = rte_be_to_cpu_16(vlan_spec->tci) >> 13;
1028 return 0;
1029}
1030
1031static int
1032i40e_hash_parse_global_conf(const struct rte_eth_dev *dev,
1033 const struct rte_flow_item pattern[],
1034 const struct rte_flow_action_rss *rss_act,
1035 struct i40e_rte_flow_rss_conf *rss_conf,
1036 struct rte_flow_error *error)
1037{
1038 if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
1039 return rte_flow_error_set(error, EINVAL,
1040 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1041 NULL,
1042 "Symmetric function should be set with pattern types");
1043
1044 rss_conf->conf.func = rss_act->func;
1045
1046 if (rss_act->types)
1047 PMD_DRV_LOG(WARNING,
1048 "RSS types are ignored when no pattern specified");
1049
1050 if (pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1051 return i40e_hash_parse_queue_region(dev, pattern, rss_act,
1052 rss_conf, error);
1053
1054 if (rss_act->queue)
1055 return i40e_hash_parse_queues(dev, rss_act, rss_conf, error);
1056
1057 if (rss_act->key_len) {
1058 i40e_hash_parse_key(rss_act, rss_conf);
1059 return 0;
1060 }
1061
1062 if (rss_act->func == RTE_ETH_HASH_FUNCTION_DEFAULT)
1063 PMD_DRV_LOG(WARNING, "Nothing change");
1064 return 0;
1065}
1066
1067static bool
1068i40e_hash_validate_rss_types(uint64_t rss_types)
1069{
1070 uint64_t type, mask;
1071
1072
1073 type = ETH_RSS_ETH & rss_types;
1074 mask = (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY) & rss_types;
1075 if (!type && mask)
1076 return false;
1077
1078
1079 type = (I40E_HASH_L4_TYPES | ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1080 ETH_RSS_NONFRAG_IPV4_OTHER | ETH_RSS_IPV6 |
1081 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER) & rss_types;
1082 mask = (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY) & rss_types;
1083 if (!type && mask)
1084 return false;
1085
1086
1087 type = (I40E_HASH_L4_TYPES | ETH_RSS_PORT) & rss_types;
1088 mask = (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY) & rss_types;
1089 if (!type && mask)
1090 return false;
1091
1092 return true;
1093}
1094
1095static int
1096i40e_hash_parse_pattern_act(const struct rte_eth_dev *dev,
1097 const struct rte_flow_item pattern[],
1098 const struct rte_flow_action_rss *rss_act,
1099 struct i40e_rte_flow_rss_conf *rss_conf,
1100 struct rte_flow_error *error)
1101{
1102 if (rss_act->queue)
1103 return rte_flow_error_set(error, EINVAL,
1104 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1105 NULL,
1106 "RSS Queues not supported when pattern specified");
1107
1108 if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
1109 rss_conf->symmetric_enable = true;
1110 else if (rss_act->func != RTE_ETH_HASH_FUNCTION_DEFAULT)
1111 return rte_flow_error_set(error, -EINVAL,
1112 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1113 NULL,
1114 "Only symmetric TOEPLITZ supported when pattern specified");
1115
1116 if (!i40e_hash_validate_rss_types(rss_act->types))
1117 return rte_flow_error_set(error, EINVAL,
1118 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1119 NULL, "RSS types are invalid");
1120
1121 if (rss_act->key_len)
1122 i40e_hash_parse_key(rss_act, rss_conf);
1123
1124 rss_conf->conf.func = rss_act->func;
1125 rss_conf->conf.types = rss_act->types;
1126 rss_conf->inset = i40e_hash_get_inset(rss_act->types);
1127
1128 return i40e_hash_get_pattern_pctypes(dev, pattern, rss_act,
1129 rss_conf, error);
1130}
1131
1132int
1133i40e_hash_parse(const struct rte_eth_dev *dev,
1134 const struct rte_flow_item pattern[],
1135 const struct rte_flow_action actions[],
1136 struct i40e_rte_flow_rss_conf *rss_conf,
1137 struct rte_flow_error *error)
1138{
1139 const struct rte_flow_action_rss *rss_act;
1140
1141 if (actions[1].type != RTE_FLOW_ACTION_TYPE_END)
1142 return rte_flow_error_set(error, EINVAL,
1143 RTE_FLOW_ERROR_TYPE_ACTION,
1144 &actions[1],
1145 "Only support one action for RSS.");
1146
1147 rss_act = (const struct rte_flow_action_rss *)actions[0].conf;
1148 if (rss_act->level)
1149 return rte_flow_error_set(error, ENOTSUP,
1150 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1151 actions,
1152 "RSS level is not supported");
1153
1154 while (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
1155 pattern++;
1156
1157 if (pattern[0].type == RTE_FLOW_ITEM_TYPE_END ||
1158 pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1159 return i40e_hash_parse_global_conf(dev, pattern, rss_act,
1160 rss_conf, error);
1161
1162 return i40e_hash_parse_pattern_act(dev, pattern, rss_act,
1163 rss_conf, error);
1164}
1165
1166static void
1167i40e_invalid_rss_filter(const struct i40e_rte_flow_rss_conf *ref_conf,
1168 struct i40e_rte_flow_rss_conf *conf)
1169{
1170 uint32_t reset_flags = conf->misc_reset_flags;
1171
1172 conf->misc_reset_flags &= ~ref_conf->misc_reset_flags;
1173
1174 if ((reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1175 (ref_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1176 (conf->region_queue_start != ref_conf->region_queue_start ||
1177 conf->region_queue_num != ref_conf->region_queue_num))
1178 conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
1179
1180 conf->reset_config_pctypes &= ~ref_conf->reset_config_pctypes;
1181 conf->reset_symmetric_pctypes &= ~ref_conf->reset_symmetric_pctypes;
1182}
1183
1184int
1185i40e_hash_filter_restore(struct i40e_pf *pf)
1186{
1187 struct i40e_rss_filter *filter;
1188 int ret;
1189
1190 TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1191 struct i40e_rte_flow_rss_conf *rss_conf =
1192 &filter->rss_filter_info;
1193 struct i40e_rss_filter *prev;
1194
1195 rss_conf->misc_reset_flags = 0;
1196 rss_conf->reset_config_pctypes = 0;
1197 rss_conf->reset_symmetric_pctypes = 0;
1198
1199 ret = i40e_hash_config(pf, rss_conf);
1200 if (ret) {
1201 pf->hash_filter_enabled = 0;
1202 i40e_pf_disable_rss(pf);
1203 PMD_DRV_LOG(ERR,
1204 "Re-configure RSS failed, RSS has been disabled");
1205 return ret;
1206 }
1207
1208
1209 TAILQ_FOREACH(prev, &pf->rss_config_list, next) {
1210 if (prev == filter)
1211 break;
1212 i40e_invalid_rss_filter(rss_conf,
1213 &prev->rss_filter_info);
1214 }
1215 }
1216
1217 return 0;
1218}
1219
1220int
1221i40e_hash_filter_create(struct i40e_pf *pf,
1222 struct i40e_rte_flow_rss_conf *rss_conf)
1223{
1224 struct i40e_rss_filter *filter, *prev;
1225 struct i40e_rte_flow_rss_conf *new_conf;
1226 int ret;
1227
1228 filter = rte_zmalloc("i40e_rss_filter", sizeof(*filter), 0);
1229 if (!filter) {
1230 PMD_DRV_LOG(ERR, "Failed to allocate memory.");
1231 return -ENOMEM;
1232 }
1233
1234 new_conf = &filter->rss_filter_info;
1235
1236 memcpy(new_conf, rss_conf, sizeof(*new_conf));
1237 if (new_conf->conf.queue_num)
1238 new_conf->conf.queue = new_conf->queue;
1239 if (new_conf->conf.key_len)
1240 new_conf->conf.key = new_conf->key;
1241
1242 ret = i40e_hash_config(pf, new_conf);
1243 if (ret) {
1244 rte_free(filter);
1245 if (i40e_pf_config_rss(pf))
1246 return ret;
1247
1248 (void)i40e_hash_filter_restore(pf);
1249 return ret;
1250 }
1251
1252
1253 TAILQ_FOREACH(prev, &pf->rss_config_list, next)
1254 i40e_invalid_rss_filter(new_conf, &prev->rss_filter_info);
1255
1256 TAILQ_INSERT_TAIL(&pf->rss_config_list, filter, next);
1257 return 0;
1258}
1259
1260static int
1261i40e_hash_reset_conf(struct i40e_pf *pf,
1262 struct i40e_rte_flow_rss_conf *rss_conf)
1263{
1264 struct i40e_hw *hw = &pf->adapter->hw;
1265 uint64_t inset;
1266 uint32_t idx;
1267 int ret;
1268
1269 if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_FUNC) {
1270 ret = i40e_hash_config_func(hw, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
1271 if (ret)
1272 return ret;
1273
1274 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_FUNC;
1275 }
1276
1277 if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) {
1278 ret = i40e_flush_queue_region_all_conf(pf->adapter->eth_dev,
1279 hw, pf, 0);
1280 if (ret)
1281 return ret;
1282
1283 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_REGION;
1284 }
1285
1286 if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_KEY) {
1287 ret = i40e_pf_reset_rss_key(pf);
1288 if (ret)
1289 return ret;
1290
1291 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_KEY;
1292 }
1293
1294 if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_QUEUE) {
1295 if (!pf->adapter->rss_reta_updated) {
1296 ret = i40e_pf_reset_rss_reta(pf);
1297 if (ret)
1298 return ret;
1299 }
1300
1301 pf->hash_enabled_queues = 0;
1302 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_QUEUE;
1303 }
1304
1305 while (rss_conf->reset_config_pctypes) {
1306 idx = rte_bsf64(rss_conf->reset_config_pctypes);
1307
1308 i40e_hash_enable_pctype(hw, idx, false);
1309 inset = i40e_get_default_input_set(idx);
1310 if (inset) {
1311 ret = i40e_set_hash_inset(hw, inset, idx, false);
1312 if (ret)
1313 return ret;
1314 }
1315
1316 rss_conf->reset_config_pctypes &= ~BIT_ULL(idx);
1317 }
1318
1319 while (rss_conf->reset_symmetric_pctypes) {
1320 idx = rte_bsf64(rss_conf->reset_symmetric_pctypes);
1321
1322 ret = i40e_hash_config_pctype_symmetric(hw, idx, false);
1323 if (ret)
1324 return ret;
1325
1326 rss_conf->reset_symmetric_pctypes &= ~BIT_ULL(idx);
1327 }
1328
1329 return 0;
1330}
1331
1332int
1333i40e_hash_filter_destroy(struct i40e_pf *pf,
1334 const struct i40e_rss_filter *rss_filter)
1335{
1336 struct i40e_rss_filter *filter;
1337 int ret;
1338
1339 TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1340 if (rss_filter == filter) {
1341 ret = i40e_hash_reset_conf(pf,
1342 &filter->rss_filter_info);
1343 if (ret)
1344 return ret;
1345
1346 TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1347 rte_free(filter);
1348 return 0;
1349 }
1350 }
1351
1352 return -ENOENT;
1353}
1354
1355int
1356i40e_hash_filter_flush(struct i40e_pf *pf)
1357{
1358 struct rte_flow *flow, *next;
1359
1360 TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, next) {
1361 if (flow->filter_type != RTE_ETH_FILTER_HASH)
1362 continue;
1363
1364 if (flow->rule) {
1365 struct i40e_rss_filter *filter = flow->rule;
1366 int ret;
1367
1368 ret = i40e_hash_reset_conf(pf,
1369 &filter->rss_filter_info);
1370 if (ret)
1371 return ret;
1372
1373 TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1374 rte_free(filter);
1375 }
1376
1377 TAILQ_REMOVE(&pf->flow_list, flow, node);
1378 rte_free(flow);
1379 }
1380
1381 assert(!pf->rss_config_list.tqh_first);
1382 return 0;
1383}
1384
