1
2
3
4
5
6#include <rdma/ib_user_verbs.h>
7#include <rdma/ib_verbs.h>
8#include <rdma/uverbs_types.h>
9#include <rdma/uverbs_ioctl.h>
10#include <rdma/uverbs_std_types.h>
11#include <rdma/mlx5_user_ioctl_cmds.h>
12#include <rdma/mlx5_user_ioctl_verbs.h>
13#include <rdma/ib_umem.h>
14#include <linux/mlx5/driver.h>
15#include <linux/mlx5/fs.h>
16#include <linux/mlx5/fs_helpers.h>
17#include <linux/mlx5/accel.h>
18#include <linux/mlx5/eswitch.h>
19#include "mlx5_ib.h"
20#include "counters.h"
21#include "devx.h"
22#include "fs.h"
23
24#define UVERBS_MODULE_NAME mlx5_ib
25#include <rdma/uverbs_named_ioctl.h>
26
27enum {
28 MATCH_CRITERIA_ENABLE_OUTER_BIT,
29 MATCH_CRITERIA_ENABLE_MISC_BIT,
30 MATCH_CRITERIA_ENABLE_INNER_BIT,
31 MATCH_CRITERIA_ENABLE_MISC2_BIT
32};
33
34#define HEADER_IS_ZERO(match_criteria, headers) \
35 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
36 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
37
38static u8 get_match_criteria_enable(u32 *match_criteria)
39{
40 u8 match_criteria_enable;
41
42 match_criteria_enable =
43 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
44 MATCH_CRITERIA_ENABLE_OUTER_BIT;
45 match_criteria_enable |=
46 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
47 MATCH_CRITERIA_ENABLE_MISC_BIT;
48 match_criteria_enable |=
49 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
50 MATCH_CRITERIA_ENABLE_INNER_BIT;
51 match_criteria_enable |=
52 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
53 MATCH_CRITERIA_ENABLE_MISC2_BIT;
54
55 return match_criteria_enable;
56}
57
58static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
59{
60 u8 entry_mask;
61 u8 entry_val;
62 int err = 0;
63
64 if (!mask)
65 goto out;
66
67 entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
68 ip_protocol);
69 entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
70 ip_protocol);
71 if (!entry_mask) {
72 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
73 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
74 goto out;
75 }
76
77 if (mask != entry_mask || val != entry_val)
78 err = -EINVAL;
79out:
80 return err;
81}
82
83static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
84 bool inner)
85{
86 if (inner) {
87 MLX5_SET(fte_match_set_misc,
88 misc_c, inner_ipv6_flow_label, mask);
89 MLX5_SET(fte_match_set_misc,
90 misc_v, inner_ipv6_flow_label, val);
91 } else {
92 MLX5_SET(fte_match_set_misc,
93 misc_c, outer_ipv6_flow_label, mask);
94 MLX5_SET(fte_match_set_misc,
95 misc_v, outer_ipv6_flow_label, val);
96 }
97}
98
99static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
100{
101 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
102 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
103 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
104 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
105}
106
107static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
108{
109 if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
110 !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
111 return -EOPNOTSUPP;
112
113 if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
114 !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
115 return -EOPNOTSUPP;
116
117 if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
118 !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
119 return -EOPNOTSUPP;
120
121 if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
122 !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
123 return -EOPNOTSUPP;
124
125 return 0;
126}
127
128#define LAST_ETH_FIELD vlan_tag
129#define LAST_IB_FIELD sl
130#define LAST_IPV4_FIELD tos
131#define LAST_IPV6_FIELD traffic_class
132#define LAST_TCP_UDP_FIELD src_port
133#define LAST_TUNNEL_FIELD tunnel_id
134#define LAST_FLOW_TAG_FIELD tag_id
135#define LAST_DROP_FIELD size
136#define LAST_COUNTERS_FIELD counters
137
138
139#define FIELDS_NOT_SUPPORTED(filter, field) \
140 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \
141 sizeof(filter) - offsetofend(typeof(filter), field))
142
143int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
144 bool is_egress,
145 struct mlx5_flow_act *action)
146{
147
148 switch (maction->ib_action.type) {
149 case IB_FLOW_ACTION_ESP:
150 if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
151 MLX5_FLOW_CONTEXT_ACTION_DECRYPT))
152 return -EINVAL;
153
154 action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx;
155 action->action |= is_egress ?
156 MLX5_FLOW_CONTEXT_ACTION_ENCRYPT :
157 MLX5_FLOW_CONTEXT_ACTION_DECRYPT;
158 return 0;
159 case IB_FLOW_ACTION_UNSPECIFIED:
160 if (maction->flow_action_raw.sub_type ==
161 MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
162 if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
163 return -EINVAL;
164 action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
165 action->modify_hdr =
166 maction->flow_action_raw.modify_hdr;
167 return 0;
168 }
169 if (maction->flow_action_raw.sub_type ==
170 MLX5_IB_FLOW_ACTION_DECAP) {
171 if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
172 return -EINVAL;
173 action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
174 return 0;
175 }
176 if (maction->flow_action_raw.sub_type ==
177 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
178 if (action->action &
179 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
180 return -EINVAL;
181 action->action |=
182 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
183 action->pkt_reformat =
184 maction->flow_action_raw.pkt_reformat;
185 return 0;
186 }
187 fallthrough;
188 default:
189 return -EOPNOTSUPP;
190 }
191}
192
193static int parse_flow_attr(struct mlx5_core_dev *mdev,
194 struct mlx5_flow_spec *spec,
195 const union ib_flow_spec *ib_spec,
196 const struct ib_flow_attr *flow_attr,
197 struct mlx5_flow_act *action, u32 prev_type)
198{
199 struct mlx5_flow_context *flow_context = &spec->flow_context;
200 u32 *match_c = spec->match_criteria;
201 u32 *match_v = spec->match_value;
202 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
203 misc_parameters);
204 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
205 misc_parameters);
206 void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
207 misc_parameters_2);
208 void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
209 misc_parameters_2);
210 void *headers_c;
211 void *headers_v;
212 int match_ipv;
213 int ret;
214
215 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
216 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
217 inner_headers);
218 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
219 inner_headers);
220 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
221 ft_field_support.inner_ip_version);
222 } else {
223 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
224 outer_headers);
225 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
226 outer_headers);
227 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
228 ft_field_support.outer_ip_version);
229 }
230
231 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
232 case IB_FLOW_SPEC_ETH:
233 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
234 return -EOPNOTSUPP;
235
236 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
237 dmac_47_16),
238 ib_spec->eth.mask.dst_mac);
239 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
240 dmac_47_16),
241 ib_spec->eth.val.dst_mac);
242
243 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
244 smac_47_16),
245 ib_spec->eth.mask.src_mac);
246 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
247 smac_47_16),
248 ib_spec->eth.val.src_mac);
249
250 if (ib_spec->eth.mask.vlan_tag) {
251 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
252 cvlan_tag, 1);
253 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
254 cvlan_tag, 1);
255
256 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
257 first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
258 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
259 first_vid, ntohs(ib_spec->eth.val.vlan_tag));
260
261 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
262 first_cfi,
263 ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
264 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
265 first_cfi,
266 ntohs(ib_spec->eth.val.vlan_tag) >> 12);
267
268 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
269 first_prio,
270 ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
271 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
272 first_prio,
273 ntohs(ib_spec->eth.val.vlan_tag) >> 13);
274 }
275 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
276 ethertype, ntohs(ib_spec->eth.mask.ether_type));
277 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
278 ethertype, ntohs(ib_spec->eth.val.ether_type));
279 break;
280 case IB_FLOW_SPEC_IPV4:
281 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
282 return -EOPNOTSUPP;
283
284 if (match_ipv) {
285 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
286 ip_version, 0xf);
287 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
288 ip_version, MLX5_FS_IPV4_VERSION);
289 } else {
290 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
291 ethertype, 0xffff);
292 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
293 ethertype, ETH_P_IP);
294 }
295
296 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
297 src_ipv4_src_ipv6.ipv4_layout.ipv4),
298 &ib_spec->ipv4.mask.src_ip,
299 sizeof(ib_spec->ipv4.mask.src_ip));
300 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
301 src_ipv4_src_ipv6.ipv4_layout.ipv4),
302 &ib_spec->ipv4.val.src_ip,
303 sizeof(ib_spec->ipv4.val.src_ip));
304 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
305 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
306 &ib_spec->ipv4.mask.dst_ip,
307 sizeof(ib_spec->ipv4.mask.dst_ip));
308 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
309 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
310 &ib_spec->ipv4.val.dst_ip,
311 sizeof(ib_spec->ipv4.val.dst_ip));
312
313 set_tos(headers_c, headers_v,
314 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
315
316 if (set_proto(headers_c, headers_v,
317 ib_spec->ipv4.mask.proto,
318 ib_spec->ipv4.val.proto))
319 return -EINVAL;
320 break;
321 case IB_FLOW_SPEC_IPV6:
322 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
323 return -EOPNOTSUPP;
324
325 if (match_ipv) {
326 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
327 ip_version, 0xf);
328 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
329 ip_version, MLX5_FS_IPV6_VERSION);
330 } else {
331 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
332 ethertype, 0xffff);
333 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
334 ethertype, ETH_P_IPV6);
335 }
336
337 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
338 src_ipv4_src_ipv6.ipv6_layout.ipv6),
339 &ib_spec->ipv6.mask.src_ip,
340 sizeof(ib_spec->ipv6.mask.src_ip));
341 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
342 src_ipv4_src_ipv6.ipv6_layout.ipv6),
343 &ib_spec->ipv6.val.src_ip,
344 sizeof(ib_spec->ipv6.val.src_ip));
345 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
346 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
347 &ib_spec->ipv6.mask.dst_ip,
348 sizeof(ib_spec->ipv6.mask.dst_ip));
349 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
350 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
351 &ib_spec->ipv6.val.dst_ip,
352 sizeof(ib_spec->ipv6.val.dst_ip));
353
354 set_tos(headers_c, headers_v,
355 ib_spec->ipv6.mask.traffic_class,
356 ib_spec->ipv6.val.traffic_class);
357
358 if (set_proto(headers_c, headers_v,
359 ib_spec->ipv6.mask.next_hdr,
360 ib_spec->ipv6.val.next_hdr))
361 return -EINVAL;
362
363 set_flow_label(misc_params_c, misc_params_v,
364 ntohl(ib_spec->ipv6.mask.flow_label),
365 ntohl(ib_spec->ipv6.val.flow_label),
366 ib_spec->type & IB_FLOW_SPEC_INNER);
367 break;
368 case IB_FLOW_SPEC_ESP:
369 if (ib_spec->esp.mask.seq)
370 return -EOPNOTSUPP;
371
372 MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi,
373 ntohl(ib_spec->esp.mask.spi));
374 MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,
375 ntohl(ib_spec->esp.val.spi));
376 break;
377 case IB_FLOW_SPEC_TCP:
378 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
379 LAST_TCP_UDP_FIELD))
380 return -EOPNOTSUPP;
381
382 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
383 return -EINVAL;
384
385 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
386 ntohs(ib_spec->tcp_udp.mask.src_port));
387 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
388 ntohs(ib_spec->tcp_udp.val.src_port));
389
390 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
391 ntohs(ib_spec->tcp_udp.mask.dst_port));
392 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
393 ntohs(ib_spec->tcp_udp.val.dst_port));
394 break;
395 case IB_FLOW_SPEC_UDP:
396 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
397 LAST_TCP_UDP_FIELD))
398 return -EOPNOTSUPP;
399
400 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
401 return -EINVAL;
402
403 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
404 ntohs(ib_spec->tcp_udp.mask.src_port));
405 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
406 ntohs(ib_spec->tcp_udp.val.src_port));
407
408 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
409 ntohs(ib_spec->tcp_udp.mask.dst_port));
410 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
411 ntohs(ib_spec->tcp_udp.val.dst_port));
412 break;
413 case IB_FLOW_SPEC_GRE:
414 if (ib_spec->gre.mask.c_ks_res0_ver)
415 return -EOPNOTSUPP;
416
417 if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
418 return -EINVAL;
419
420 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
421 0xff);
422 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
423 IPPROTO_GRE);
424
425 MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
426 ntohs(ib_spec->gre.mask.protocol));
427 MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
428 ntohs(ib_spec->gre.val.protocol));
429
430 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
431 gre_key.nvgre.hi),
432 &ib_spec->gre.mask.key,
433 sizeof(ib_spec->gre.mask.key));
434 memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
435 gre_key.nvgre.hi),
436 &ib_spec->gre.val.key,
437 sizeof(ib_spec->gre.val.key));
438 break;
439 case IB_FLOW_SPEC_MPLS:
440 switch (prev_type) {
441 case IB_FLOW_SPEC_UDP:
442 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
443 ft_field_support.outer_first_mpls_over_udp),
444 &ib_spec->mpls.mask.tag))
445 return -EOPNOTSUPP;
446
447 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
448 outer_first_mpls_over_udp),
449 &ib_spec->mpls.val.tag,
450 sizeof(ib_spec->mpls.val.tag));
451 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
452 outer_first_mpls_over_udp),
453 &ib_spec->mpls.mask.tag,
454 sizeof(ib_spec->mpls.mask.tag));
455 break;
456 case IB_FLOW_SPEC_GRE:
457 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
458 ft_field_support.outer_first_mpls_over_gre),
459 &ib_spec->mpls.mask.tag))
460 return -EOPNOTSUPP;
461
462 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
463 outer_first_mpls_over_gre),
464 &ib_spec->mpls.val.tag,
465 sizeof(ib_spec->mpls.val.tag));
466 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
467 outer_first_mpls_over_gre),
468 &ib_spec->mpls.mask.tag,
469 sizeof(ib_spec->mpls.mask.tag));
470 break;
471 default:
472 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
473 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
474 ft_field_support.inner_first_mpls),
475 &ib_spec->mpls.mask.tag))
476 return -EOPNOTSUPP;
477
478 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
479 inner_first_mpls),
480 &ib_spec->mpls.val.tag,
481 sizeof(ib_spec->mpls.val.tag));
482 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
483 inner_first_mpls),
484 &ib_spec->mpls.mask.tag,
485 sizeof(ib_spec->mpls.mask.tag));
486 } else {
487 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
488 ft_field_support.outer_first_mpls),
489 &ib_spec->mpls.mask.tag))
490 return -EOPNOTSUPP;
491
492 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
493 outer_first_mpls),
494 &ib_spec->mpls.val.tag,
495 sizeof(ib_spec->mpls.val.tag));
496 memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
497 outer_first_mpls),
498 &ib_spec->mpls.mask.tag,
499 sizeof(ib_spec->mpls.mask.tag));
500 }
501 }
502 break;
503 case IB_FLOW_SPEC_VXLAN_TUNNEL:
504 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
505 LAST_TUNNEL_FIELD))
506 return -EOPNOTSUPP;
507
508 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
509 ntohl(ib_spec->tunnel.mask.tunnel_id));
510 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
511 ntohl(ib_spec->tunnel.val.tunnel_id));
512 break;
513 case IB_FLOW_SPEC_ACTION_TAG:
514 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
515 LAST_FLOW_TAG_FIELD))
516 return -EOPNOTSUPP;
517 if (ib_spec->flow_tag.tag_id >= BIT(24))
518 return -EINVAL;
519
520 flow_context->flow_tag = ib_spec->flow_tag.tag_id;
521 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
522 break;
523 case IB_FLOW_SPEC_ACTION_DROP:
524 if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
525 LAST_DROP_FIELD))
526 return -EOPNOTSUPP;
527 action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
528 break;
529 case IB_FLOW_SPEC_ACTION_HANDLE:
530 ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
531 flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
532 if (ret)
533 return ret;
534 break;
535 case IB_FLOW_SPEC_ACTION_COUNT:
536 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
537 LAST_COUNTERS_FIELD))
538 return -EOPNOTSUPP;
539
540
541 if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
542 return -EINVAL;
543
544 action->counters = ib_spec->flow_count.counters;
545 action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
546 break;
547 default:
548 return -EINVAL;
549 }
550
551 return 0;
552}
553
554
555
556
557
558static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
559{
560 union ib_flow_spec *flow_spec;
561
562 if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
563 ib_attr->num_of_specs < 1)
564 return false;
565
566 flow_spec = (union ib_flow_spec *)(ib_attr + 1);
567 if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
568 struct ib_flow_spec_ipv4 *ipv4_spec;
569
570 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
571 if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
572 return true;
573
574 return false;
575 }
576
577 if (flow_spec->type == IB_FLOW_SPEC_ETH) {
578 struct ib_flow_spec_eth *eth_spec;
579
580 eth_spec = (struct ib_flow_spec_eth *)flow_spec;
581 return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
582 is_multicast_ether_addr(eth_spec->val.dst_mac);
583 }
584
585 return false;
586}
587
588enum valid_spec {
589 VALID_SPEC_INVALID,
590 VALID_SPEC_VALID,
591 VALID_SPEC_NA,
592};
593
594static enum valid_spec
595is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev,
596 const struct mlx5_flow_spec *spec,
597 const struct mlx5_flow_act *flow_act,
598 bool egress)
599{
600 const u32 *match_c = spec->match_criteria;
601 bool is_crypto =
602 (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
603 MLX5_FLOW_CONTEXT_ACTION_DECRYPT));
604 bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c);
605 bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP;
606
607
608
609
610
611 if (!is_crypto)
612 return VALID_SPEC_NA;
613
614 return is_crypto && is_ipsec &&
615 (!egress || (!is_drop &&
616 !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ?
617 VALID_SPEC_VALID : VALID_SPEC_INVALID;
618}
619
620static bool is_valid_spec(struct mlx5_core_dev *mdev,
621 const struct mlx5_flow_spec *spec,
622 const struct mlx5_flow_act *flow_act,
623 bool egress)
624{
625
626 return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID;
627}
628
629static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
630 const struct ib_flow_attr *flow_attr,
631 bool check_inner)
632{
633 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
634 int match_ipv = check_inner ?
635 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
636 ft_field_support.inner_ip_version) :
637 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
638 ft_field_support.outer_ip_version);
639 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
640 bool ipv4_spec_valid, ipv6_spec_valid;
641 unsigned int ip_spec_type = 0;
642 bool has_ethertype = false;
643 unsigned int spec_index;
644 bool mask_valid = true;
645 u16 eth_type = 0;
646 bool type_valid;
647
648
649 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
650 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
651 ib_spec->eth.mask.ether_type) {
652 mask_valid = (ib_spec->eth.mask.ether_type ==
653 htons(0xffff));
654 has_ethertype = true;
655 eth_type = ntohs(ib_spec->eth.val.ether_type);
656 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
657 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
658 ip_spec_type = ib_spec->type;
659 }
660 ib_spec = (void *)ib_spec + ib_spec->size;
661 }
662
663 type_valid = (!has_ethertype) || (!ip_spec_type);
664 if (!type_valid && mask_valid) {
665 ipv4_spec_valid = (eth_type == ETH_P_IP) &&
666 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
667 ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
668 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
669
670 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
671 (((eth_type == ETH_P_MPLS_UC) ||
672 (eth_type == ETH_P_MPLS_MC)) && match_ipv);
673 }
674
675 return type_valid;
676}
677
678static bool is_valid_attr(struct mlx5_core_dev *mdev,
679 const struct ib_flow_attr *flow_attr)
680{
681 return is_valid_ethertype(mdev, flow_attr, false) &&
682 is_valid_ethertype(mdev, flow_attr, true);
683}
684
685static void put_flow_table(struct mlx5_ib_dev *dev,
686 struct mlx5_ib_flow_prio *prio, bool ft_added)
687{
688 prio->refcount -= !!ft_added;
689 if (!prio->refcount) {
690 mlx5_destroy_flow_table(prio->flow_table);
691 prio->flow_table = NULL;
692 }
693}
694
695static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
696{
697 struct mlx5_ib_flow_handler *handler = container_of(flow_id,
698 struct mlx5_ib_flow_handler,
699 ibflow);
700 struct mlx5_ib_flow_handler *iter, *tmp;
701 struct mlx5_ib_dev *dev = handler->dev;
702
703 mutex_lock(&dev->flow_db->lock);
704
705 list_for_each_entry_safe(iter, tmp, &handler->list, list) {
706 mlx5_del_flow_rules(iter->rule);
707 put_flow_table(dev, iter->prio, true);
708 list_del(&iter->list);
709 kfree(iter);
710 }
711
712 mlx5_del_flow_rules(handler->rule);
713 put_flow_table(dev, handler->prio, true);
714 mlx5_ib_counters_clear_description(handler->ibcounters);
715 mutex_unlock(&dev->flow_db->lock);
716 if (handler->flow_matcher)
717 atomic_dec(&handler->flow_matcher->usecnt);
718 kfree(handler);
719
720 return 0;
721}
722
723static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
724{
725 priority *= 2;
726 if (!dont_trap)
727 priority++;
728 return priority;
729}
730
731enum flow_table_type {
732 MLX5_IB_FT_RX,
733 MLX5_IB_FT_TX
734};
735
736#define MLX5_FS_MAX_TYPES 6
737#define MLX5_FS_MAX_ENTRIES BIT(16)
738
739static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,
740 struct mlx5_ib_flow_prio *prio,
741 int priority,
742 int num_entries, int num_groups,
743 u32 flags)
744{
745 struct mlx5_flow_table_attr ft_attr = {};
746 struct mlx5_flow_table *ft;
747
748 ft_attr.prio = priority;
749 ft_attr.max_fte = num_entries;
750 ft_attr.flags = flags;
751 ft_attr.autogroup.max_num_groups = num_groups;
752 ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
753 if (IS_ERR(ft))
754 return ERR_CAST(ft);
755
756 prio->flow_table = ft;
757 prio->refcount = 0;
758 return prio;
759}
760
761static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
762 struct ib_flow_attr *flow_attr,
763 enum flow_table_type ft_type)
764{
765 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
766 struct mlx5_flow_namespace *ns = NULL;
767 enum mlx5_flow_namespace_type fn_type;
768 struct mlx5_ib_flow_prio *prio;
769 struct mlx5_flow_table *ft;
770 int max_table_size;
771 int num_entries;
772 int num_groups;
773 bool esw_encap;
774 u32 flags = 0;
775 int priority;
776
777 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
778 log_max_ft_size));
779 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
780 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
781 switch (flow_attr->type) {
782 case IB_FLOW_ATTR_NORMAL:
783 if (flow_is_multicast_only(flow_attr) && !dont_trap)
784 priority = MLX5_IB_FLOW_MCAST_PRIO;
785 else
786 priority = ib_prio_to_core_prio(flow_attr->priority,
787 dont_trap);
788 if (ft_type == MLX5_IB_FT_RX) {
789 fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
790 prio = &dev->flow_db->prios[priority];
791 if (!dev->is_rep && !esw_encap &&
792 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
793 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
794 if (!dev->is_rep && !esw_encap &&
795 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
796 reformat_l3_tunnel_to_l2))
797 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
798 } else {
799 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX(
800 dev->mdev, log_max_ft_size));
801 fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
802 prio = &dev->flow_db->egress_prios[priority];
803 if (!dev->is_rep && !esw_encap &&
804 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
805 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
806 }
807 ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
808 num_entries = MLX5_FS_MAX_ENTRIES;
809 num_groups = MLX5_FS_MAX_TYPES;
810 break;
811 case IB_FLOW_ATTR_ALL_DEFAULT:
812 case IB_FLOW_ATTR_MC_DEFAULT:
813 ns = mlx5_get_flow_namespace(dev->mdev,
814 MLX5_FLOW_NAMESPACE_LEFTOVERS);
815 build_leftovers_ft_param(&priority, &num_entries, &num_groups);
816 prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
817 break;
818 case IB_FLOW_ATTR_SNIFFER:
819 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
820 allow_sniffer_and_nic_rx_shared_tir))
821 return ERR_PTR(-EOPNOTSUPP);
822
823 ns = mlx5_get_flow_namespace(
824 dev->mdev, ft_type == MLX5_IB_FT_RX ?
825 MLX5_FLOW_NAMESPACE_SNIFFER_RX :
826 MLX5_FLOW_NAMESPACE_SNIFFER_TX);
827
828 prio = &dev->flow_db->sniffer[ft_type];
829 priority = 0;
830 num_entries = 1;
831 num_groups = 1;
832 break;
833 default:
834 break;
835 }
836
837 if (!ns)
838 return ERR_PTR(-EOPNOTSUPP);
839
840 max_table_size = min_t(int, num_entries, max_table_size);
841
842 ft = prio->flow_table;
843 if (!ft)
844 return _get_prio(ns, prio, priority, max_table_size, num_groups,
845 flags);
846
847 return prio;
848}
849
850static void set_underlay_qp(struct mlx5_ib_dev *dev,
851 struct mlx5_flow_spec *spec,
852 u32 underlay_qpn)
853{
854 void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
855 spec->match_criteria,
856 misc_parameters);
857 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
858 misc_parameters);
859
860 if (underlay_qpn &&
861 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
862 ft_field_support.bth_dst_qp)) {
863 MLX5_SET(fte_match_set_misc,
864 misc_params_v, bth_dst_qp, underlay_qpn);
865 MLX5_SET(fte_match_set_misc,
866 misc_params_c, bth_dst_qp, 0xffffff);
867 }
868}
869
870static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
871 struct mlx5_flow_spec *spec,
872 struct mlx5_eswitch_rep *rep)
873{
874 struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
875 void *misc;
876
877 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
878 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
879 misc_parameters_2);
880
881 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
882 mlx5_eswitch_get_vport_metadata_for_match(esw,
883 rep->vport));
884 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
885 misc_parameters_2);
886
887 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
888 mlx5_eswitch_get_vport_metadata_mask());
889 } else {
890 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
891 misc_parameters);
892
893 MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
894
895 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
896 misc_parameters);
897
898 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
899 }
900}
901
902static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
903 struct mlx5_ib_flow_prio *ft_prio,
904 const struct ib_flow_attr *flow_attr,
905 struct mlx5_flow_destination *dst,
906 u32 underlay_qpn,
907 struct mlx5_ib_create_flow *ucmd)
908{
909 struct mlx5_flow_table *ft = ft_prio->flow_table;
910 struct mlx5_ib_flow_handler *handler;
911 struct mlx5_flow_act flow_act = {};
912 struct mlx5_flow_spec *spec;
913 struct mlx5_flow_destination dest_arr[2] = {};
914 struct mlx5_flow_destination *rule_dst = dest_arr;
915 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
916 unsigned int spec_index;
917 u32 prev_type = 0;
918 int err = 0;
919 int dest_num = 0;
920 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
921
922 if (!is_valid_attr(dev->mdev, flow_attr))
923 return ERR_PTR(-EINVAL);
924
925 if (dev->is_rep && is_egress)
926 return ERR_PTR(-EINVAL);
927
928 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
929 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
930 if (!handler || !spec) {
931 err = -ENOMEM;
932 goto free;
933 }
934
935 INIT_LIST_HEAD(&handler->list);
936
937 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
938 err = parse_flow_attr(dev->mdev, spec,
939 ib_flow, flow_attr, &flow_act,
940 prev_type);
941 if (err < 0)
942 goto free;
943
944 prev_type = ((union ib_flow_spec *)ib_flow)->type;
945 ib_flow += ((union ib_flow_spec *)ib_flow)->size;
946 }
947
948 if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
949 memcpy(&dest_arr[0], dst, sizeof(*dst));
950 dest_num++;
951 }
952
953 if (!flow_is_multicast_only(flow_attr))
954 set_underlay_qp(dev, spec, underlay_qpn);
955
956 if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) {
957 struct mlx5_eswitch_rep *rep;
958
959 rep = dev->port[flow_attr->port - 1].rep;
960 if (!rep) {
961 err = -EINVAL;
962 goto free;
963 }
964
965 mlx5_ib_set_rule_source_port(dev, spec, rep);
966 }
967
968 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
969
970 if (is_egress &&
971 !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) {
972 err = -EINVAL;
973 goto free;
974 }
975
976 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
977 struct mlx5_ib_mcounters *mcounters;
978
979 err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd);
980 if (err)
981 goto free;
982
983 mcounters = to_mcounters(flow_act.counters);
984 handler->ibcounters = flow_act.counters;
985 dest_arr[dest_num].type =
986 MLX5_FLOW_DESTINATION_TYPE_COUNTER;
987 dest_arr[dest_num].counter_id =
988 mlx5_fc_id(mcounters->hw_cntrs_hndl);
989 dest_num++;
990 }
991
992 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
993 if (!dest_num)
994 rule_dst = NULL;
995 } else {
996 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
997 flow_act.action |=
998 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
999 if (is_egress)
1000 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1001 else if (dest_num)
1002 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1003 }
1004
1005 if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) &&
1006 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1007 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1008 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
1009 spec->flow_context.flow_tag, flow_attr->type);
1010 err = -EINVAL;
1011 goto free;
1012 }
1013 handler->rule = mlx5_add_flow_rules(ft, spec,
1014 &flow_act,
1015 rule_dst, dest_num);
1016
1017 if (IS_ERR(handler->rule)) {
1018 err = PTR_ERR(handler->rule);
1019 goto free;
1020 }
1021
1022 ft_prio->refcount++;
1023 handler->prio = ft_prio;
1024 handler->dev = dev;
1025
1026 ft_prio->flow_table = ft;
1027free:
1028 if (err && handler) {
1029 mlx5_ib_counters_clear_description(handler->ibcounters);
1030 kfree(handler);
1031 }
1032 kvfree(spec);
1033 return err ? ERR_PTR(err) : handler;
1034}
1035
1036static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
1037 struct mlx5_ib_flow_prio *ft_prio,
1038 const struct ib_flow_attr *flow_attr,
1039 struct mlx5_flow_destination *dst)
1040{
1041 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
1042}
1043
1044enum {
1045 LEFTOVERS_MC,
1046 LEFTOVERS_UC,
1047};
1048
1049static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
1050 struct mlx5_ib_flow_prio *ft_prio,
1051 struct ib_flow_attr *flow_attr,
1052 struct mlx5_flow_destination *dst)
1053{
1054 struct mlx5_ib_flow_handler *handler_ucast = NULL;
1055 struct mlx5_ib_flow_handler *handler = NULL;
1056
1057 static struct {
1058 struct ib_flow_attr flow_attr;
1059 struct ib_flow_spec_eth eth_flow;
1060 } leftovers_specs[] = {
1061 [LEFTOVERS_MC] = {
1062 .flow_attr = {
1063 .num_of_specs = 1,
1064 .size = sizeof(leftovers_specs[0])
1065 },
1066 .eth_flow = {
1067 .type = IB_FLOW_SPEC_ETH,
1068 .size = sizeof(struct ib_flow_spec_eth),
1069 .mask = {.dst_mac = {0x1} },
1070 .val = {.dst_mac = {0x1} }
1071 }
1072 },
1073 [LEFTOVERS_UC] = {
1074 .flow_attr = {
1075 .num_of_specs = 1,
1076 .size = sizeof(leftovers_specs[0])
1077 },
1078 .eth_flow = {
1079 .type = IB_FLOW_SPEC_ETH,
1080 .size = sizeof(struct ib_flow_spec_eth),
1081 .mask = {.dst_mac = {0x1} },
1082 .val = {.dst_mac = {} }
1083 }
1084 }
1085 };
1086
1087 handler = create_flow_rule(dev, ft_prio,
1088 &leftovers_specs[LEFTOVERS_MC].flow_attr,
1089 dst);
1090 if (!IS_ERR(handler) &&
1091 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
1092 handler_ucast = create_flow_rule(dev, ft_prio,
1093 &leftovers_specs[LEFTOVERS_UC].flow_attr,
1094 dst);
1095 if (IS_ERR(handler_ucast)) {
1096 mlx5_del_flow_rules(handler->rule);
1097 ft_prio->refcount--;
1098 kfree(handler);
1099 handler = handler_ucast;
1100 } else {
1101 list_add(&handler_ucast->list, &handler->list);
1102 }
1103 }
1104
1105 return handler;
1106}
1107
1108static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
1109 struct mlx5_ib_flow_prio *ft_rx,
1110 struct mlx5_ib_flow_prio *ft_tx,
1111 struct mlx5_flow_destination *dst)
1112{
1113 struct mlx5_ib_flow_handler *handler_rx;
1114 struct mlx5_ib_flow_handler *handler_tx;
1115 int err;
1116 static const struct ib_flow_attr flow_attr = {
1117 .num_of_specs = 0,
1118 .type = IB_FLOW_ATTR_SNIFFER,
1119 .size = sizeof(flow_attr)
1120 };
1121
1122 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
1123 if (IS_ERR(handler_rx)) {
1124 err = PTR_ERR(handler_rx);
1125 goto err;
1126 }
1127
1128 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
1129 if (IS_ERR(handler_tx)) {
1130 err = PTR_ERR(handler_tx);
1131 goto err_tx;
1132 }
1133
1134 list_add(&handler_tx->list, &handler_rx->list);
1135
1136 return handler_rx;
1137
1138err_tx:
1139 mlx5_del_flow_rules(handler_rx->rule);
1140 ft_rx->refcount--;
1141 kfree(handler_rx);
1142err:
1143 return ERR_PTR(err);
1144}
1145
1146static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
1147 struct ib_flow_attr *flow_attr,
1148 struct ib_udata *udata)
1149{
1150 struct mlx5_ib_dev *dev = to_mdev(qp->device);
1151 struct mlx5_ib_qp *mqp = to_mqp(qp);
1152 struct mlx5_ib_flow_handler *handler = NULL;
1153 struct mlx5_flow_destination *dst = NULL;
1154 struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
1155 struct mlx5_ib_flow_prio *ft_prio;
1156 bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1157 struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
1158 size_t min_ucmd_sz, required_ucmd_sz;
1159 int err;
1160 int underlay_qpn;
1161
1162 if (udata && udata->inlen) {
1163 min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved);
1164 if (udata->inlen < min_ucmd_sz)
1165 return ERR_PTR(-EOPNOTSUPP);
1166
1167 err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
1168 if (err)
1169 return ERR_PTR(err);
1170
1171
1172 if (ucmd_hdr.ncounters_data > 1)
1173 return ERR_PTR(-EINVAL);
1174
1175 required_ucmd_sz = min_ucmd_sz +
1176 sizeof(struct mlx5_ib_flow_counters_data) *
1177 ucmd_hdr.ncounters_data;
1178 if (udata->inlen > required_ucmd_sz &&
1179 !ib_is_udata_cleared(udata, required_ucmd_sz,
1180 udata->inlen - required_ucmd_sz))
1181 return ERR_PTR(-EOPNOTSUPP);
1182
1183 ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
1184 if (!ucmd)
1185 return ERR_PTR(-ENOMEM);
1186
1187 err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
1188 if (err)
1189 goto free_ucmd;
1190 }
1191
1192 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
1193 err = -ENOMEM;
1194 goto free_ucmd;
1195 }
1196
1197 if (flow_attr->port > dev->num_ports ||
1198 (flow_attr->flags &
1199 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS))) {
1200 err = -EINVAL;
1201 goto free_ucmd;
1202 }
1203
1204 if (is_egress &&
1205 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1206 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1207 err = -EINVAL;
1208 goto free_ucmd;
1209 }
1210
1211 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1212 if (!dst) {
1213 err = -ENOMEM;
1214 goto free_ucmd;
1215 }
1216
1217 mutex_lock(&dev->flow_db->lock);
1218
1219 ft_prio = get_flow_table(dev, flow_attr,
1220 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
1221 if (IS_ERR(ft_prio)) {
1222 err = PTR_ERR(ft_prio);
1223 goto unlock;
1224 }
1225 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
1226 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
1227 if (IS_ERR(ft_prio_tx)) {
1228 err = PTR_ERR(ft_prio_tx);
1229 ft_prio_tx = NULL;
1230 goto destroy_ft;
1231 }
1232 }
1233
1234 if (is_egress) {
1235 dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
1236 } else {
1237 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1238 if (mqp->is_rss)
1239 dst->tir_num = mqp->rss_qp.tirn;
1240 else
1241 dst->tir_num = mqp->raw_packet_qp.rq.tirn;
1242 }
1243
1244 switch (flow_attr->type) {
1245 case IB_FLOW_ATTR_NORMAL:
1246 underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
1247 mqp->underlay_qpn :
1248 0;
1249 handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
1250 underlay_qpn, ucmd);
1251 break;
1252 case IB_FLOW_ATTR_ALL_DEFAULT:
1253 case IB_FLOW_ATTR_MC_DEFAULT:
1254 handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);
1255 break;
1256 case IB_FLOW_ATTR_SNIFFER:
1257 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
1258 break;
1259 default:
1260 err = -EINVAL;
1261 goto destroy_ft;
1262 }
1263
1264 if (IS_ERR(handler)) {
1265 err = PTR_ERR(handler);
1266 handler = NULL;
1267 goto destroy_ft;
1268 }
1269
1270 mutex_unlock(&dev->flow_db->lock);
1271 kfree(dst);
1272 kfree(ucmd);
1273
1274 return &handler->ibflow;
1275
1276destroy_ft:
1277 put_flow_table(dev, ft_prio, false);
1278 if (ft_prio_tx)
1279 put_flow_table(dev, ft_prio_tx, false);
1280unlock:
1281 mutex_unlock(&dev->flow_db->lock);
1282 kfree(dst);
1283free_ucmd:
1284 kfree(ucmd);
1285 return ERR_PTR(err);
1286}
1287
1288static struct mlx5_ib_flow_prio *
1289_get_flow_table(struct mlx5_ib_dev *dev,
1290 struct mlx5_ib_flow_matcher *fs_matcher,
1291 bool mcast)
1292{
1293 struct mlx5_flow_namespace *ns = NULL;
1294 struct mlx5_ib_flow_prio *prio = NULL;
1295 int max_table_size = 0;
1296 bool esw_encap;
1297 u32 flags = 0;
1298 int priority;
1299
1300 if (mcast)
1301 priority = MLX5_IB_FLOW_MCAST_PRIO;
1302 else
1303 priority = ib_prio_to_core_prio(fs_matcher->priority, false);
1304
1305 esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
1306 DEVLINK_ESWITCH_ENCAP_MODE_NONE;
1307 switch (fs_matcher->ns_type) {
1308 case MLX5_FLOW_NAMESPACE_BYPASS:
1309 max_table_size = BIT(
1310 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size));
1311 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
1312 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1313 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1314 reformat_l3_tunnel_to_l2) &&
1315 !esw_encap)
1316 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1317 break;
1318 case MLX5_FLOW_NAMESPACE_EGRESS:
1319 max_table_size = BIT(
1320 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
1321 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) &&
1322 !esw_encap)
1323 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1324 break;
1325 case MLX5_FLOW_NAMESPACE_FDB:
1326 max_table_size = BIT(
1327 MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
1328 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
1329 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1330 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev,
1331 reformat_l3_tunnel_to_l2) &&
1332 esw_encap)
1333 flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1334 priority = FDB_BYPASS_PATH;
1335 break;
1336 case MLX5_FLOW_NAMESPACE_RDMA_RX:
1337 max_table_size = BIT(
1338 MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size));
1339 priority = fs_matcher->priority;
1340 break;
1341 case MLX5_FLOW_NAMESPACE_RDMA_TX:
1342 max_table_size = BIT(
1343 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size));
1344 priority = fs_matcher->priority;
1345 break;
1346 default:
1347 break;
1348 }
1349
1350 max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
1351
1352 ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type);
1353 if (!ns)
1354 return ERR_PTR(-EOPNOTSUPP);
1355
1356 switch (fs_matcher->ns_type) {
1357 case MLX5_FLOW_NAMESPACE_BYPASS:
1358 prio = &dev->flow_db->prios[priority];
1359 break;
1360 case MLX5_FLOW_NAMESPACE_EGRESS:
1361 prio = &dev->flow_db->egress_prios[priority];
1362 break;
1363 case MLX5_FLOW_NAMESPACE_FDB:
1364 prio = &dev->flow_db->fdb;
1365 break;
1366 case MLX5_FLOW_NAMESPACE_RDMA_RX:
1367 prio = &dev->flow_db->rdma_rx[priority];
1368 break;
1369 case MLX5_FLOW_NAMESPACE_RDMA_TX:
1370 prio = &dev->flow_db->rdma_tx[priority];
1371 break;
1372 default: return ERR_PTR(-EINVAL);
1373 }
1374
1375 if (!prio)
1376 return ERR_PTR(-EINVAL);
1377
1378 if (prio->flow_table)
1379 return prio;
1380
1381 return _get_prio(ns, prio, priority, max_table_size,
1382 MLX5_FS_MAX_TYPES, flags);
1383}
1384
1385static struct mlx5_ib_flow_handler *
1386_create_raw_flow_rule(struct mlx5_ib_dev *dev,
1387 struct mlx5_ib_flow_prio *ft_prio,
1388 struct mlx5_flow_destination *dst,
1389 struct mlx5_ib_flow_matcher *fs_matcher,
1390 struct mlx5_flow_context *flow_context,
1391 struct mlx5_flow_act *flow_act,
1392 void *cmd_in, int inlen,
1393 int dst_num)
1394{
1395 struct mlx5_ib_flow_handler *handler;
1396 struct mlx5_flow_spec *spec;
1397 struct mlx5_flow_table *ft = ft_prio->flow_table;
1398 int err = 0;
1399
1400 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1401 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
1402 if (!handler || !spec) {
1403 err = -ENOMEM;
1404 goto free;
1405 }
1406
1407 INIT_LIST_HEAD(&handler->list);
1408
1409 memcpy(spec->match_value, cmd_in, inlen);
1410 memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
1411 fs_matcher->mask_len);
1412 spec->match_criteria_enable = fs_matcher->match_criteria_enable;
1413 spec->flow_context = *flow_context;
1414
1415 handler->rule = mlx5_add_flow_rules(ft, spec,
1416 flow_act, dst, dst_num);
1417
1418 if (IS_ERR(handler->rule)) {
1419 err = PTR_ERR(handler->rule);
1420 goto free;
1421 }
1422
1423 ft_prio->refcount++;
1424 handler->prio = ft_prio;
1425 handler->dev = dev;
1426 ft_prio->flow_table = ft;
1427
1428free:
1429 if (err)
1430 kfree(handler);
1431 kvfree(spec);
1432 return err ? ERR_PTR(err) : handler;
1433}
1434
1435static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
1436 void *match_v)
1437{
1438 void *match_c;
1439 void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
1440 void *dmac, *dmac_mask;
1441 void *ipv4, *ipv4_mask;
1442
1443 if (!(fs_matcher->match_criteria_enable &
1444 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
1445 return false;
1446
1447 match_c = fs_matcher->matcher_mask.match_params;
1448 match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
1449 outer_headers);
1450 match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
1451 outer_headers);
1452
1453 dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
1454 dmac_47_16);
1455 dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
1456 dmac_47_16);
1457
1458 if (is_multicast_ether_addr(dmac) &&
1459 is_multicast_ether_addr(dmac_mask))
1460 return true;
1461
1462 ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
1463 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
1464
1465 ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
1466 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
1467
1468 if (ipv4_is_multicast(*(__be32 *)(ipv4)) &&
1469 ipv4_is_multicast(*(__be32 *)(ipv4_mask)))
1470 return true;
1471
1472 return false;
1473}
1474
1475static struct mlx5_ib_flow_handler *raw_fs_rule_add(
1476 struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
1477 struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,
1478 u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type)
1479{
1480 struct mlx5_flow_destination *dst;
1481 struct mlx5_ib_flow_prio *ft_prio;
1482 struct mlx5_ib_flow_handler *handler;
1483 int dst_num = 0;
1484 bool mcast;
1485 int err;
1486
1487 if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
1488 return ERR_PTR(-EOPNOTSUPP);
1489
1490 if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
1491 return ERR_PTR(-ENOMEM);
1492
1493 dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
1494 if (!dst)
1495 return ERR_PTR(-ENOMEM);
1496
1497 mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
1498 mutex_lock(&dev->flow_db->lock);
1499
1500 ft_prio = _get_flow_table(dev, fs_matcher, mcast);
1501 if (IS_ERR(ft_prio)) {
1502 err = PTR_ERR(ft_prio);
1503 goto unlock;
1504 }
1505
1506 switch (dest_type) {
1507 case MLX5_FLOW_DESTINATION_TYPE_TIR:
1508 dst[dst_num].type = dest_type;
1509 dst[dst_num++].tir_num = dest_id;
1510 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1511 break;
1512 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
1513 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
1514 dst[dst_num++].ft_num = dest_id;
1515 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1516 break;
1517 case MLX5_FLOW_DESTINATION_TYPE_PORT:
1518 dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
1519 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1520 break;
1521 default:
1522 break;
1523 }
1524
1525 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1526 dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1527 dst[dst_num].counter_id = counter_id;
1528 dst_num++;
1529 }
1530
1531 handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher,
1532 flow_context, flow_act,
1533 cmd_in, inlen, dst_num);
1534
1535 if (IS_ERR(handler)) {
1536 err = PTR_ERR(handler);
1537 goto destroy_ft;
1538 }
1539
1540 mutex_unlock(&dev->flow_db->lock);
1541 atomic_inc(&fs_matcher->usecnt);
1542 handler->flow_matcher = fs_matcher;
1543
1544 kfree(dst);
1545
1546 return handler;
1547
1548destroy_ft:
1549 put_flow_table(dev, ft_prio, false);
1550unlock:
1551 mutex_unlock(&dev->flow_db->lock);
1552 kfree(dst);
1553
1554 return ERR_PTR(err);
1555}
1556
1557static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags)
1558{
1559 u32 flags = 0;
1560
1561 if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA)
1562 flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA;
1563
1564 return flags;
1565}
1566
1567#define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED \
1568 MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA
1569static struct ib_flow_action *
1570mlx5_ib_create_flow_action_esp(struct ib_device *device,
1571 const struct ib_flow_action_attrs_esp *attr,
1572 struct uverbs_attr_bundle *attrs)
1573{
1574 struct mlx5_ib_dev *mdev = to_mdev(device);
1575 struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm;
1576 struct mlx5_accel_esp_xfrm_attrs accel_attrs = {};
1577 struct mlx5_ib_flow_action *action;
1578 u64 action_flags;
1579 u64 flags;
1580 int err = 0;
1581
1582 err = uverbs_get_flags64(
1583 &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS,
1584 ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1));
1585 if (err)
1586 return ERR_PTR(err);
1587
1588 flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags);
1589
1590
1591
1592
1593
1594 if (!attr->keymat || attr->replay || attr->encap ||
1595 attr->spi || attr->seq || attr->tfc_pad ||
1596 attr->hard_limit_pkts ||
1597 (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
1598 IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)))
1599 return ERR_PTR(-EOPNOTSUPP);
1600
1601 if (attr->keymat->protocol !=
1602 IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM)
1603 return ERR_PTR(-EOPNOTSUPP);
1604
1605 aes_gcm = &attr->keymat->keymat.aes_gcm;
1606
1607 if (aes_gcm->icv_len != 16 ||
1608 aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
1609 return ERR_PTR(-EOPNOTSUPP);
1610
1611 action = kmalloc(sizeof(*action), GFP_KERNEL);
1612 if (!action)
1613 return ERR_PTR(-ENOMEM);
1614
1615 action->esp_aes_gcm.ib_flags = attr->flags;
1616 memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key,
1617 sizeof(accel_attrs.keymat.aes_gcm.aes_key));
1618 accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8;
1619 memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt,
1620 sizeof(accel_attrs.keymat.aes_gcm.salt));
1621 memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv,
1622 sizeof(accel_attrs.keymat.aes_gcm.seq_iv));
1623 accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8;
1624 accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ;
1625 accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;
1626
1627 accel_attrs.esn = attr->esn;
1628 if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED)
1629 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
1630 if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
1631 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
1632
1633 if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT)
1634 accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT;
1635
1636 action->esp_aes_gcm.ctx =
1637 mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags);
1638 if (IS_ERR(action->esp_aes_gcm.ctx)) {
1639 err = PTR_ERR(action->esp_aes_gcm.ctx);
1640 goto err_parse;
1641 }
1642
1643 action->esp_aes_gcm.ib_flags = attr->flags;
1644
1645 return &action->ib_action;
1646
1647err_parse:
1648 kfree(action);
1649 return ERR_PTR(err);
1650}
1651
1652static int
1653mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action,
1654 const struct ib_flow_action_attrs_esp *attr,
1655 struct uverbs_attr_bundle *attrs)
1656{
1657 struct mlx5_ib_flow_action *maction = to_mflow_act(action);
1658 struct mlx5_accel_esp_xfrm_attrs accel_attrs;
1659 int err = 0;
1660
1661 if (attr->keymat || attr->replay || attr->encap ||
1662 attr->spi || attr->seq || attr->tfc_pad ||
1663 attr->hard_limit_pkts ||
1664 (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
1665 IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS |
1666 IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)))
1667 return -EOPNOTSUPP;
1668
1669
1670
1671
1672 if (!(maction->esp_aes_gcm.ib_flags &
1673 IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) &&
1674 attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED |
1675 IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))
1676 return -EINVAL;
1677
1678 memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs,
1679 sizeof(accel_attrs));
1680
1681 accel_attrs.esn = attr->esn;
1682 if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)
1683 accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
1684 else
1685 accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
1686
1687 err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx,
1688 &accel_attrs);
1689 if (err)
1690 return err;
1691
1692 maction->esp_aes_gcm.ib_flags &=
1693 ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
1694 maction->esp_aes_gcm.ib_flags |=
1695 attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW;
1696
1697 return 0;
1698}
1699
1700static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
1701{
1702 switch (maction->flow_action_raw.sub_type) {
1703 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
1704 mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
1705 maction->flow_action_raw.modify_hdr);
1706 break;
1707 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
1708 mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
1709 maction->flow_action_raw.pkt_reformat);
1710 break;
1711 case MLX5_IB_FLOW_ACTION_DECAP:
1712 break;
1713 default:
1714 break;
1715 }
1716}
1717
1718static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
1719{
1720 struct mlx5_ib_flow_action *maction = to_mflow_act(action);
1721
1722 switch (action->type) {
1723 case IB_FLOW_ACTION_ESP:
1724
1725
1726
1727
1728 mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx);
1729 break;
1730 case IB_FLOW_ACTION_UNSPECIFIED:
1731 destroy_flow_action_raw(maction);
1732 break;
1733 default:
1734 WARN_ON(true);
1735 break;
1736 }
1737
1738 kfree(maction);
1739 return 0;
1740}
1741
1742static int
1743mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
1744 enum mlx5_flow_namespace_type *namespace)
1745{
1746 switch (table_type) {
1747 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
1748 *namespace = MLX5_FLOW_NAMESPACE_BYPASS;
1749 break;
1750 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
1751 *namespace = MLX5_FLOW_NAMESPACE_EGRESS;
1752 break;
1753 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
1754 *namespace = MLX5_FLOW_NAMESPACE_FDB;
1755 break;
1756 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
1757 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
1758 break;
1759 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
1760 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
1761 break;
1762 default:
1763 return -EINVAL;
1764 }
1765
1766 return 0;
1767}
1768
1769static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
1770 [MLX5_IB_FLOW_TYPE_NORMAL] = {
1771 .type = UVERBS_ATTR_TYPE_PTR_IN,
1772 .u.ptr = {
1773 .len = sizeof(u16),
1774 .min_len = sizeof(u16),
1775 }
1776 },
1777 [MLX5_IB_FLOW_TYPE_SNIFFER] = {
1778 .type = UVERBS_ATTR_TYPE_PTR_IN,
1779 UVERBS_ATTR_NO_DATA(),
1780 },
1781 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
1782 .type = UVERBS_ATTR_TYPE_PTR_IN,
1783 UVERBS_ATTR_NO_DATA(),
1784 },
1785 [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
1786 .type = UVERBS_ATTR_TYPE_PTR_IN,
1787 UVERBS_ATTR_NO_DATA(),
1788 },
1789};
1790
1791static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)
1792{
1793 struct devx_obj *devx_obj = obj;
1794 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
1795
1796 switch (opcode) {
1797 case MLX5_CMD_OP_DESTROY_TIR:
1798 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1799 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
1800 obj_id);
1801 return true;
1802
1803 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
1804 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1805 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
1806 table_id);
1807 return true;
1808 default:
1809 return false;
1810 }
1811}
1812
1813static int get_dests(struct uverbs_attr_bundle *attrs,
1814 struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
1815 int *dest_type, struct ib_qp **qp, u32 *flags)
1816{
1817 bool dest_devx, dest_qp;
1818 void *devx_obj;
1819 int err;
1820
1821 dest_devx = uverbs_attr_is_valid(attrs,
1822 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
1823 dest_qp = uverbs_attr_is_valid(attrs,
1824 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
1825
1826 *flags = 0;
1827 err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
1828 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
1829 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
1830 if (err)
1831 return err;
1832
1833
1834 if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
1835 *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
1836 return -EINVAL;
1837
1838 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
1839 if (dest_devx && (dest_qp || *flags))
1840 return -EINVAL;
1841 else if (dest_qp && *flags)
1842 return -EINVAL;
1843 }
1844
1845
1846 if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx ||
1847 (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
1848 return -EINVAL;
1849
1850
1851 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
1852 ((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
1853 return -EINVAL;
1854
1855 *qp = NULL;
1856 if (dest_devx) {
1857 devx_obj =
1858 uverbs_attr_get_obj(attrs,
1859 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
1860
1861
1862
1863
1864 if (!is_flow_dest(devx_obj, dest_id, dest_type))
1865 return -EINVAL;
1866
1867 if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB ||
1868 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
1869 *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
1870 return -EINVAL;
1871 } else if (dest_qp) {
1872 struct mlx5_ib_qp *mqp;
1873
1874 *qp = uverbs_attr_get_obj(attrs,
1875 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
1876 if (IS_ERR(*qp))
1877 return PTR_ERR(*qp);
1878
1879 if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
1880 return -EINVAL;
1881
1882 mqp = to_mqp(*qp);
1883 if (mqp->is_rss)
1884 *dest_id = mqp->rss_qp.tirn;
1885 else
1886 *dest_id = mqp->raw_packet_qp.rq.tirn;
1887 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1888 } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
1889 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
1890 *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
1891 }
1892
1893 if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
1894 (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
1895 fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX))
1896 return -EINVAL;
1897
1898 return 0;
1899}
1900
1901static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id)
1902{
1903 struct devx_obj *devx_obj = obj;
1904 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
1905
1906 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
1907
1908 if (offset && offset >= devx_obj->flow_counter_bulk_size)
1909 return false;
1910
1911 *counter_id = MLX5_GET(dealloc_flow_counter_in,
1912 devx_obj->dinbox,
1913 flow_counter_id);
1914 *counter_id += offset;
1915 return true;
1916 }
1917
1918 return false;
1919}
1920
1921#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
1922static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
1923 struct uverbs_attr_bundle *attrs)
1924{
1925 struct mlx5_flow_context flow_context = {.flow_tag =
1926 MLX5_FS_DEFAULT_FLOW_TAG};
1927 u32 *offset_attr, offset = 0, counter_id = 0;
1928 int dest_id, dest_type = -1, inlen, len, ret, i;
1929 struct mlx5_ib_flow_handler *flow_handler;
1930 struct mlx5_ib_flow_matcher *fs_matcher;
1931 struct ib_uobject **arr_flow_actions;
1932 struct ib_uflow_resources *uflow_res;
1933 struct mlx5_flow_act flow_act = {};
1934 struct ib_qp *qp = NULL;
1935 void *devx_obj, *cmd_in;
1936 struct ib_uobject *uobj;
1937 struct mlx5_ib_dev *dev;
1938 u32 flags;
1939
1940 if (!capable(CAP_NET_RAW))
1941 return -EPERM;
1942
1943 fs_matcher = uverbs_attr_get_obj(attrs,
1944 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
1945 uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
1946 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1947
1948 if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
1949 return -EINVAL;
1950
1951 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
1952 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
1953
1954 if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
1955 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
1956
1957 len = uverbs_attr_get_uobjs_arr(attrs,
1958 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
1959 if (len) {
1960 devx_obj = arr_flow_actions[0]->object;
1961
1962 if (uverbs_attr_is_valid(attrs,
1963 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
1964
1965 int num_offsets = uverbs_attr_ptr_get_array_size(
1966 attrs,
1967 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
1968 sizeof(u32));
1969
1970 if (num_offsets != 1)
1971 return -EINVAL;
1972
1973 offset_attr = uverbs_attr_get_alloced_ptr(
1974 attrs,
1975 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
1976 offset = *offset_attr;
1977 }
1978
1979 if (!is_flow_counter(devx_obj, offset, &counter_id))
1980 return -EINVAL;
1981
1982 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
1983 }
1984
1985 cmd_in = uverbs_attr_get_alloced_ptr(
1986 attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
1987 inlen = uverbs_attr_get_len(attrs,
1988 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
1989
1990 uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
1991 if (!uflow_res)
1992 return -ENOMEM;
1993
1994 len = uverbs_attr_get_uobjs_arr(attrs,
1995 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
1996 for (i = 0; i < len; i++) {
1997 struct mlx5_ib_flow_action *maction =
1998 to_mflow_act(arr_flow_actions[i]->object);
1999
2000 ret = parse_flow_flow_action(maction, false, &flow_act);
2001 if (ret)
2002 goto err_out;
2003 flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
2004 arr_flow_actions[i]->object);
2005 }
2006
2007 ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
2008 MLX5_IB_ATTR_CREATE_FLOW_TAG);
2009 if (!ret) {
2010 if (flow_context.flow_tag >= BIT(24)) {
2011 ret = -EINVAL;
2012 goto err_out;
2013 }
2014 flow_context.flags |= FLOW_CONTEXT_HAS_TAG;
2015 }
2016
2017 flow_handler =
2018 raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act,
2019 counter_id, cmd_in, inlen, dest_id, dest_type);
2020 if (IS_ERR(flow_handler)) {
2021 ret = PTR_ERR(flow_handler);
2022 goto err_out;
2023 }
2024
2025 ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
2026
2027 return 0;
2028err_out:
2029 ib_uverbs_flow_resources_free(uflow_res);
2030 return ret;
2031}
2032
2033static int flow_matcher_cleanup(struct ib_uobject *uobject,
2034 enum rdma_remove_reason why,
2035 struct uverbs_attr_bundle *attrs)
2036{
2037 struct mlx5_ib_flow_matcher *obj = uobject->object;
2038 int ret;
2039
2040 ret = ib_destroy_usecnt(&obj->usecnt, why, uobject);
2041 if (ret)
2042 return ret;
2043
2044 kfree(obj);
2045 return 0;
2046}
2047
2048static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
2049 struct mlx5_ib_flow_matcher *obj)
2050{
2051 enum mlx5_ib_uapi_flow_table_type ft_type =
2052 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
2053 u32 flags;
2054 int err;
2055
2056
2057
2058
2059 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
2060 uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
2061 return -EINVAL;
2062
2063 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
2064 err = uverbs_get_const(&ft_type, attrs,
2065 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
2066 if (err)
2067 return err;
2068
2069 err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
2070 if (err)
2071 return err;
2072
2073 return 0;
2074 }
2075
2076 if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
2077 err = uverbs_get_flags32(&flags, attrs,
2078 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
2079 IB_FLOW_ATTR_FLAGS_EGRESS);
2080 if (err)
2081 return err;
2082
2083 if (flags) {
2084 mlx5_ib_ft_type_to_namespace(
2085 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
2086 &obj->ns_type);
2087 return 0;
2088 }
2089 }
2090
2091 obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
2092
2093 return 0;
2094}
2095
2096static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
2097 struct uverbs_attr_bundle *attrs)
2098{
2099 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2100 attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
2101 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2102 struct mlx5_ib_flow_matcher *obj;
2103 int err;
2104
2105 obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
2106 if (!obj)
2107 return -ENOMEM;
2108
2109 obj->mask_len = uverbs_attr_get_len(
2110 attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2111 err = uverbs_copy_from(&obj->matcher_mask,
2112 attrs,
2113 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2114 if (err)
2115 goto end;
2116
2117 obj->flow_type = uverbs_attr_get_enum_id(
2118 attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2119
2120 if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
2121 err = uverbs_copy_from(&obj->priority,
2122 attrs,
2123 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2124 if (err)
2125 goto end;
2126 }
2127
2128 err = uverbs_copy_from(&obj->match_criteria_enable,
2129 attrs,
2130 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
2131 if (err)
2132 goto end;
2133
2134 err = mlx5_ib_matcher_ns(attrs, obj);
2135 if (err)
2136 goto end;
2137
2138 uobj->object = obj;
2139 obj->mdev = dev->mdev;
2140 atomic_set(&obj->usecnt, 0);
2141 return 0;
2142
2143end:
2144 kfree(obj);
2145 return err;
2146}
2147
2148static struct ib_flow_action *
2149mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
2150 enum mlx5_ib_uapi_flow_table_type ft_type,
2151 u8 num_actions, void *in)
2152{
2153 enum mlx5_flow_namespace_type namespace;
2154 struct mlx5_ib_flow_action *maction;
2155 int ret;
2156
2157 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
2158 if (ret)
2159 return ERR_PTR(-EINVAL);
2160
2161 maction = kzalloc(sizeof(*maction), GFP_KERNEL);
2162 if (!maction)
2163 return ERR_PTR(-ENOMEM);
2164
2165 maction->flow_action_raw.modify_hdr =
2166 mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
2167
2168 if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
2169 ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
2170 kfree(maction);
2171 return ERR_PTR(ret);
2172 }
2173 maction->flow_action_raw.sub_type =
2174 MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
2175 maction->flow_action_raw.dev = dev;
2176
2177 return &maction->ib_action;
2178}
2179
2180static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
2181{
2182 return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
2183 max_modify_header_actions) ||
2184 MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
2185 max_modify_header_actions) ||
2186 MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
2187 max_modify_header_actions);
2188}
2189
2190static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
2191 struct uverbs_attr_bundle *attrs)
2192{
2193 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2194 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
2195 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
2196 enum mlx5_ib_uapi_flow_table_type ft_type;
2197 struct ib_flow_action *action;
2198 int num_actions;
2199 void *in;
2200 int ret;
2201
2202 if (!mlx5_ib_modify_header_supported(mdev))
2203 return -EOPNOTSUPP;
2204
2205 in = uverbs_attr_get_alloced_ptr(attrs,
2206 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
2207
2208 num_actions = uverbs_attr_ptr_get_array_size(
2209 attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
2210 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
2211 if (num_actions < 0)
2212 return num_actions;
2213
2214 ret = uverbs_get_const(&ft_type, attrs,
2215 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
2216 if (ret)
2217 return ret;
2218 action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
2219 if (IS_ERR(action))
2220 return PTR_ERR(action);
2221
2222 uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
2223 IB_FLOW_ACTION_UNSPECIFIED);
2224
2225 return 0;
2226}
2227
2228static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
2229 u8 packet_reformat_type,
2230 u8 ft_type)
2231{
2232 switch (packet_reformat_type) {
2233 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
2234 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
2235 return MLX5_CAP_FLOWTABLE(ibdev->mdev,
2236 encap_general_header);
2237 break;
2238 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
2239 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
2240 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
2241 reformat_l2_to_l3_tunnel);
2242 break;
2243 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
2244 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
2245 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
2246 reformat_l3_tunnel_to_l2);
2247 break;
2248 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
2249 if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
2250 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
2251 break;
2252 default:
2253 break;
2254 }
2255
2256 return false;
2257}
2258
2259static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
2260{
2261 switch (dv_prt) {
2262 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
2263 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
2264 break;
2265 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
2266 *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
2267 break;
2268 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
2269 *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
2270 break;
2271 default:
2272 return -EINVAL;
2273 }
2274
2275 return 0;
2276}
2277
2278static int mlx5_ib_flow_action_create_packet_reformat_ctx(
2279 struct mlx5_ib_dev *dev,
2280 struct mlx5_ib_flow_action *maction,
2281 u8 ft_type, u8 dv_prt,
2282 void *in, size_t len)
2283{
2284 enum mlx5_flow_namespace_type namespace;
2285 u8 prm_prt;
2286 int ret;
2287
2288 ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
2289 if (ret)
2290 return ret;
2291
2292 ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
2293 if (ret)
2294 return ret;
2295
2296 maction->flow_action_raw.pkt_reformat =
2297 mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len,
2298 in, namespace);
2299 if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
2300 ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
2301 return ret;
2302 }
2303
2304 maction->flow_action_raw.sub_type =
2305 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
2306 maction->flow_action_raw.dev = dev;
2307
2308 return 0;
2309}
2310
2311static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
2312 struct uverbs_attr_bundle *attrs)
2313{
2314 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
2315 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
2316 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
2317 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
2318 enum mlx5_ib_uapi_flow_table_type ft_type;
2319 struct mlx5_ib_flow_action *maction;
2320 int ret;
2321
2322 ret = uverbs_get_const(&ft_type, attrs,
2323 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
2324 if (ret)
2325 return ret;
2326
2327 ret = uverbs_get_const(&dv_prt, attrs,
2328 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
2329 if (ret)
2330 return ret;
2331
2332 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
2333 return -EOPNOTSUPP;
2334
2335 maction = kzalloc(sizeof(*maction), GFP_KERNEL);
2336 if (!maction)
2337 return -ENOMEM;
2338
2339 if (dv_prt ==
2340 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
2341 maction->flow_action_raw.sub_type =
2342 MLX5_IB_FLOW_ACTION_DECAP;
2343 maction->flow_action_raw.dev = mdev;
2344 } else {
2345 void *in;
2346 int len;
2347
2348 in = uverbs_attr_get_alloced_ptr(attrs,
2349 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
2350 if (IS_ERR(in)) {
2351 ret = PTR_ERR(in);
2352 goto free_maction;
2353 }
2354
2355 len = uverbs_attr_get_len(attrs,
2356 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
2357
2358 ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
2359 maction, ft_type, dv_prt, in, len);
2360 if (ret)
2361 goto free_maction;
2362 }
2363
2364 uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
2365 IB_FLOW_ACTION_UNSPECIFIED);
2366 return 0;
2367
2368free_maction:
2369 kfree(maction);
2370 return ret;
2371}
2372
2373DECLARE_UVERBS_NAMED_METHOD(
2374 MLX5_IB_METHOD_CREATE_FLOW,
2375 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
2376 UVERBS_OBJECT_FLOW,
2377 UVERBS_ACCESS_NEW,
2378 UA_MANDATORY),
2379 UVERBS_ATTR_PTR_IN(
2380 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
2381 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
2382 UA_MANDATORY,
2383 UA_ALLOC_AND_COPY),
2384 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
2385 MLX5_IB_OBJECT_FLOW_MATCHER,
2386 UVERBS_ACCESS_READ,
2387 UA_MANDATORY),
2388 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
2389 UVERBS_OBJECT_QP,
2390 UVERBS_ACCESS_READ),
2391 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
2392 MLX5_IB_OBJECT_DEVX_OBJ,
2393 UVERBS_ACCESS_READ),
2394 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
2395 UVERBS_OBJECT_FLOW_ACTION,
2396 UVERBS_ACCESS_READ, 1,
2397 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
2398 UA_OPTIONAL),
2399 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
2400 UVERBS_ATTR_TYPE(u32),
2401 UA_OPTIONAL),
2402 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
2403 MLX5_IB_OBJECT_DEVX_OBJ,
2404 UVERBS_ACCESS_READ, 1, 1,
2405 UA_OPTIONAL),
2406 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
2407 UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
2408 UA_OPTIONAL,
2409 UA_ALLOC_AND_COPY),
2410 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
2411 enum mlx5_ib_create_flow_flags,
2412 UA_OPTIONAL));
2413
2414DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2415 MLX5_IB_METHOD_DESTROY_FLOW,
2416 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
2417 UVERBS_OBJECT_FLOW,
2418 UVERBS_ACCESS_DESTROY,
2419 UA_MANDATORY));
2420
2421ADD_UVERBS_METHODS(mlx5_ib_fs,
2422 UVERBS_OBJECT_FLOW,
2423 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
2424 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
2425
2426DECLARE_UVERBS_NAMED_METHOD(
2427 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
2428 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
2429 UVERBS_OBJECT_FLOW_ACTION,
2430 UVERBS_ACCESS_NEW,
2431 UA_MANDATORY),
2432 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
2433 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
2434 set_add_copy_action_in_auto)),
2435 UA_MANDATORY,
2436 UA_ALLOC_AND_COPY),
2437 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
2438 enum mlx5_ib_uapi_flow_table_type,
2439 UA_MANDATORY));
2440
2441DECLARE_UVERBS_NAMED_METHOD(
2442 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
2443 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
2444 UVERBS_OBJECT_FLOW_ACTION,
2445 UVERBS_ACCESS_NEW,
2446 UA_MANDATORY),
2447 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
2448 UVERBS_ATTR_MIN_SIZE(1),
2449 UA_ALLOC_AND_COPY,
2450 UA_OPTIONAL),
2451 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
2452 enum mlx5_ib_uapi_flow_action_packet_reformat_type,
2453 UA_MANDATORY),
2454 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
2455 enum mlx5_ib_uapi_flow_table_type,
2456 UA_MANDATORY));
2457
2458ADD_UVERBS_METHODS(
2459 mlx5_ib_flow_actions,
2460 UVERBS_OBJECT_FLOW_ACTION,
2461 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
2462 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
2463
2464DECLARE_UVERBS_NAMED_METHOD(
2465 MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
2466 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
2467 MLX5_IB_OBJECT_FLOW_MATCHER,
2468 UVERBS_ACCESS_NEW,
2469 UA_MANDATORY),
2470 UVERBS_ATTR_PTR_IN(
2471 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
2472 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
2473 UA_MANDATORY),
2474 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
2475 mlx5_ib_flow_type,
2476 UA_MANDATORY),
2477 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
2478 UVERBS_ATTR_TYPE(u8),
2479 UA_MANDATORY),
2480 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
2481 enum ib_flow_flags,
2482 UA_OPTIONAL),
2483 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
2484 enum mlx5_ib_uapi_flow_table_type,
2485 UA_OPTIONAL));
2486
2487DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2488 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
2489 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
2490 MLX5_IB_OBJECT_FLOW_MATCHER,
2491 UVERBS_ACCESS_DESTROY,
2492 UA_MANDATORY));
2493
2494DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
2495 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
2496 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
2497 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
2498
2499const struct uapi_definition mlx5_ib_flow_defs[] = {
2500 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2501 MLX5_IB_OBJECT_FLOW_MATCHER),
2502 UAPI_DEF_CHAIN_OBJ_TREE(
2503 UVERBS_OBJECT_FLOW,
2504 &mlx5_ib_fs),
2505 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
2506 &mlx5_ib_flow_actions),
2507 {},
2508};
2509
2510static const struct ib_device_ops flow_ops = {
2511 .create_flow = mlx5_ib_create_flow,
2512 .destroy_flow = mlx5_ib_destroy_flow,
2513 .destroy_flow_action = mlx5_ib_destroy_flow_action,
2514};
2515
2516static const struct ib_device_ops flow_ipsec_ops = {
2517 .create_flow_action_esp = mlx5_ib_create_flow_action_esp,
2518 .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp,
2519};
2520
2521int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
2522{
2523 dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
2524
2525 if (!dev->flow_db)
2526 return -ENOMEM;
2527
2528 mutex_init(&dev->flow_db->lock);
2529
2530 ib_set_device_ops(&dev->ib_dev, &flow_ops);
2531 if (mlx5_accel_ipsec_device_caps(dev->mdev) &
2532 MLX5_ACCEL_IPSEC_CAP_DEVICE)
2533 ib_set_device_ops(&dev->ib_dev, &flow_ipsec_ops);
2534
2535 return 0;
2536}
2537
