1
2
3
4#include <rte_string_fns.h>
5#include <rte_reorder.h>
6#include <rte_cryptodev.h>
7#include <cryptodev_pmd.h>
8#include <rte_malloc.h>
9
10#include "rte_cryptodev_scheduler.h"
11#include "scheduler_pmd_private.h"
12
13
14
15
16
17
18static uint32_t
19sync_caps(struct rte_cryptodev_capabilities *caps,
20 uint32_t nb_caps,
21 const struct rte_cryptodev_capabilities *worker_caps)
22{
23 uint32_t sync_nb_caps = nb_caps, nb_worker_caps = 0;
24 uint32_t i;
25
26 while (worker_caps[nb_worker_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED)
27 nb_worker_caps++;
28
29 if (nb_caps == 0) {
30 rte_memcpy(caps, worker_caps, sizeof(*caps) * nb_worker_caps);
31 return nb_worker_caps;
32 }
33
34 for (i = 0; i < sync_nb_caps; i++) {
35 struct rte_cryptodev_capabilities *cap = &caps[i];
36 uint32_t j;
37
38 for (j = 0; j < nb_worker_caps; j++) {
39 const struct rte_cryptodev_capabilities *s_cap =
40 &worker_caps[j];
41
42 if (s_cap->op != cap->op || s_cap->sym.xform_type !=
43 cap->sym.xform_type)
44 continue;
45
46 if (s_cap->sym.xform_type ==
47 RTE_CRYPTO_SYM_XFORM_AUTH) {
48 if (s_cap->sym.auth.algo !=
49 cap->sym.auth.algo)
50 continue;
51
52 cap->sym.auth.digest_size.min =
53 s_cap->sym.auth.digest_size.min <
54 cap->sym.auth.digest_size.min ?
55 s_cap->sym.auth.digest_size.min :
56 cap->sym.auth.digest_size.min;
57 cap->sym.auth.digest_size.max =
58 s_cap->sym.auth.digest_size.max <
59 cap->sym.auth.digest_size.max ?
60 s_cap->sym.auth.digest_size.max :
61 cap->sym.auth.digest_size.max;
62
63 }
64
65 if (s_cap->sym.xform_type ==
66 RTE_CRYPTO_SYM_XFORM_CIPHER)
67 if (s_cap->sym.cipher.algo !=
68 cap->sym.cipher.algo)
69 continue;
70
71
72 break;
73 }
74
75 if (j < nb_worker_caps)
76 continue;
77
78
79 for (j = i; j < sync_nb_caps - 1; j++)
80 rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap));
81
82 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap));
83 sync_nb_caps--;
84 }
85
86 return sync_nb_caps;
87}
88
89static int
90update_scheduler_capability(struct scheduler_ctx *sched_ctx)
91{
92 struct rte_cryptodev_capabilities tmp_caps[256] = { {0} };
93 uint32_t nb_caps = 0, i;
94
95 if (sched_ctx->capabilities) {
96 rte_free(sched_ctx->capabilities);
97 sched_ctx->capabilities = NULL;
98 }
99
100 for (i = 0; i < sched_ctx->nb_workers; i++) {
101 struct rte_cryptodev_info dev_info;
102
103 rte_cryptodev_info_get(sched_ctx->workers[i].dev_id, &dev_info);
104
105 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities);
106 if (nb_caps == 0)
107 return -1;
108 }
109
110 sched_ctx->capabilities = rte_zmalloc_socket(NULL,
111 sizeof(struct rte_cryptodev_capabilities) *
112 (nb_caps + 1), 0, SOCKET_ID_ANY);
113 if (!sched_ctx->capabilities)
114 return -ENOMEM;
115
116 rte_memcpy(sched_ctx->capabilities, tmp_caps,
117 sizeof(struct rte_cryptodev_capabilities) * nb_caps);
118
119 return 0;
120}
121
122static void
123update_scheduler_feature_flag(struct rte_cryptodev *dev)
124{
125 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
126 uint32_t i;
127
128 dev->feature_flags = 0;
129
130 for (i = 0; i < sched_ctx->nb_workers; i++) {
131 struct rte_cryptodev_info dev_info;
132
133 rte_cryptodev_info_get(sched_ctx->workers[i].dev_id, &dev_info);
134
135 dev->feature_flags |= dev_info.feature_flags;
136 }
137}
138
139static void
140update_max_nb_qp(struct scheduler_ctx *sched_ctx)
141{
142 uint32_t i;
143 uint32_t max_nb_qp;
144
145 if (!sched_ctx->nb_workers)
146 return;
147
148 max_nb_qp = sched_ctx->nb_workers ? UINT32_MAX : 0;
149
150 for (i = 0; i < sched_ctx->nb_workers; i++) {
151 struct rte_cryptodev_info dev_info;
152
153 rte_cryptodev_info_get(sched_ctx->workers[i].dev_id, &dev_info);
154 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ?
155 dev_info.max_nb_queue_pairs : max_nb_qp;
156 }
157
158 sched_ctx->max_nb_queue_pairs = max_nb_qp;
159}
160
161
162int
163rte_cryptodev_scheduler_worker_attach(uint8_t scheduler_id, uint8_t worker_id)
164{
165 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
166 struct scheduler_ctx *sched_ctx;
167 struct scheduler_worker *worker;
168 struct rte_cryptodev_info dev_info;
169 uint32_t i;
170
171 if (!dev) {
172 CR_SCHED_LOG(ERR, "Operation not supported");
173 return -ENOTSUP;
174 }
175
176 if (dev->driver_id != cryptodev_scheduler_driver_id) {
177 CR_SCHED_LOG(ERR, "Operation not supported");
178 return -ENOTSUP;
179 }
180
181 if (dev->data->dev_started) {
182 CR_SCHED_LOG(ERR, "Illegal operation");
183 return -EBUSY;
184 }
185
186 sched_ctx = dev->data->dev_private;
187 if (sched_ctx->nb_workers >=
188 RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) {
189 CR_SCHED_LOG(ERR, "Too many workers attached");
190 return -ENOMEM;
191 }
192
193 for (i = 0; i < sched_ctx->nb_workers; i++)
194 if (sched_ctx->workers[i].dev_id == worker_id) {
195 CR_SCHED_LOG(ERR, "Worker already added");
196 return -ENOTSUP;
197 }
198
199 worker = &sched_ctx->workers[sched_ctx->nb_workers];
200
201 rte_cryptodev_info_get(worker_id, &dev_info);
202
203 worker->dev_id = worker_id;
204 worker->driver_id = dev_info.driver_id;
205 sched_ctx->nb_workers++;
206
207 if (update_scheduler_capability(sched_ctx) < 0) {
208 worker->dev_id = 0;
209 worker->driver_id = 0;
210 sched_ctx->nb_workers--;
211
212 CR_SCHED_LOG(ERR, "capabilities update failed");
213 return -ENOTSUP;
214 }
215
216 update_scheduler_feature_flag(dev);
217
218 update_max_nb_qp(sched_ctx);
219
220 return 0;
221}
222
223int
224rte_cryptodev_scheduler_worker_detach(uint8_t scheduler_id, uint8_t worker_id)
225{
226 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
227 struct scheduler_ctx *sched_ctx;
228 uint32_t i, worker_pos;
229
230 if (!dev) {
231 CR_SCHED_LOG(ERR, "Operation not supported");
232 return -ENOTSUP;
233 }
234
235 if (dev->driver_id != cryptodev_scheduler_driver_id) {
236 CR_SCHED_LOG(ERR, "Operation not supported");
237 return -ENOTSUP;
238 }
239
240 if (dev->data->dev_started) {
241 CR_SCHED_LOG(ERR, "Illegal operation");
242 return -EBUSY;
243 }
244
245 sched_ctx = dev->data->dev_private;
246
247 for (worker_pos = 0; worker_pos < sched_ctx->nb_workers; worker_pos++)
248 if (sched_ctx->workers[worker_pos].dev_id == worker_id)
249 break;
250 if (worker_pos == sched_ctx->nb_workers) {
251 CR_SCHED_LOG(ERR, "Cannot find worker");
252 return -ENOTSUP;
253 }
254
255 if (sched_ctx->ops.worker_detach(dev, worker_id) < 0) {
256 CR_SCHED_LOG(ERR, "Failed to detach worker");
257 return -ENOTSUP;
258 }
259
260 for (i = worker_pos; i < sched_ctx->nb_workers - 1; i++) {
261 memcpy(&sched_ctx->workers[i], &sched_ctx->workers[i+1],
262 sizeof(struct scheduler_worker));
263 }
264 memset(&sched_ctx->workers[sched_ctx->nb_workers - 1], 0,
265 sizeof(struct scheduler_worker));
266 sched_ctx->nb_workers--;
267
268 if (update_scheduler_capability(sched_ctx) < 0) {
269 CR_SCHED_LOG(ERR, "capabilities update failed");
270 return -ENOTSUP;
271 }
272
273 update_scheduler_feature_flag(dev);
274
275 update_max_nb_qp(sched_ctx);
276
277 return 0;
278}
279
280int
281rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
282 enum rte_cryptodev_scheduler_mode mode)
283{
284 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
285 struct scheduler_ctx *sched_ctx;
286
287 if (!dev) {
288 CR_SCHED_LOG(ERR, "Operation not supported");
289 return -ENOTSUP;
290 }
291
292 if (dev->driver_id != cryptodev_scheduler_driver_id) {
293 CR_SCHED_LOG(ERR, "Operation not supported");
294 return -ENOTSUP;
295 }
296
297 if (dev->data->dev_started) {
298 CR_SCHED_LOG(ERR, "Illegal operation");
299 return -EBUSY;
300 }
301
302 sched_ctx = dev->data->dev_private;
303
304 if (mode == sched_ctx->mode)
305 return 0;
306
307 switch (mode) {
308 case CDEV_SCHED_MODE_ROUNDROBIN:
309 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
310 crypto_scheduler_roundrobin) < 0) {
311 CR_SCHED_LOG(ERR, "Failed to load scheduler");
312 return -1;
313 }
314 break;
315 case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
316 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
317 crypto_scheduler_pkt_size_based_distr) < 0) {
318 CR_SCHED_LOG(ERR, "Failed to load scheduler");
319 return -1;
320 }
321 break;
322 case CDEV_SCHED_MODE_FAILOVER:
323 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
324 crypto_scheduler_failover) < 0) {
325 CR_SCHED_LOG(ERR, "Failed to load scheduler");
326 return -1;
327 }
328 break;
329 case CDEV_SCHED_MODE_MULTICORE:
330 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
331 crypto_scheduler_multicore) < 0) {
332 CR_SCHED_LOG(ERR, "Failed to load scheduler");
333 return -1;
334 }
335 break;
336 default:
337 CR_SCHED_LOG(ERR, "Not yet supported");
338 return -ENOTSUP;
339 }
340
341 return 0;
342}
343
344enum rte_cryptodev_scheduler_mode
345rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id)
346{
347 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
348 struct scheduler_ctx *sched_ctx;
349
350 if (!dev) {
351 CR_SCHED_LOG(ERR, "Operation not supported");
352 return -ENOTSUP;
353 }
354
355 if (dev->driver_id != cryptodev_scheduler_driver_id) {
356 CR_SCHED_LOG(ERR, "Operation not supported");
357 return -ENOTSUP;
358 }
359
360 sched_ctx = dev->data->dev_private;
361
362 return sched_ctx->mode;
363}
364
365int
366rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id,
367 uint32_t enable_reorder)
368{
369 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
370 struct scheduler_ctx *sched_ctx;
371
372 if (!dev) {
373 CR_SCHED_LOG(ERR, "Operation not supported");
374 return -ENOTSUP;
375 }
376
377 if (dev->driver_id != cryptodev_scheduler_driver_id) {
378 CR_SCHED_LOG(ERR, "Operation not supported");
379 return -ENOTSUP;
380 }
381
382 if (dev->data->dev_started) {
383 CR_SCHED_LOG(ERR, "Illegal operation");
384 return -EBUSY;
385 }
386
387 sched_ctx = dev->data->dev_private;
388
389 sched_ctx->reordering_enabled = enable_reorder;
390
391 return 0;
392}
393
394int
395rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id)
396{
397 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
398 struct scheduler_ctx *sched_ctx;
399
400 if (!dev) {
401 CR_SCHED_LOG(ERR, "Operation not supported");
402 return -ENOTSUP;
403 }
404
405 if (dev->driver_id != cryptodev_scheduler_driver_id) {
406 CR_SCHED_LOG(ERR, "Operation not supported");
407 return -ENOTSUP;
408 }
409
410 sched_ctx = dev->data->dev_private;
411
412 return (int)sched_ctx->reordering_enabled;
413}
414
415int
416rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
417 struct rte_cryptodev_scheduler *scheduler) {
418
419 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
420 struct scheduler_ctx *sched_ctx;
421
422 if (!dev) {
423 CR_SCHED_LOG(ERR, "Operation not supported");
424 return -ENOTSUP;
425 }
426
427 if (dev->driver_id != cryptodev_scheduler_driver_id) {
428 CR_SCHED_LOG(ERR, "Operation not supported");
429 return -ENOTSUP;
430 }
431
432 if (dev->data->dev_started) {
433 CR_SCHED_LOG(ERR, "Illegal operation");
434 return -EBUSY;
435 }
436
437 sched_ctx = dev->data->dev_private;
438
439 if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
440 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
441 "%u bytes.", scheduler->name,
442 RTE_CRYPTODEV_NAME_MAX_LEN);
443 return -EINVAL;
444 }
445 strlcpy(sched_ctx->name, scheduler->name, sizeof(sched_ctx->name));
446
447 if (strlen(scheduler->description) >
448 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) {
449 CR_SCHED_LOG(ERR, "Invalid description %s, should be less than "
450 "%u bytes.", scheduler->description,
451 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1);
452 return -EINVAL;
453 }
454 strlcpy(sched_ctx->description, scheduler->description,
455 sizeof(sched_ctx->description));
456
457
458 sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair;
459 sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx;
460 sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start;
461 sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop;
462 sched_ctx->ops.worker_attach = scheduler->ops->worker_attach;
463 sched_ctx->ops.worker_detach = scheduler->ops->worker_detach;
464 sched_ctx->ops.option_set = scheduler->ops->option_set;
465 sched_ctx->ops.option_get = scheduler->ops->option_get;
466
467 if (sched_ctx->private_ctx) {
468 rte_free(sched_ctx->private_ctx);
469 sched_ctx->private_ctx = NULL;
470 }
471
472 if (sched_ctx->ops.create_private_ctx) {
473 int ret = (*sched_ctx->ops.create_private_ctx)(dev);
474
475 if (ret < 0) {
476 CR_SCHED_LOG(ERR, "Unable to create scheduler private "
477 "context");
478 return ret;
479 }
480 }
481
482 sched_ctx->mode = scheduler->mode;
483
484 return 0;
485}
486
487int
488rte_cryptodev_scheduler_workers_get(uint8_t scheduler_id, uint8_t *workers)
489{
490 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
491 struct scheduler_ctx *sched_ctx;
492 uint32_t nb_workers = 0;
493
494 if (!dev) {
495 CR_SCHED_LOG(ERR, "Operation not supported");
496 return -ENOTSUP;
497 }
498
499 if (dev->driver_id != cryptodev_scheduler_driver_id) {
500 CR_SCHED_LOG(ERR, "Operation not supported");
501 return -ENOTSUP;
502 }
503
504 sched_ctx = dev->data->dev_private;
505
506 nb_workers = sched_ctx->nb_workers;
507
508 if (workers && nb_workers) {
509 uint32_t i;
510
511 for (i = 0; i < nb_workers; i++)
512 workers[i] = sched_ctx->workers[i].dev_id;
513 }
514
515 return (int)nb_workers;
516}
517
518int
519rte_cryptodev_scheduler_option_set(uint8_t scheduler_id,
520 enum rte_cryptodev_schedule_option_type option_type,
521 void *option)
522{
523 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
524 struct scheduler_ctx *sched_ctx;
525
526 if (option_type == CDEV_SCHED_OPTION_NOT_SET ||
527 option_type >= CDEV_SCHED_OPTION_COUNT) {
528 CR_SCHED_LOG(ERR, "Invalid option parameter");
529 return -EINVAL;
530 }
531
532 if (!option) {
533 CR_SCHED_LOG(ERR, "Invalid option parameter");
534 return -EINVAL;
535 }
536
537 if (dev->data->dev_started) {
538 CR_SCHED_LOG(ERR, "Illegal operation");
539 return -EBUSY;
540 }
541
542 sched_ctx = dev->data->dev_private;
543
544 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP);
545
546 return (*sched_ctx->ops.option_set)(dev, option_type, option);
547}
548
549int
550rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
551 enum rte_cryptodev_schedule_option_type option_type,
552 void *option)
553{
554 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
555 struct scheduler_ctx *sched_ctx;
556
557 if (!dev) {
558 CR_SCHED_LOG(ERR, "Operation not supported");
559 return -ENOTSUP;
560 }
561
562 if (!option) {
563 CR_SCHED_LOG(ERR, "Invalid option parameter");
564 return -EINVAL;
565 }
566
567 if (dev->driver_id != cryptodev_scheduler_driver_id) {
568 CR_SCHED_LOG(ERR, "Operation not supported");
569 return -ENOTSUP;
570 }
571
572 sched_ctx = dev->data->dev_private;
573
574 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP);
575
576 return (*sched_ctx->ops.option_get)(dev, option_type, option);
577}
578
579
580RTE_LOG_REGISTER_DEFAULT(scheduler_logtype_driver, INFO);
581
