1
2
3
4
5#include <rte_cryptodev.h>
6#include <rte_malloc.h>
7
8#include "rte_cryptodev_scheduler_operations.h"
9#include "scheduler_pmd_private.h"
10
11#define DEF_PKT_SIZE_THRESHOLD (0xffffff80)
12#define WORKER_IDX_SWITCH_MASK (0x01)
13#define PRIMARY_WORKER_IDX 0
14#define SECONDARY_WORKER_IDX 1
15#define NB_PKT_SIZE_WORKERS 2
16
17
18struct psd_scheduler_ctx {
19 uint32_t threshold;
20};
21
22
23struct psd_scheduler_qp_ctx {
24 struct scheduler_worker primary_worker;
25 struct scheduler_worker secondary_worker;
26 uint32_t threshold;
27 uint8_t deq_idx;
28} __rte_cache_aligned;
29
30
31struct psd_schedule_op {
32 uint8_t worker_idx;
33 uint16_t pos;
34};
35
36static uint16_t
37schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
38{
39 struct scheduler_qp_ctx *qp_ctx = qp;
40 struct psd_scheduler_qp_ctx *psd_qp_ctx = qp_ctx->private_qp_ctx;
41 struct rte_crypto_op *sched_ops[NB_PKT_SIZE_WORKERS][nb_ops];
42 uint32_t in_flight_ops[NB_PKT_SIZE_WORKERS] = {
43 psd_qp_ctx->primary_worker.nb_inflight_cops,
44 psd_qp_ctx->secondary_worker.nb_inflight_cops
45 };
46 struct psd_schedule_op enq_ops[NB_PKT_SIZE_WORKERS] = {
47 {PRIMARY_WORKER_IDX, 0}, {SECONDARY_WORKER_IDX, 0}
48 };
49 struct psd_schedule_op *p_enq_op;
50 uint16_t i, processed_ops_pri = 0, processed_ops_sec = 0;
51 uint32_t job_len;
52
53 if (unlikely(nb_ops == 0))
54 return 0;
55
56 for (i = 0; i < nb_ops && i < 4; i++) {
57 rte_prefetch0(ops[i]->sym);
58 rte_prefetch0(ops[i]->sym->session);
59 }
60
61 for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) {
62 rte_prefetch0(ops[i + 4]->sym);
63 rte_prefetch0(ops[i + 4]->sym->session);
64 rte_prefetch0(ops[i + 5]->sym);
65 rte_prefetch0(ops[i + 5]->sym->session);
66 rte_prefetch0(ops[i + 6]->sym);
67 rte_prefetch0(ops[i + 6]->sym->session);
68 rte_prefetch0(ops[i + 7]->sym);
69 rte_prefetch0(ops[i + 7]->sym->session);
70
71
72
73
74 job_len = ops[i]->sym->cipher.data.length;
75 job_len += (ops[i]->sym->cipher.data.length == 0) *
76 ops[i]->sym->auth.data.length;
77
78 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
79
80
81
82
83 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==
84 qp_ctx->max_nb_objs) {
85 i = nb_ops;
86 break;
87 }
88
89 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i];
90 p_enq_op->pos++;
91
92 job_len = ops[i+1]->sym->cipher.data.length;
93 job_len += (ops[i+1]->sym->cipher.data.length == 0) *
94 ops[i+1]->sym->auth.data.length;
95 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
96
97 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==
98 qp_ctx->max_nb_objs) {
99 i = nb_ops;
100 break;
101 }
102
103 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i+1];
104 p_enq_op->pos++;
105
106 job_len = ops[i+2]->sym->cipher.data.length;
107 job_len += (ops[i+2]->sym->cipher.data.length == 0) *
108 ops[i+2]->sym->auth.data.length;
109 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
110
111 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==
112 qp_ctx->max_nb_objs) {
113 i = nb_ops;
114 break;
115 }
116
117 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i+2];
118 p_enq_op->pos++;
119
120 job_len = ops[i+3]->sym->cipher.data.length;
121 job_len += (ops[i+3]->sym->cipher.data.length == 0) *
122 ops[i+3]->sym->auth.data.length;
123 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
124
125 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==
126 qp_ctx->max_nb_objs) {
127 i = nb_ops;
128 break;
129 }
130
131 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i+3];
132 p_enq_op->pos++;
133 }
134
135 for (; i < nb_ops; i++) {
136 job_len = ops[i]->sym->cipher.data.length;
137 job_len += (ops[i]->sym->cipher.data.length == 0) *
138 ops[i]->sym->auth.data.length;
139 p_enq_op = &enq_ops[!(job_len & psd_qp_ctx->threshold)];
140
141 if (p_enq_op->pos + in_flight_ops[p_enq_op->worker_idx] ==
142 qp_ctx->max_nb_objs) {
143 i = nb_ops;
144 break;
145 }
146
147 sched_ops[p_enq_op->worker_idx][p_enq_op->pos] = ops[i];
148 p_enq_op->pos++;
149 }
150
151 processed_ops_pri = rte_cryptodev_enqueue_burst(
152 psd_qp_ctx->primary_worker.dev_id,
153 psd_qp_ctx->primary_worker.qp_id,
154 sched_ops[PRIMARY_WORKER_IDX],
155 enq_ops[PRIMARY_WORKER_IDX].pos);
156
157 RTE_ASSERT(processed_ops_pri == enq_ops[PRIMARY_WORKER_IDX].pos);
158
159 psd_qp_ctx->primary_worker.nb_inflight_cops += processed_ops_pri;
160
161 processed_ops_sec = rte_cryptodev_enqueue_burst(
162 psd_qp_ctx->secondary_worker.dev_id,
163 psd_qp_ctx->secondary_worker.qp_id,
164 sched_ops[SECONDARY_WORKER_IDX],
165 enq_ops[SECONDARY_WORKER_IDX].pos);
166 RTE_ASSERT(processed_ops_sec == enq_ops[SECONDARY_WORKER_IDX].pos);
167
168 psd_qp_ctx->secondary_worker.nb_inflight_cops += processed_ops_sec;
169
170 return processed_ops_pri + processed_ops_sec;
171}
172
173static uint16_t
174schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops,
175 uint16_t nb_ops)
176{
177 struct rte_ring *order_ring =
178 ((struct scheduler_qp_ctx *)qp)->order_ring;
179 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
180 nb_ops);
181 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
182 nb_ops_to_enq);
183
184 scheduler_order_insert(order_ring, ops, nb_ops_enqd);
185
186 return nb_ops_enqd;
187}
188
189static uint16_t
190schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
191{
192 struct psd_scheduler_qp_ctx *qp_ctx =
193 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
194 struct scheduler_worker *workers[NB_PKT_SIZE_WORKERS] = {
195 &qp_ctx->primary_worker, &qp_ctx->secondary_worker};
196 struct scheduler_worker *worker = workers[qp_ctx->deq_idx];
197 uint16_t nb_deq_ops_pri = 0, nb_deq_ops_sec = 0;
198
199 if (worker->nb_inflight_cops) {
200 nb_deq_ops_pri = rte_cryptodev_dequeue_burst(worker->dev_id,
201 worker->qp_id, ops, nb_ops);
202 worker->nb_inflight_cops -= nb_deq_ops_pri;
203 }
204
205 qp_ctx->deq_idx = (~qp_ctx->deq_idx) & WORKER_IDX_SWITCH_MASK;
206
207 if (nb_deq_ops_pri == nb_ops)
208 return nb_deq_ops_pri;
209
210 worker = workers[qp_ctx->deq_idx];
211
212 if (worker->nb_inflight_cops) {
213 nb_deq_ops_sec = rte_cryptodev_dequeue_burst(worker->dev_id,
214 worker->qp_id, &ops[nb_deq_ops_pri],
215 nb_ops - nb_deq_ops_pri);
216 worker->nb_inflight_cops -= nb_deq_ops_sec;
217
218 if (!worker->nb_inflight_cops)
219 qp_ctx->deq_idx = (~qp_ctx->deq_idx) &
220 WORKER_IDX_SWITCH_MASK;
221 }
222
223 return nb_deq_ops_pri + nb_deq_ops_sec;
224}
225
226static uint16_t
227schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops,
228 uint16_t nb_ops)
229{
230 struct rte_ring *order_ring =
231 ((struct scheduler_qp_ctx *)qp)->order_ring;
232
233 schedule_dequeue(qp, ops, nb_ops);
234
235 return scheduler_order_drain(order_ring, ops, nb_ops);
236}
237
238static int
239worker_attach(__rte_unused struct rte_cryptodev *dev,
240 __rte_unused uint8_t worker_id)
241{
242 return 0;
243}
244
245static int
246worker_detach(__rte_unused struct rte_cryptodev *dev,
247 __rte_unused uint8_t worker_id)
248{
249 return 0;
250}
251
252static int
253scheduler_start(struct rte_cryptodev *dev)
254{
255 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
256 struct psd_scheduler_ctx *psd_ctx = sched_ctx->private_ctx;
257 uint16_t i;
258
259
260 if (sched_ctx->nb_workers < NB_PKT_SIZE_WORKERS) {
261 CR_SCHED_LOG(ERR, "not enough workers to start");
262 return -1;
263 }
264
265 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
266 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
267 struct psd_scheduler_qp_ctx *ps_qp_ctx =
268 qp_ctx->private_qp_ctx;
269
270 ps_qp_ctx->primary_worker.dev_id =
271 sched_ctx->workers[PRIMARY_WORKER_IDX].dev_id;
272 ps_qp_ctx->primary_worker.qp_id = i;
273 ps_qp_ctx->primary_worker.nb_inflight_cops = 0;
274
275 ps_qp_ctx->secondary_worker.dev_id =
276 sched_ctx->workers[SECONDARY_WORKER_IDX].dev_id;
277 ps_qp_ctx->secondary_worker.qp_id = i;
278 ps_qp_ctx->secondary_worker.nb_inflight_cops = 0;
279
280 ps_qp_ctx->threshold = psd_ctx->threshold;
281 }
282
283 if (sched_ctx->reordering_enabled) {
284 dev->enqueue_burst = &schedule_enqueue_ordering;
285 dev->dequeue_burst = &schedule_dequeue_ordering;
286 } else {
287 dev->enqueue_burst = &schedule_enqueue;
288 dev->dequeue_burst = &schedule_dequeue;
289 }
290
291 return 0;
292}
293
294static int
295scheduler_stop(struct rte_cryptodev *dev)
296{
297 uint16_t i;
298
299 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
300 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
301 struct psd_scheduler_qp_ctx *ps_qp_ctx = qp_ctx->private_qp_ctx;
302
303 if (ps_qp_ctx->primary_worker.nb_inflight_cops +
304 ps_qp_ctx->secondary_worker.nb_inflight_cops) {
305 CR_SCHED_LOG(ERR, "Some crypto ops left in worker queue");
306 return -1;
307 }
308 }
309
310 return 0;
311}
312
313static int
314scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
315{
316 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
317 struct psd_scheduler_qp_ctx *ps_qp_ctx;
318
319 ps_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*ps_qp_ctx), 0,
320 rte_socket_id());
321 if (!ps_qp_ctx) {
322 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
323 return -ENOMEM;
324 }
325
326 qp_ctx->private_qp_ctx = (void *)ps_qp_ctx;
327
328 return 0;
329}
330
331static int
332scheduler_create_private_ctx(struct rte_cryptodev *dev)
333{
334 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
335 struct psd_scheduler_ctx *psd_ctx;
336
337 if (sched_ctx->private_ctx) {
338 rte_free(sched_ctx->private_ctx);
339 sched_ctx->private_ctx = NULL;
340 }
341
342 psd_ctx = rte_zmalloc_socket(NULL, sizeof(struct psd_scheduler_ctx), 0,
343 rte_socket_id());
344 if (!psd_ctx) {
345 CR_SCHED_LOG(ERR, "failed allocate memory");
346 return -ENOMEM;
347 }
348
349 psd_ctx->threshold = DEF_PKT_SIZE_THRESHOLD;
350
351 sched_ctx->private_ctx = (void *)psd_ctx;
352
353 return 0;
354}
355static int
356scheduler_option_set(struct rte_cryptodev *dev, uint32_t option_type,
357 void *option)
358{
359 struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *)
360 dev->data->dev_private)->private_ctx;
361 uint32_t threshold;
362
363 if ((enum rte_cryptodev_schedule_option_type)option_type !=
364 CDEV_SCHED_OPTION_THRESHOLD) {
365 CR_SCHED_LOG(ERR, "Option not supported");
366 return -EINVAL;
367 }
368
369 threshold = ((struct rte_cryptodev_scheduler_threshold_option *)
370 option)->threshold;
371 if (!rte_is_power_of_2(threshold)) {
372 CR_SCHED_LOG(ERR, "Threshold is not power of 2");
373 return -EINVAL;
374 }
375
376 psd_ctx->threshold = ~(threshold - 1);
377
378 return 0;
379}
380
381static int
382scheduler_option_get(struct rte_cryptodev *dev, uint32_t option_type,
383 void *option)
384{
385 struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *)
386 dev->data->dev_private)->private_ctx;
387 struct rte_cryptodev_scheduler_threshold_option *threshold_option;
388
389 if ((enum rte_cryptodev_schedule_option_type)option_type !=
390 CDEV_SCHED_OPTION_THRESHOLD) {
391 CR_SCHED_LOG(ERR, "Option not supported");
392 return -EINVAL;
393 }
394
395 threshold_option = option;
396 threshold_option->threshold = (~psd_ctx->threshold) + 1;
397
398 return 0;
399}
400
401static struct rte_cryptodev_scheduler_ops scheduler_ps_ops = {
402 worker_attach,
403 worker_detach,
404 scheduler_start,
405 scheduler_stop,
406 scheduler_config_qp,
407 scheduler_create_private_ctx,
408 scheduler_option_set,
409 scheduler_option_get
410};
411
412static struct rte_cryptodev_scheduler psd_scheduler = {
413 .name = "packet-size-based-scheduler",
414 .description = "scheduler which will distribute crypto op "
415 "burst based on the packet size",
416 .mode = CDEV_SCHED_MODE_PKT_SIZE_DISTR,
417 .ops = &scheduler_ps_ops
418};
419
420struct rte_cryptodev_scheduler *crypto_scheduler_pkt_size_based_distr = &psd_scheduler;
421
