1
2#include <linux/gfp.h>
3#include <linux/workqueue.h>
4#include <crypto/internal/skcipher.h>
5
6#include "nitrox_dev.h"
7#include "nitrox_req.h"
8#include "nitrox_csr.h"
9
10
11#define MIN_UDD_LEN 16
12
13#define FDATA_SIZE 32
14
15#define SOLICIT_BASE_DPORT 256
16
17#define REQ_NOT_POSTED 1
18#define REQ_BACKLOG 2
19#define REQ_POSTED 3
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44static inline int incr_index(int index, int count, int max)
45{
46 if ((index + count) >= max)
47 index = index + count - max;
48 else
49 index += count;
50
51 return index;
52}
53
54static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
55{
56 struct nitrox_device *ndev = sr->ndev;
57 struct device *dev = DEV(ndev);
58
59
60 dma_unmap_sg(dev, sr->in.sg, sr->in.sgmap_cnt, DMA_BIDIRECTIONAL);
61 dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
62 DMA_TO_DEVICE);
63 kfree(sr->in.sgcomp);
64 sr->in.sg = NULL;
65 sr->in.sgmap_cnt = 0;
66
67 dma_unmap_sg(dev, sr->out.sg, sr->out.sgmap_cnt,
68 DMA_BIDIRECTIONAL);
69 dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
70 DMA_TO_DEVICE);
71 kfree(sr->out.sgcomp);
72 sr->out.sg = NULL;
73 sr->out.sgmap_cnt = 0;
74}
75
76static void softreq_destroy(struct nitrox_softreq *sr)
77{
78 softreq_unmap_sgbufs(sr);
79 kfree(sr);
80}
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105static int create_sg_component(struct nitrox_softreq *sr,
106 struct nitrox_sgtable *sgtbl, int map_nents)
107{
108 struct nitrox_device *ndev = sr->ndev;
109 struct nitrox_sgcomp *sgcomp;
110 struct scatterlist *sg;
111 dma_addr_t dma;
112 size_t sz_comp;
113 int i, j, nr_sgcomp;
114
115 nr_sgcomp = roundup(map_nents, 4) / 4;
116
117
118 sz_comp = nr_sgcomp * sizeof(*sgcomp);
119 sgcomp = kzalloc(sz_comp, sr->gfp);
120 if (!sgcomp)
121 return -ENOMEM;
122
123 sgtbl->sgcomp = sgcomp;
124
125 sg = sgtbl->sg;
126
127 for (i = 0; i < nr_sgcomp; i++) {
128 for (j = 0; j < 4 && sg; j++) {
129 sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
130 sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
131 sg = sg_next(sg);
132 }
133 }
134
135 dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
136 if (dma_mapping_error(DEV(ndev), dma)) {
137 kfree(sgtbl->sgcomp);
138 sgtbl->sgcomp = NULL;
139 return -ENOMEM;
140 }
141
142 sgtbl->sgcomp_dma = dma;
143 sgtbl->sgcomp_len = sz_comp;
144
145 return 0;
146}
147
148
149
150
151
152
153
154
155
156static int dma_map_inbufs(struct nitrox_softreq *sr,
157 struct se_crypto_request *req)
158{
159 struct device *dev = DEV(sr->ndev);
160 struct scatterlist *sg = req->src;
161 int i, nents, ret = 0;
162
163 nents = dma_map_sg(dev, req->src, sg_nents(req->src),
164 DMA_BIDIRECTIONAL);
165 if (!nents)
166 return -EINVAL;
167
168 for_each_sg(req->src, sg, nents, i)
169 sr->in.total_bytes += sg_dma_len(sg);
170
171 sr->in.sg = req->src;
172 sr->in.sgmap_cnt = nents;
173 ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
174 if (ret)
175 goto incomp_err;
176
177 return 0;
178
179incomp_err:
180 dma_unmap_sg(dev, req->src, nents, DMA_BIDIRECTIONAL);
181 sr->in.sgmap_cnt = 0;
182 return ret;
183}
184
185static int dma_map_outbufs(struct nitrox_softreq *sr,
186 struct se_crypto_request *req)
187{
188 struct device *dev = DEV(sr->ndev);
189 int nents, ret = 0;
190
191 nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
192 DMA_BIDIRECTIONAL);
193 if (!nents)
194 return -EINVAL;
195
196 sr->out.sg = req->dst;
197 sr->out.sgmap_cnt = nents;
198 ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
199 if (ret)
200 goto outcomp_map_err;
201
202 return 0;
203
204outcomp_map_err:
205 dma_unmap_sg(dev, req->dst, nents, DMA_BIDIRECTIONAL);
206 sr->out.sgmap_cnt = 0;
207 sr->out.sg = NULL;
208 return ret;
209}
210
211static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
212 struct se_crypto_request *creq)
213{
214 int ret;
215
216 ret = dma_map_inbufs(sr, creq);
217 if (ret)
218 return ret;
219
220 ret = dma_map_outbufs(sr, creq);
221 if (ret)
222 softreq_unmap_sgbufs(sr);
223
224 return ret;
225}
226
227static inline void backlog_list_add(struct nitrox_softreq *sr,
228 struct nitrox_cmdq *cmdq)
229{
230 INIT_LIST_HEAD(&sr->backlog);
231
232 spin_lock_bh(&cmdq->backlog_qlock);
233 list_add_tail(&sr->backlog, &cmdq->backlog_head);
234 atomic_inc(&cmdq->backlog_count);
235 atomic_set(&sr->status, REQ_BACKLOG);
236 spin_unlock_bh(&cmdq->backlog_qlock);
237}
238
239static inline void response_list_add(struct nitrox_softreq *sr,
240 struct nitrox_cmdq *cmdq)
241{
242 INIT_LIST_HEAD(&sr->response);
243
244 spin_lock_bh(&cmdq->resp_qlock);
245 list_add_tail(&sr->response, &cmdq->response_head);
246 spin_unlock_bh(&cmdq->resp_qlock);
247}
248
249static inline void response_list_del(struct nitrox_softreq *sr,
250 struct nitrox_cmdq *cmdq)
251{
252 spin_lock_bh(&cmdq->resp_qlock);
253 list_del(&sr->response);
254 spin_unlock_bh(&cmdq->resp_qlock);
255}
256
257static struct nitrox_softreq *
258get_first_response_entry(struct nitrox_cmdq *cmdq)
259{
260 return list_first_entry_or_null(&cmdq->response_head,
261 struct nitrox_softreq, response);
262}
263
264static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
265{
266 if (atomic_inc_return(&cmdq->pending_count) > qlen) {
267 atomic_dec(&cmdq->pending_count);
268
269 smp_mb__after_atomic();
270 return true;
271 }
272
273 smp_mb__after_atomic();
274 return false;
275}
276
277
278
279
280
281
282
283
284static void post_se_instr(struct nitrox_softreq *sr,
285 struct nitrox_cmdq *cmdq)
286{
287 struct nitrox_device *ndev = sr->ndev;
288 int idx;
289 u8 *ent;
290
291 spin_lock_bh(&cmdq->cmd_qlock);
292
293 idx = cmdq->write_idx;
294
295 ent = cmdq->base + (idx * cmdq->instr_size);
296 memcpy(ent, &sr->instr, cmdq->instr_size);
297
298 atomic_set(&sr->status, REQ_POSTED);
299 response_list_add(sr, cmdq);
300 sr->tstamp = jiffies;
301
302 dma_wmb();
303
304
305 writeq(1, cmdq->dbell_csr_addr);
306
307 mmiowb();
308
309 cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
310
311 spin_unlock_bh(&cmdq->cmd_qlock);
312
313
314 atomic64_inc(&ndev->stats.posted);
315}
316
317static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
318{
319 struct nitrox_device *ndev = cmdq->ndev;
320 struct nitrox_softreq *sr, *tmp;
321 int ret = 0;
322
323 if (!atomic_read(&cmdq->backlog_count))
324 return 0;
325
326 spin_lock_bh(&cmdq->backlog_qlock);
327
328 list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
329
330 if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
331 ret = -ENOSPC;
332 break;
333 }
334
335 list_del(&sr->backlog);
336 atomic_dec(&cmdq->backlog_count);
337
338 smp_mb__after_atomic();
339
340
341 post_se_instr(sr, cmdq);
342 }
343 spin_unlock_bh(&cmdq->backlog_qlock);
344
345 return ret;
346}
347
348static int nitrox_enqueue_request(struct nitrox_softreq *sr)
349{
350 struct nitrox_cmdq *cmdq = sr->cmdq;
351 struct nitrox_device *ndev = sr->ndev;
352
353
354 post_backlog_cmds(cmdq);
355
356 if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
357 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
358
359 atomic64_inc(&ndev->stats.dropped);
360 return -ENOSPC;
361 }
362
363 backlog_list_add(sr, cmdq);
364 return -EINPROGRESS;
365 }
366 post_se_instr(sr, cmdq);
367
368 return -EINPROGRESS;
369}
370
371
372
373
374
375
376
377
378int nitrox_process_se_request(struct nitrox_device *ndev,
379 struct se_crypto_request *req,
380 completion_t callback,
381 void *cb_arg)
382{
383 struct nitrox_softreq *sr;
384 dma_addr_t ctx_handle = 0;
385 int qno, ret = 0;
386
387 if (!nitrox_ready(ndev))
388 return -ENODEV;
389
390 sr = kzalloc(sizeof(*sr), req->gfp);
391 if (!sr)
392 return -ENOMEM;
393
394 sr->ndev = ndev;
395 sr->flags = req->flags;
396 sr->gfp = req->gfp;
397 sr->callback = callback;
398 sr->cb_arg = cb_arg;
399
400 atomic_set(&sr->status, REQ_NOT_POSTED);
401
402 sr->resp.orh = req->orh;
403 sr->resp.completion = req->comp;
404
405 ret = softreq_map_iobuf(sr, req);
406 if (ret) {
407 kfree(sr);
408 return ret;
409 }
410
411
412 if (req->ctx_handle) {
413 struct ctx_hdr *hdr;
414 u8 *ctx_ptr;
415
416 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
417 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
418 ctx_handle = hdr->ctx_dma;
419 }
420
421
422 qno = smp_processor_id() % ndev->nr_queues;
423
424 sr->cmdq = &ndev->pkt_inq[qno];
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444 sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
445
446
447 sr->instr.ih.value = 0;
448 sr->instr.ih.s.g = 1;
449 sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
450 sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
451 sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
452 sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
453 sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
454
455
456 sr->instr.irh.value[0] = 0;
457 sr->instr.irh.s.uddl = MIN_UDD_LEN;
458
459 sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
460
461 sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
462 sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
463 sr->instr.irh.s.arg = req->ctrl.s.arg;
464 sr->instr.irh.s.opcode = req->opcode;
465 sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]);
466
467
468 sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
469
470
471 sr->instr.slc.value[0] = 0;
472 sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
473 sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
474
475
476 sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
477
478
479
480
481
482
483 sr->instr.fdata[0] = *((u64 *)&req->gph);
484 sr->instr.fdata[1] = 0;
485
486 ret = nitrox_enqueue_request(sr);
487 if (ret == -ENOSPC)
488 goto send_fail;
489
490 return ret;
491
492send_fail:
493 softreq_destroy(sr);
494 return ret;
495}
496
497static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
498{
499 return time_after_eq(jiffies, (tstamp + timeout));
500}
501
502void backlog_qflush_work(struct work_struct *work)
503{
504 struct nitrox_cmdq *cmdq;
505
506 cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
507 post_backlog_cmds(cmdq);
508}
509
510static bool sr_completed(struct nitrox_softreq *sr)
511{
512 u64 orh = READ_ONCE(*sr->resp.orh);
513 unsigned long timeout = jiffies + msecs_to_jiffies(1);
514
515 if ((orh != PENDING_SIG) && (orh & 0xff))
516 return true;
517
518 while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
519 if (time_after(jiffies, timeout)) {
520 pr_err("comp not done\n");
521 return false;
522 }
523 }
524
525 return true;
526}
527
528
529
530
531
532
533
534
535static void process_response_list(struct nitrox_cmdq *cmdq)
536{
537 struct nitrox_device *ndev = cmdq->ndev;
538 struct nitrox_softreq *sr;
539 int req_completed = 0, err = 0, budget;
540 completion_t callback;
541 void *cb_arg;
542
543
544 budget = atomic_read(&cmdq->pending_count);
545
546 while (req_completed < budget) {
547 sr = get_first_response_entry(cmdq);
548 if (!sr)
549 break;
550
551 if (atomic_read(&sr->status) != REQ_POSTED)
552 break;
553
554
555 if (!sr_completed(sr)) {
556
557 if (!cmd_timeout(sr->tstamp, ndev->timeout))
558 break;
559 dev_err_ratelimited(DEV(ndev),
560 "Request timeout, orh 0x%016llx\n",
561 READ_ONCE(*sr->resp.orh));
562 }
563 atomic_dec(&cmdq->pending_count);
564 atomic64_inc(&ndev->stats.completed);
565
566 smp_mb__after_atomic();
567
568 response_list_del(sr, cmdq);
569
570 err = READ_ONCE(*sr->resp.orh) & 0xff;
571 callback = sr->callback;
572 cb_arg = sr->cb_arg;
573 softreq_destroy(sr);
574 if (callback)
575 callback(cb_arg, err);
576
577 req_completed++;
578 }
579}
580
581
582
583
584void pkt_slc_resp_tasklet(unsigned long data)
585{
586 struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
587 struct nitrox_cmdq *cmdq = qvec->cmdq;
588 union nps_pkt_slc_cnts slc_cnts;
589
590
591 slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
592
593 slc_cnts.s.resend = 1;
594
595 process_response_list(cmdq);
596
597
598
599
600
601 writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
602
603 mmiowb();
604
605 if (atomic_read(&cmdq->backlog_count))
606 schedule_work(&cmdq->backlog_qflush);
607}
608