1
2
3
4
5
6
7
8
9#include "cptvf.h"
10#include "request_manager.h"
11
12
13
14
15
16
17static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
18 int qlen)
19{
20 struct pending_entry *ent = NULL;
21
22 ent = &q->head[q->rear];
23 if (unlikely(ent->busy)) {
24 ent = NULL;
25 goto no_free_entry;
26 }
27
28 q->rear++;
29 if (unlikely(q->rear == qlen))
30 q->rear = 0;
31
32no_free_entry:
33 return ent;
34}
35
36static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
37 int qno)
38{
39 struct pending_queue *queue = &pqinfo->queue[qno];
40
41 queue->front++;
42 if (unlikely(queue->front == pqinfo->qlen))
43 queue->front = 0;
44}
45
46static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
47 int buf_count, u8 *buffer)
48{
49 int ret = 0, i, j;
50 int components;
51 struct sglist_component *sg_ptr = NULL;
52 struct pci_dev *pdev = cptvf->pdev;
53
54 if (unlikely(!list)) {
55 dev_err(&pdev->dev, "Input List pointer is NULL\n");
56 return -EFAULT;
57 }
58
59 for (i = 0; i < buf_count; i++) {
60 if (likely(list[i].vptr)) {
61 list[i].dma_addr = dma_map_single(&pdev->dev,
62 list[i].vptr,
63 list[i].size,
64 DMA_BIDIRECTIONAL);
65 if (unlikely(dma_mapping_error(&pdev->dev,
66 list[i].dma_addr))) {
67 dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
68 i);
69 ret = -EIO;
70 goto sg_cleanup;
71 }
72 }
73 }
74
75 components = buf_count / 4;
76 sg_ptr = (struct sglist_component *)buffer;
77 for (i = 0; i < components; i++) {
78 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
79 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
80 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
81 sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
82 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
83 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
84 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
85 sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
86 sg_ptr++;
87 }
88
89 components = buf_count % 4;
90
91 switch (components) {
92 case 3:
93 sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
94 sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
95
96 case 2:
97 sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
98 sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
99
100 case 1:
101 sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
102 sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
103 break;
104 default:
105 break;
106 }
107
108 return ret;
109
110sg_cleanup:
111 for (j = 0; j < i; j++) {
112 if (list[j].dma_addr) {
113 dma_unmap_single(&pdev->dev, list[i].dma_addr,
114 list[i].size, DMA_BIDIRECTIONAL);
115 }
116
117 list[j].dma_addr = 0;
118 }
119
120 return ret;
121}
122
123static inline int setup_sgio_list(struct cpt_vf *cptvf,
124 struct cpt_info_buffer *info,
125 struct cpt_request_info *req)
126{
127 u16 g_sz_bytes = 0, s_sz_bytes = 0;
128 int ret = 0;
129 struct pci_dev *pdev = cptvf->pdev;
130
131 if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
132 dev_err(&pdev->dev, "Request SG components are higher than supported\n");
133 ret = -EINVAL;
134 goto scatter_gather_clean;
135 }
136
137
138 g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
139 info->gather_components = kzalloc(g_sz_bytes, GFP_KERNEL);
140 if (!info->gather_components) {
141 ret = -ENOMEM;
142 goto scatter_gather_clean;
143 }
144
145 ret = setup_sgio_components(cptvf, req->in,
146 req->incnt,
147 info->gather_components);
148 if (ret) {
149 dev_err(&pdev->dev, "Failed to setup gather list\n");
150 ret = -EFAULT;
151 goto scatter_gather_clean;
152 }
153
154
155 s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
156 info->scatter_components = kzalloc(s_sz_bytes, GFP_KERNEL);
157 if (!info->scatter_components) {
158 ret = -ENOMEM;
159 goto scatter_gather_clean;
160 }
161
162 ret = setup_sgio_components(cptvf, req->out,
163 req->outcnt,
164 info->scatter_components);
165 if (ret) {
166 dev_err(&pdev->dev, "Failed to setup gather list\n");
167 ret = -EFAULT;
168 goto scatter_gather_clean;
169 }
170
171
172 info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
173 info->in_buffer = kzalloc(info->dlen, GFP_KERNEL);
174 if (!info->in_buffer) {
175 ret = -ENOMEM;
176 goto scatter_gather_clean;
177 }
178
179 ((u16 *)info->in_buffer)[0] = req->outcnt;
180 ((u16 *)info->in_buffer)[1] = req->incnt;
181 ((u16 *)info->in_buffer)[2] = 0;
182 ((u16 *)info->in_buffer)[3] = 0;
183 *(u64 *)info->in_buffer = cpu_to_be64p((u64 *)info->in_buffer);
184
185 memcpy(&info->in_buffer[8], info->gather_components,
186 g_sz_bytes);
187 memcpy(&info->in_buffer[8 + g_sz_bytes],
188 info->scatter_components, s_sz_bytes);
189
190 info->dptr_baddr = dma_map_single(&pdev->dev,
191 (void *)info->in_buffer,
192 info->dlen,
193 DMA_BIDIRECTIONAL);
194 if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
195 dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
196 ret = -EIO;
197 goto scatter_gather_clean;
198 }
199
200
201 info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, GFP_KERNEL);
202 if (!info->out_buffer) {
203 ret = -ENOMEM;
204 goto scatter_gather_clean;
205 }
206
207 *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
208 info->alternate_caddr = (u64 *)info->out_buffer;
209 info->rptr_baddr = dma_map_single(&pdev->dev,
210 (void *)info->out_buffer,
211 COMPLETION_CODE_SIZE,
212 DMA_BIDIRECTIONAL);
213 if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
214 dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
215 COMPLETION_CODE_SIZE);
216 ret = -EIO;
217 goto scatter_gather_clean;
218 }
219
220 return 0;
221
222scatter_gather_clean:
223 return ret;
224}
225
226int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
227 u32 qno)
228{
229 struct pci_dev *pdev = cptvf->pdev;
230 struct command_qinfo *qinfo = NULL;
231 struct command_queue *queue;
232 struct command_chunk *chunk;
233 u8 *ent;
234 int ret = 0;
235
236 if (unlikely(qno >= cptvf->nr_queues)) {
237 dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
238 qno, cptvf->nr_queues);
239 return -EINVAL;
240 }
241
242 qinfo = &cptvf->cqinfo;
243 queue = &qinfo->queue[qno];
244
245 spin_lock(&queue->lock);
246 ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
247 memcpy(ent, (void *)cmd, qinfo->cmd_size);
248
249 if (++queue->idx >= queue->qhead->size / 64) {
250 struct hlist_node *node;
251
252 hlist_for_each(node, &queue->chead) {
253 chunk = hlist_entry(node, struct command_chunk,
254 nextchunk);
255 if (chunk == queue->qhead) {
256 continue;
257 } else {
258 queue->qhead = chunk;
259 break;
260 }
261 }
262 queue->idx = 0;
263 }
264
265 smp_wmb();
266 cptvf_write_vq_doorbell(cptvf, 1);
267
268 spin_unlock(&queue->lock);
269
270 return ret;
271}
272
273void do_request_cleanup(struct cpt_vf *cptvf,
274 struct cpt_info_buffer *info)
275{
276 int i;
277 struct pci_dev *pdev = cptvf->pdev;
278 struct cpt_request_info *req;
279
280 if (info->dptr_baddr)
281 dma_unmap_single(&pdev->dev, info->dptr_baddr,
282 info->dlen, DMA_BIDIRECTIONAL);
283
284 if (info->rptr_baddr)
285 dma_unmap_single(&pdev->dev, info->rptr_baddr,
286 COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
287
288 if (info->comp_baddr)
289 dma_unmap_single(&pdev->dev, info->comp_baddr,
290 sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
291
292 if (info->req) {
293 req = info->req;
294 for (i = 0; i < req->outcnt; i++) {
295 if (req->out[i].dma_addr)
296 dma_unmap_single(&pdev->dev,
297 req->out[i].dma_addr,
298 req->out[i].size,
299 DMA_BIDIRECTIONAL);
300 }
301
302 for (i = 0; i < req->incnt; i++) {
303 if (req->in[i].dma_addr)
304 dma_unmap_single(&pdev->dev,
305 req->in[i].dma_addr,
306 req->in[i].size,
307 DMA_BIDIRECTIONAL);
308 }
309 }
310
311 if (info->scatter_components)
312 kzfree(info->scatter_components);
313
314 if (info->gather_components)
315 kzfree(info->gather_components);
316
317 if (info->out_buffer)
318 kzfree(info->out_buffer);
319
320 if (info->in_buffer)
321 kzfree(info->in_buffer);
322
323 if (info->completion_addr)
324 kzfree((void *)info->completion_addr);
325
326 kzfree(info);
327}
328
329void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
330{
331 struct pci_dev *pdev = cptvf->pdev;
332
333 if (!info) {
334 dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
335 return;
336 }
337
338 do_request_cleanup(cptvf, info);
339}
340
341static inline void process_pending_queue(struct cpt_vf *cptvf,
342 struct pending_qinfo *pqinfo,
343 int qno)
344{
345 struct pci_dev *pdev = cptvf->pdev;
346 struct pending_queue *pqueue = &pqinfo->queue[qno];
347 struct pending_entry *pentry = NULL;
348 struct cpt_info_buffer *info = NULL;
349 union cpt_res_s *status = NULL;
350 unsigned char ccode;
351
352 while (1) {
353 spin_lock_bh(&pqueue->lock);
354 pentry = &pqueue->head[pqueue->front];
355 if (unlikely(!pentry->busy)) {
356 spin_unlock_bh(&pqueue->lock);
357 break;
358 }
359
360 info = (struct cpt_info_buffer *)pentry->post_arg;
361 if (unlikely(!info)) {
362 dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
363 pending_queue_inc_front(pqinfo, qno);
364 spin_unlock_bh(&pqueue->lock);
365 continue;
366 }
367
368 status = (union cpt_res_s *)pentry->completion_addr;
369 ccode = status->s.compcode;
370 if ((status->s.compcode == CPT_COMP_E_FAULT) ||
371 (status->s.compcode == CPT_COMP_E_SWERR)) {
372 dev_err(&pdev->dev, "Request failed with %s\n",
373 (status->s.compcode == CPT_COMP_E_FAULT) ?
374 "DMA Fault" : "Software error");
375 pentry->completion_addr = NULL;
376 pentry->busy = false;
377 atomic64_dec((&pqueue->pending_count));
378 pentry->post_arg = NULL;
379 pending_queue_inc_front(pqinfo, qno);
380 do_request_cleanup(cptvf, info);
381 spin_unlock_bh(&pqueue->lock);
382 break;
383 } else if (status->s.compcode == COMPLETION_CODE_INIT) {
384
385 if (time_after_eq(jiffies,
386 (info->time_in +
387 (CPT_COMMAND_TIMEOUT * HZ)))) {
388 dev_err(&pdev->dev, "Request timed out");
389 pentry->completion_addr = NULL;
390 pentry->busy = false;
391 atomic64_dec((&pqueue->pending_count));
392 pentry->post_arg = NULL;
393 pending_queue_inc_front(pqinfo, qno);
394 do_request_cleanup(cptvf, info);
395 spin_unlock_bh(&pqueue->lock);
396 break;
397 } else if ((*info->alternate_caddr ==
398 (~COMPLETION_CODE_INIT)) &&
399 (info->extra_time < TIME_IN_RESET_COUNT)) {
400 info->time_in = jiffies;
401 info->extra_time++;
402 spin_unlock_bh(&pqueue->lock);
403 break;
404 }
405 }
406
407 pentry->completion_addr = NULL;
408 pentry->busy = false;
409 pentry->post_arg = NULL;
410 atomic64_dec((&pqueue->pending_count));
411 pending_queue_inc_front(pqinfo, qno);
412 spin_unlock_bh(&pqueue->lock);
413
414 do_post_process(info->cptvf, info);
415
416
417
418
419 pentry->callback(ccode, pentry->callback_arg);
420 }
421}
422
423int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
424{
425 int ret = 0, clear = 0, queue = 0;
426 struct cpt_info_buffer *info = NULL;
427 struct cptvf_request *cpt_req = NULL;
428 union ctrl_info *ctrl = NULL;
429 union cpt_res_s *result = NULL;
430 struct pending_entry *pentry = NULL;
431 struct pending_queue *pqueue = NULL;
432 struct pci_dev *pdev = cptvf->pdev;
433 u8 group = 0;
434 struct cpt_vq_command vq_cmd;
435 union cpt_inst_s cptinst;
436
437 info = kzalloc(sizeof(*info), GFP_KERNEL);
438 if (unlikely(!info)) {
439 dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
440 return -ENOMEM;
441 }
442
443 cpt_req = (struct cptvf_request *)&req->req;
444 ctrl = (union ctrl_info *)&req->ctrl;
445
446 info->cptvf = cptvf;
447 group = ctrl->s.grp;
448 ret = setup_sgio_list(cptvf, info, req);
449 if (ret) {
450 dev_err(&pdev->dev, "Setting up SG list failed");
451 goto request_cleanup;
452 }
453
454 cpt_req->dlen = info->dlen;
455
456
457
458
459 info->completion_addr = kzalloc(sizeof(union cpt_res_s), GFP_KERNEL);
460 if (unlikely(!info->completion_addr)) {
461 dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
462 ret = -ENOMEM;
463 goto request_cleanup;
464 }
465
466 result = (union cpt_res_s *)info->completion_addr;
467 result->s.compcode = COMPLETION_CODE_INIT;
468 info->comp_baddr = dma_map_single(&pdev->dev,
469 (void *)info->completion_addr,
470 sizeof(union cpt_res_s),
471 DMA_BIDIRECTIONAL);
472 if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
473 dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
474 sizeof(union cpt_res_s));
475 ret = -EFAULT;
476 goto request_cleanup;
477 }
478
479
480 vq_cmd.cmd.u64 = 0;
481 vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
482 vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
483 vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
484 vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
485
486
487 vq_cmd.cmd.u64 = cpu_to_be64(vq_cmd.cmd.u64);
488 vq_cmd.dptr = info->dptr_baddr;
489 vq_cmd.rptr = info->rptr_baddr;
490 vq_cmd.cptr.u64 = 0;
491 vq_cmd.cptr.s.grp = group;
492
493
494 queue = 0;
495 pqueue = &cptvf->pqinfo.queue[queue];
496
497 if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
498 dev_err(&pdev->dev, "pending threshold reached\n");
499 process_pending_queue(cptvf, &cptvf->pqinfo, queue);
500 }
501
502get_pending_entry:
503 spin_lock_bh(&pqueue->lock);
504 pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
505 if (unlikely(!pentry)) {
506 spin_unlock_bh(&pqueue->lock);
507 if (clear == 0) {
508 process_pending_queue(cptvf, &cptvf->pqinfo, queue);
509 clear = 1;
510 goto get_pending_entry;
511 }
512 dev_err(&pdev->dev, "Get free entry failed\n");
513 dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
514 queue, pqueue->rear, pqueue->front);
515 ret = -EFAULT;
516 goto request_cleanup;
517 }
518
519 pentry->completion_addr = info->completion_addr;
520 pentry->post_arg = (void *)info;
521 pentry->callback = req->callback;
522 pentry->callback_arg = req->callback_arg;
523 info->pentry = pentry;
524 pentry->busy = true;
525 atomic64_inc(&pqueue->pending_count);
526
527
528 info->pentry = pentry;
529 info->time_in = jiffies;
530 info->req = req;
531
532
533 cptinst.s.doneint = true;
534 cptinst.s.res_addr = (u64)info->comp_baddr;
535 cptinst.s.tag = 0;
536 cptinst.s.grp = 0;
537 cptinst.s.wq_ptr = 0;
538 cptinst.s.ei0 = vq_cmd.cmd.u64;
539 cptinst.s.ei1 = vq_cmd.dptr;
540 cptinst.s.ei2 = vq_cmd.rptr;
541 cptinst.s.ei3 = vq_cmd.cptr.u64;
542
543 ret = send_cpt_command(cptvf, &cptinst, queue);
544 spin_unlock_bh(&pqueue->lock);
545 if (unlikely(ret)) {
546 dev_err(&pdev->dev, "Send command failed for AE\n");
547 ret = -EFAULT;
548 goto request_cleanup;
549 }
550
551 return 0;
552
553request_cleanup:
554 dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
555 do_request_cleanup(cptvf, info);
556
557 return ret;
558}
559
560void vq_post_process(struct cpt_vf *cptvf, u32 qno)
561{
562 struct pci_dev *pdev = cptvf->pdev;
563
564 if (unlikely(qno > cptvf->nr_queues)) {
565 dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
566 qno);
567 return;
568 }
569
570 process_pending_queue(cptvf, &cptvf->pqinfo, qno);
571}
572
573int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
574{
575 struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
576 struct pci_dev *pdev = cptvf->pdev;
577
578 if (!cpt_device_ready(cptvf)) {
579 dev_err(&pdev->dev, "CPT Device is not ready");
580 return -ENODEV;
581 }
582
583 if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
584 dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
585 cptvf->vfid);
586 return -EINVAL;
587 } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
588 dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
589 cptvf->vfid);
590 return -EINVAL;
591 }
592
593 return process_request(cptvf, req);
594}
595
