1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/kernel.h>
27#include <linux/types.h>
28#include <linux/module.h>
29#include <linux/pci.h>
30#include <linux/string.h>
31#include <linux/fs.h>
32#include <linux/sched.h>
33#include <linux/wait.h>
34#include <linux/delay.h>
35#include <linux/atomic.h>
36
37#include "card_base.h"
38#include "card_ddcb.h"
39
40static int genwqe_open_files(struct genwqe_dev *cd)
41{
42 int rc;
43 unsigned long flags;
44
45 spin_lock_irqsave(&cd->file_lock, flags);
46 rc = list_empty(&cd->file_list);
47 spin_unlock_irqrestore(&cd->file_lock, flags);
48 return !rc;
49}
50
51static void genwqe_add_file(struct genwqe_dev *cd, struct genwqe_file *cfile)
52{
53 unsigned long flags;
54
55 cfile->owner = current;
56 spin_lock_irqsave(&cd->file_lock, flags);
57 list_add(&cfile->list, &cd->file_list);
58 spin_unlock_irqrestore(&cd->file_lock, flags);
59}
60
61static int genwqe_del_file(struct genwqe_dev *cd, struct genwqe_file *cfile)
62{
63 unsigned long flags;
64
65 spin_lock_irqsave(&cd->file_lock, flags);
66 list_del(&cfile->list);
67 spin_unlock_irqrestore(&cd->file_lock, flags);
68
69 return 0;
70}
71
72static void genwqe_add_pin(struct genwqe_file *cfile, struct dma_mapping *m)
73{
74 unsigned long flags;
75
76 spin_lock_irqsave(&cfile->pin_lock, flags);
77 list_add(&m->pin_list, &cfile->pin_list);
78 spin_unlock_irqrestore(&cfile->pin_lock, flags);
79}
80
81static int genwqe_del_pin(struct genwqe_file *cfile, struct dma_mapping *m)
82{
83 unsigned long flags;
84
85 spin_lock_irqsave(&cfile->pin_lock, flags);
86 list_del(&m->pin_list);
87 spin_unlock_irqrestore(&cfile->pin_lock, flags);
88
89 return 0;
90}
91
92
93
94
95
96
97
98
99
100
101static struct dma_mapping *genwqe_search_pin(struct genwqe_file *cfile,
102 unsigned long u_addr,
103 unsigned int size,
104 void **virt_addr)
105{
106 unsigned long flags;
107 struct dma_mapping *m;
108
109 spin_lock_irqsave(&cfile->pin_lock, flags);
110
111 list_for_each_entry(m, &cfile->pin_list, pin_list) {
112 if ((((u64)m->u_vaddr) <= (u_addr)) &&
113 (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {
114
115 if (virt_addr)
116 *virt_addr = m->k_vaddr +
117 (u_addr - (u64)m->u_vaddr);
118
119 spin_unlock_irqrestore(&cfile->pin_lock, flags);
120 return m;
121 }
122 }
123 spin_unlock_irqrestore(&cfile->pin_lock, flags);
124 return NULL;
125}
126
127static void __genwqe_add_mapping(struct genwqe_file *cfile,
128 struct dma_mapping *dma_map)
129{
130 unsigned long flags;
131
132 spin_lock_irqsave(&cfile->map_lock, flags);
133 list_add(&dma_map->card_list, &cfile->map_list);
134 spin_unlock_irqrestore(&cfile->map_lock, flags);
135}
136
137static void __genwqe_del_mapping(struct genwqe_file *cfile,
138 struct dma_mapping *dma_map)
139{
140 unsigned long flags;
141
142 spin_lock_irqsave(&cfile->map_lock, flags);
143 list_del(&dma_map->card_list);
144 spin_unlock_irqrestore(&cfile->map_lock, flags);
145}
146
147
148
149
150
151
152
153
154
155
156static struct dma_mapping *__genwqe_search_mapping(struct genwqe_file *cfile,
157 unsigned long u_addr,
158 unsigned int size,
159 dma_addr_t *dma_addr,
160 void **virt_addr)
161{
162 unsigned long flags;
163 struct dma_mapping *m;
164 struct pci_dev *pci_dev = cfile->cd->pci_dev;
165
166 spin_lock_irqsave(&cfile->map_lock, flags);
167 list_for_each_entry(m, &cfile->map_list, card_list) {
168
169 if ((((u64)m->u_vaddr) <= (u_addr)) &&
170 (((u64)m->u_vaddr + m->size) >= (u_addr + size))) {
171
172
173
174 if (dma_addr)
175 *dma_addr = m->dma_addr +
176 (u_addr - (u64)m->u_vaddr);
177
178 if (virt_addr)
179 *virt_addr = m->k_vaddr +
180 (u_addr - (u64)m->u_vaddr);
181
182 spin_unlock_irqrestore(&cfile->map_lock, flags);
183 return m;
184 }
185 }
186 spin_unlock_irqrestore(&cfile->map_lock, flags);
187
188 dev_err(&pci_dev->dev,
189 "[%s] Entry not found: u_addr=%lx, size=%x\n",
190 __func__, u_addr, size);
191
192 return NULL;
193}
194
195static void genwqe_remove_mappings(struct genwqe_file *cfile)
196{
197 int i = 0;
198 struct list_head *node, *next;
199 struct dma_mapping *dma_map;
200 struct genwqe_dev *cd = cfile->cd;
201 struct pci_dev *pci_dev = cfile->cd->pci_dev;
202
203 list_for_each_safe(node, next, &cfile->map_list) {
204 dma_map = list_entry(node, struct dma_mapping, card_list);
205
206 list_del_init(&dma_map->card_list);
207
208
209
210
211
212
213
214
215 dev_err(&pci_dev->dev,
216 "[%s] %d. cleanup mapping: u_vaddr=%p "
217 "u_kaddr=%016lx dma_addr=%lx\n", __func__, i++,
218 dma_map->u_vaddr, (unsigned long)dma_map->k_vaddr,
219 (unsigned long)dma_map->dma_addr);
220
221 if (dma_map->type == GENWQE_MAPPING_RAW) {
222
223 __genwqe_free_consistent(cd, dma_map->size,
224 dma_map->k_vaddr,
225 dma_map->dma_addr);
226 kfree(dma_map);
227 } else if (dma_map->type == GENWQE_MAPPING_SGL_TEMP) {
228
229 genwqe_user_vunmap(cd, dma_map, NULL);
230 }
231 }
232}
233
234static void genwqe_remove_pinnings(struct genwqe_file *cfile)
235{
236 struct list_head *node, *next;
237 struct dma_mapping *dma_map;
238 struct genwqe_dev *cd = cfile->cd;
239
240 list_for_each_safe(node, next, &cfile->pin_list) {
241 dma_map = list_entry(node, struct dma_mapping, pin_list);
242
243
244
245
246
247
248
249
250
251 list_del_init(&dma_map->pin_list);
252 genwqe_user_vunmap(cd, dma_map, NULL);
253 kfree(dma_map);
254 }
255}
256
257
258
259
260
261
262static int genwqe_kill_fasync(struct genwqe_dev *cd, int sig)
263{
264 unsigned int files = 0;
265 unsigned long flags;
266 struct genwqe_file *cfile;
267
268 spin_lock_irqsave(&cd->file_lock, flags);
269 list_for_each_entry(cfile, &cd->file_list, list) {
270 if (cfile->async_queue)
271 kill_fasync(&cfile->async_queue, sig, POLL_HUP);
272 files++;
273 }
274 spin_unlock_irqrestore(&cd->file_lock, flags);
275 return files;
276}
277
278static int genwqe_force_sig(struct genwqe_dev *cd, int sig)
279{
280 unsigned int files = 0;
281 unsigned long flags;
282 struct genwqe_file *cfile;
283
284 spin_lock_irqsave(&cd->file_lock, flags);
285 list_for_each_entry(cfile, &cd->file_list, list) {
286 force_sig(sig, cfile->owner);
287 files++;
288 }
289 spin_unlock_irqrestore(&cd->file_lock, flags);
290 return files;
291}
292
293
294
295
296
297
298
299
300
301
302
303static int genwqe_open(struct inode *inode, struct file *filp)
304{
305 struct genwqe_dev *cd;
306 struct genwqe_file *cfile;
307 struct pci_dev *pci_dev;
308
309 cfile = kzalloc(sizeof(*cfile), GFP_KERNEL);
310 if (cfile == NULL)
311 return -ENOMEM;
312
313 cd = container_of(inode->i_cdev, struct genwqe_dev, cdev_genwqe);
314 pci_dev = cd->pci_dev;
315 cfile->cd = cd;
316 cfile->filp = filp;
317 cfile->client = NULL;
318
319 spin_lock_init(&cfile->map_lock);
320 INIT_LIST_HEAD(&cfile->map_list);
321
322 spin_lock_init(&cfile->pin_lock);
323 INIT_LIST_HEAD(&cfile->pin_list);
324
325 filp->private_data = cfile;
326
327 genwqe_add_file(cd, cfile);
328 return 0;
329}
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346static int genwqe_fasync(int fd, struct file *filp, int mode)
347{
348 struct genwqe_file *cdev = (struct genwqe_file *)filp->private_data;
349 return fasync_helper(fd, filp, mode, &cdev->async_queue);
350}
351
352
353
354
355
356
357
358
359
360
361
362static int genwqe_release(struct inode *inode, struct file *filp)
363{
364 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
365 struct genwqe_dev *cd = cfile->cd;
366
367
368 genwqe_remove_mappings(cfile);
369 genwqe_remove_pinnings(cfile);
370
371
372 genwqe_fasync(-1, filp, 0);
373
374
375
376
377
378
379 genwqe_del_file(cd, cfile);
380 kfree(cfile);
381 return 0;
382}
383
384static void genwqe_vma_open(struct vm_area_struct *vma)
385{
386
387}
388
389
390
391
392
393
394static void genwqe_vma_close(struct vm_area_struct *vma)
395{
396 unsigned long vsize = vma->vm_end - vma->vm_start;
397 struct inode *inode = vma->vm_file->f_dentry->d_inode;
398 struct dma_mapping *dma_map;
399 struct genwqe_dev *cd = container_of(inode->i_cdev, struct genwqe_dev,
400 cdev_genwqe);
401 struct pci_dev *pci_dev = cd->pci_dev;
402 dma_addr_t d_addr = 0;
403 struct genwqe_file *cfile = vma->vm_private_data;
404
405 dma_map = __genwqe_search_mapping(cfile, vma->vm_start, vsize,
406 &d_addr, NULL);
407 if (dma_map == NULL) {
408 dev_err(&pci_dev->dev,
409 " [%s] err: mapping not found: v=%lx, p=%lx s=%lx\n",
410 __func__, vma->vm_start, vma->vm_pgoff << PAGE_SHIFT,
411 vsize);
412 return;
413 }
414 __genwqe_del_mapping(cfile, dma_map);
415 __genwqe_free_consistent(cd, dma_map->size, dma_map->k_vaddr,
416 dma_map->dma_addr);
417 kfree(dma_map);
418}
419
420static struct vm_operations_struct genwqe_vma_ops = {
421 .open = genwqe_vma_open,
422 .close = genwqe_vma_close,
423};
424
425
426
427
428
429
430
431
432
433
434
435
436
437static int genwqe_mmap(struct file *filp, struct vm_area_struct *vma)
438{
439 int rc;
440 unsigned long pfn, vsize = vma->vm_end - vma->vm_start;
441 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
442 struct genwqe_dev *cd = cfile->cd;
443 struct dma_mapping *dma_map;
444
445 if (vsize == 0)
446 return -EINVAL;
447
448 if (get_order(vsize) > MAX_ORDER)
449 return -ENOMEM;
450
451 dma_map = kzalloc(sizeof(struct dma_mapping), GFP_ATOMIC);
452 if (dma_map == NULL)
453 return -ENOMEM;
454
455 genwqe_mapping_init(dma_map, GENWQE_MAPPING_RAW);
456 dma_map->u_vaddr = (void *)vma->vm_start;
457 dma_map->size = vsize;
458 dma_map->nr_pages = DIV_ROUND_UP(vsize, PAGE_SIZE);
459 dma_map->k_vaddr = __genwqe_alloc_consistent(cd, vsize,
460 &dma_map->dma_addr);
461 if (dma_map->k_vaddr == NULL) {
462 rc = -ENOMEM;
463 goto free_dma_map;
464 }
465
466 if (capable(CAP_SYS_ADMIN) && (vsize > sizeof(dma_addr_t)))
467 *(dma_addr_t *)dma_map->k_vaddr = dma_map->dma_addr;
468
469 pfn = virt_to_phys(dma_map->k_vaddr) >> PAGE_SHIFT;
470 rc = remap_pfn_range(vma,
471 vma->vm_start,
472 pfn,
473 vsize,
474 vma->vm_page_prot);
475 if (rc != 0) {
476 rc = -EFAULT;
477 goto free_dma_mem;
478 }
479
480 vma->vm_private_data = cfile;
481 vma->vm_ops = &genwqe_vma_ops;
482 __genwqe_add_mapping(cfile, dma_map);
483
484 return 0;
485
486 free_dma_mem:
487 __genwqe_free_consistent(cd, dma_map->size,
488 dma_map->k_vaddr,
489 dma_map->dma_addr);
490 free_dma_map:
491 kfree(dma_map);
492 return rc;
493}
494
495
496
497
498
499
500
501
502
503#define FLASH_BLOCK 0x40000
504
505static int do_flash_update(struct genwqe_file *cfile,
506 struct genwqe_bitstream *load)
507{
508 int rc = 0;
509 int blocks_to_flash;
510 dma_addr_t dma_addr;
511 u64 flash = 0;
512 size_t tocopy = 0;
513 u8 __user *buf;
514 u8 *xbuf;
515 u32 crc;
516 u8 cmdopts;
517 struct genwqe_dev *cd = cfile->cd;
518 struct pci_dev *pci_dev = cd->pci_dev;
519
520 if ((load->size & 0x3) != 0)
521 return -EINVAL;
522
523 if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0)
524 return -EINVAL;
525
526
527 switch ((char)load->partition) {
528 case '0':
529 cmdopts = 0x14;
530 break;
531 case '1':
532 cmdopts = 0x1C;
533 break;
534 case 'v':
535 cmdopts = 0x0C;
536 break;
537 default:
538 return -EINVAL;
539 }
540
541 buf = (u8 __user *)load->data_addr;
542 xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);
543 if (xbuf == NULL)
544 return -ENOMEM;
545
546 blocks_to_flash = load->size / FLASH_BLOCK;
547 while (load->size) {
548 struct genwqe_ddcb_cmd *req;
549
550
551
552
553
554 tocopy = min_t(size_t, load->size, FLASH_BLOCK);
555
556 rc = copy_from_user(xbuf, buf, tocopy);
557 if (rc) {
558 rc = -EFAULT;
559 goto free_buffer;
560 }
561 crc = genwqe_crc32(xbuf, tocopy, 0xffffffff);
562
563 dev_dbg(&pci_dev->dev,
564 "[%s] DMA: %lx CRC: %08x SZ: %ld %d\n",
565 __func__, (unsigned long)dma_addr, crc, tocopy,
566 blocks_to_flash);
567
568
569 req = ddcb_requ_alloc();
570 if (req == NULL) {
571 rc = -ENOMEM;
572 goto free_buffer;
573 }
574
575 req->cmd = SLCMD_MOVE_FLASH;
576 req->cmdopts = cmdopts;
577
578
579 if (genwqe_get_slu_id(cd) <= 0x2) {
580 *(__be64 *)&req->__asiv[0] = cpu_to_be64(dma_addr);
581 *(__be64 *)&req->__asiv[8] = cpu_to_be64(tocopy);
582 *(__be64 *)&req->__asiv[16] = cpu_to_be64(flash);
583 *(__be32 *)&req->__asiv[24] = cpu_to_be32(0);
584 req->__asiv[24] = load->uid;
585 *(__be32 *)&req->__asiv[28] = cpu_to_be32(crc);
586
587
588 *(__be64 *)&req->__asiv[88] = cpu_to_be64(load->slu_id);
589 *(__be64 *)&req->__asiv[96] = cpu_to_be64(load->app_id);
590 req->asiv_length = 32;
591 } else {
592 *(__be64 *)&req->asiv[0] = cpu_to_be64(dma_addr);
593 *(__be32 *)&req->asiv[8] = cpu_to_be32(tocopy);
594 *(__be32 *)&req->asiv[12] = cpu_to_be32(0);
595 *(__be64 *)&req->asiv[16] = cpu_to_be64(flash);
596 *(__be32 *)&req->asiv[24] = cpu_to_be32(load->uid<<24);
597 *(__be32 *)&req->asiv[28] = cpu_to_be32(crc);
598
599
600 *(__be64 *)&req->asiv[80] = cpu_to_be64(load->slu_id);
601 *(__be64 *)&req->asiv[88] = cpu_to_be64(load->app_id);
602
603
604 req->ats = 0x4ULL << 44;
605 req->asiv_length = 40;
606 }
607 req->asv_length = 8;
608
609
610 *(u64 *)&req->asv[0] = 0ULL;
611
612 rc = __genwqe_execute_raw_ddcb(cd, req);
613
614 load->retc = req->retc;
615 load->attn = req->attn;
616 load->progress = req->progress;
617
618 if (rc < 0) {
619 ddcb_requ_free(req);
620 goto free_buffer;
621 }
622
623 if (req->retc != DDCB_RETC_COMPLETE) {
624 rc = -EIO;
625 ddcb_requ_free(req);
626 goto free_buffer;
627 }
628
629 load->size -= tocopy;
630 flash += tocopy;
631 buf += tocopy;
632 blocks_to_flash--;
633 ddcb_requ_free(req);
634 }
635
636 free_buffer:
637 __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);
638 return rc;
639}
640
641static int do_flash_read(struct genwqe_file *cfile,
642 struct genwqe_bitstream *load)
643{
644 int rc, blocks_to_flash;
645 dma_addr_t dma_addr;
646 u64 flash = 0;
647 size_t tocopy = 0;
648 u8 __user *buf;
649 u8 *xbuf;
650 u8 cmdopts;
651 struct genwqe_dev *cd = cfile->cd;
652 struct pci_dev *pci_dev = cd->pci_dev;
653 struct genwqe_ddcb_cmd *cmd;
654
655 if ((load->size & 0x3) != 0)
656 return -EINVAL;
657
658 if (((unsigned long)(load->data_addr) & ~PAGE_MASK) != 0)
659 return -EINVAL;
660
661
662 switch ((char)load->partition) {
663 case '0':
664 cmdopts = 0x12;
665 break;
666 case '1':
667 cmdopts = 0x1A;
668 break;
669 case 'v':
670 cmdopts = 0x0A;
671 break;
672 default:
673 return -EINVAL;
674 }
675
676 buf = (u8 __user *)load->data_addr;
677 xbuf = __genwqe_alloc_consistent(cd, FLASH_BLOCK, &dma_addr);
678 if (xbuf == NULL)
679 return -ENOMEM;
680
681 blocks_to_flash = load->size / FLASH_BLOCK;
682 while (load->size) {
683
684
685
686
687 tocopy = min_t(size_t, load->size, FLASH_BLOCK);
688
689 dev_dbg(&pci_dev->dev,
690 "[%s] DMA: %lx SZ: %ld %d\n",
691 __func__, (unsigned long)dma_addr, tocopy,
692 blocks_to_flash);
693
694
695 cmd = ddcb_requ_alloc();
696 if (cmd == NULL) {
697 rc = -ENOMEM;
698 goto free_buffer;
699 }
700 cmd->cmd = SLCMD_MOVE_FLASH;
701 cmd->cmdopts = cmdopts;
702
703
704 if (genwqe_get_slu_id(cd) <= 0x2) {
705 *(__be64 *)&cmd->__asiv[0] = cpu_to_be64(dma_addr);
706 *(__be64 *)&cmd->__asiv[8] = cpu_to_be64(tocopy);
707 *(__be64 *)&cmd->__asiv[16] = cpu_to_be64(flash);
708 *(__be32 *)&cmd->__asiv[24] = cpu_to_be32(0);
709 cmd->__asiv[24] = load->uid;
710 *(__be32 *)&cmd->__asiv[28] = cpu_to_be32(0) ;
711 cmd->asiv_length = 32;
712 } else {
713 *(__be64 *)&cmd->asiv[0] = cpu_to_be64(dma_addr);
714 *(__be32 *)&cmd->asiv[8] = cpu_to_be32(tocopy);
715 *(__be32 *)&cmd->asiv[12] = cpu_to_be32(0);
716 *(__be64 *)&cmd->asiv[16] = cpu_to_be64(flash);
717 *(__be32 *)&cmd->asiv[24] = cpu_to_be32(load->uid<<24);
718 *(__be32 *)&cmd->asiv[28] = cpu_to_be32(0);
719
720
721 cmd->ats = 0x5ULL << 44;
722 cmd->asiv_length = 40;
723 }
724 cmd->asv_length = 8;
725
726
727 *(u64 *)&cmd->asv[0] = 0ULL;
728
729 rc = __genwqe_execute_raw_ddcb(cd, cmd);
730
731 load->retc = cmd->retc;
732 load->attn = cmd->attn;
733 load->progress = cmd->progress;
734
735 if ((rc < 0) && (rc != -EBADMSG)) {
736 ddcb_requ_free(cmd);
737 goto free_buffer;
738 }
739
740 rc = copy_to_user(buf, xbuf, tocopy);
741 if (rc) {
742 rc = -EFAULT;
743 ddcb_requ_free(cmd);
744 goto free_buffer;
745 }
746
747
748 if (((cmd->retc == DDCB_RETC_FAULT) &&
749 (cmd->attn != 0x02)) ||
750 ((cmd->retc == DDCB_RETC_COMPLETE) &&
751 (cmd->attn != 0x00))) {
752 rc = -EIO;
753 ddcb_requ_free(cmd);
754 goto free_buffer;
755 }
756
757 load->size -= tocopy;
758 flash += tocopy;
759 buf += tocopy;
760 blocks_to_flash--;
761 ddcb_requ_free(cmd);
762 }
763 rc = 0;
764
765 free_buffer:
766 __genwqe_free_consistent(cd, FLASH_BLOCK, xbuf, dma_addr);
767 return rc;
768}
769
770static int genwqe_pin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
771{
772 int rc;
773 struct genwqe_dev *cd = cfile->cd;
774 struct pci_dev *pci_dev = cfile->cd->pci_dev;
775 struct dma_mapping *dma_map;
776 unsigned long map_addr;
777 unsigned long map_size;
778
779 if ((m->addr == 0x0) || (m->size == 0))
780 return -EINVAL;
781
782 map_addr = (m->addr & PAGE_MASK);
783 map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);
784
785 dma_map = kzalloc(sizeof(struct dma_mapping), GFP_ATOMIC);
786 if (dma_map == NULL)
787 return -ENOMEM;
788
789 genwqe_mapping_init(dma_map, GENWQE_MAPPING_SGL_PINNED);
790 rc = genwqe_user_vmap(cd, dma_map, (void *)map_addr, map_size, NULL);
791 if (rc != 0) {
792 dev_err(&pci_dev->dev,
793 "[%s] genwqe_user_vmap rc=%d\n", __func__, rc);
794 kfree(dma_map);
795 return rc;
796 }
797
798 genwqe_add_pin(cfile, dma_map);
799 return 0;
800}
801
802static int genwqe_unpin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
803{
804 struct genwqe_dev *cd = cfile->cd;
805 struct dma_mapping *dma_map;
806 unsigned long map_addr;
807 unsigned long map_size;
808
809 if (m->addr == 0x0)
810 return -EINVAL;
811
812 map_addr = (m->addr & PAGE_MASK);
813 map_size = round_up(m->size + (m->addr & ~PAGE_MASK), PAGE_SIZE);
814
815 dma_map = genwqe_search_pin(cfile, map_addr, map_size, NULL);
816 if (dma_map == NULL)
817 return -ENOENT;
818
819 genwqe_del_pin(cfile, dma_map);
820 genwqe_user_vunmap(cd, dma_map, NULL);
821 kfree(dma_map);
822 return 0;
823}
824
825
826
827
828
829
830static int ddcb_cmd_cleanup(struct genwqe_file *cfile, struct ddcb_requ *req)
831{
832 unsigned int i;
833 struct dma_mapping *dma_map;
834 struct genwqe_dev *cd = cfile->cd;
835
836 for (i = 0; i < DDCB_FIXUPS; i++) {
837 dma_map = &req->dma_mappings[i];
838
839 if (dma_mapping_used(dma_map)) {
840 __genwqe_del_mapping(cfile, dma_map);
841 genwqe_user_vunmap(cd, dma_map, req);
842 }
843 if (req->sgls[i].sgl != NULL)
844 genwqe_free_sync_sgl(cd, &req->sgls[i]);
845 }
846 return 0;
847}
848
849
850
851
852
853
854
855
856
857static int ddcb_cmd_fixups(struct genwqe_file *cfile, struct ddcb_requ *req)
858{
859 int rc;
860 unsigned int asiv_offs, i;
861 struct genwqe_dev *cd = cfile->cd;
862 struct genwqe_ddcb_cmd *cmd = &req->cmd;
863 struct dma_mapping *m;
864 const char *type = "UNKNOWN";
865
866 for (i = 0, asiv_offs = 0x00; asiv_offs <= 0x58;
867 i++, asiv_offs += 0x08) {
868
869 u64 u_addr;
870 dma_addr_t d_addr;
871 u32 u_size = 0;
872 u64 ats_flags;
873
874 ats_flags = ATS_GET_FLAGS(cmd->ats, asiv_offs);
875
876 switch (ats_flags) {
877
878 case ATS_TYPE_DATA:
879 break;
880
881 case ATS_TYPE_FLAT_RDWR:
882 case ATS_TYPE_FLAT_RD: {
883 u_addr = be64_to_cpu(*((__be64 *)&cmd->
884 asiv[asiv_offs]));
885 u_size = be32_to_cpu(*((__be32 *)&cmd->
886 asiv[asiv_offs + 0x08]));
887
888
889
890
891
892
893 if (u_size == 0x0) {
894 *((__be64 *)&cmd->asiv[asiv_offs]) =
895 cpu_to_be64(0x0);
896 break;
897 }
898
899 m = __genwqe_search_mapping(cfile, u_addr, u_size,
900 &d_addr, NULL);
901 if (m == NULL) {
902 rc = -EFAULT;
903 goto err_out;
904 }
905
906 *((__be64 *)&cmd->asiv[asiv_offs]) =
907 cpu_to_be64(d_addr);
908 break;
909 }
910
911 case ATS_TYPE_SGL_RDWR:
912 case ATS_TYPE_SGL_RD: {
913 int page_offs;
914
915 u_addr = be64_to_cpu(*((__be64 *)
916 &cmd->asiv[asiv_offs]));
917 u_size = be32_to_cpu(*((__be32 *)
918 &cmd->asiv[asiv_offs + 0x08]));
919
920
921
922
923
924
925 if (u_size == 0x0) {
926 *((__be64 *)&cmd->asiv[asiv_offs]) =
927 cpu_to_be64(0x0);
928 break;
929 }
930
931 m = genwqe_search_pin(cfile, u_addr, u_size, NULL);
932 if (m != NULL) {
933 type = "PINNING";
934 page_offs = (u_addr -
935 (u64)m->u_vaddr)/PAGE_SIZE;
936 } else {
937 type = "MAPPING";
938 m = &req->dma_mappings[i];
939
940 genwqe_mapping_init(m,
941 GENWQE_MAPPING_SGL_TEMP);
942 rc = genwqe_user_vmap(cd, m, (void *)u_addr,
943 u_size, req);
944 if (rc != 0)
945 goto err_out;
946
947 __genwqe_add_mapping(cfile, m);
948 page_offs = 0;
949 }
950
951
952 rc = genwqe_alloc_sync_sgl(cd, &req->sgls[i],
953 (void __user *)u_addr,
954 u_size);
955 if (rc != 0)
956 goto err_out;
957
958 genwqe_setup_sgl(cd, &req->sgls[i],
959 &m->dma_list[page_offs]);
960
961 *((__be64 *)&cmd->asiv[asiv_offs]) =
962 cpu_to_be64(req->sgls[i].sgl_dma_addr);
963
964 break;
965 }
966 default:
967 rc = -EINVAL;
968 goto err_out;
969 }
970 }
971 return 0;
972
973 err_out:
974 ddcb_cmd_cleanup(cfile, req);
975 return rc;
976}
977
978
979
980
981
982
983
984
985static int genwqe_execute_ddcb(struct genwqe_file *cfile,
986 struct genwqe_ddcb_cmd *cmd)
987{
988 int rc;
989 struct genwqe_dev *cd = cfile->cd;
990 struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
991
992 rc = ddcb_cmd_fixups(cfile, req);
993 if (rc != 0)
994 return rc;
995
996 rc = __genwqe_execute_raw_ddcb(cd, cmd);
997 ddcb_cmd_cleanup(cfile, req);
998 return rc;
999}
1000
1001static int do_execute_ddcb(struct genwqe_file *cfile,
1002 unsigned long arg, int raw)
1003{
1004 int rc;
1005 struct genwqe_ddcb_cmd *cmd;
1006 struct ddcb_requ *req;
1007 struct genwqe_dev *cd = cfile->cd;
1008
1009 cmd = ddcb_requ_alloc();
1010 if (cmd == NULL)
1011 return -ENOMEM;
1012
1013 req = container_of(cmd, struct ddcb_requ, cmd);
1014
1015 if (copy_from_user(cmd, (void __user *)arg, sizeof(*cmd))) {
1016 ddcb_requ_free(cmd);
1017 return -EFAULT;
1018 }
1019
1020 if (!raw)
1021 rc = genwqe_execute_ddcb(cfile, cmd);
1022 else
1023 rc = __genwqe_execute_raw_ddcb(cd, cmd);
1024
1025
1026
1027 if (copy_to_user((void __user *)arg, cmd,
1028 sizeof(*cmd) - DDCB_ASIV_LENGTH)) {
1029 ddcb_requ_free(cmd);
1030 return -EFAULT;
1031 }
1032
1033 ddcb_requ_free(cmd);
1034 return rc;
1035}
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045static long genwqe_ioctl(struct file *filp, unsigned int cmd,
1046 unsigned long arg)
1047{
1048 int rc = 0;
1049 struct genwqe_file *cfile = (struct genwqe_file *)filp->private_data;
1050 struct genwqe_dev *cd = cfile->cd;
1051 struct genwqe_reg_io __user *io;
1052 u64 val;
1053 u32 reg_offs;
1054
1055 if (_IOC_TYPE(cmd) != GENWQE_IOC_CODE)
1056 return -EINVAL;
1057
1058 switch (cmd) {
1059
1060 case GENWQE_GET_CARD_STATE:
1061 put_user(cd->card_state, (enum genwqe_card_state __user *)arg);
1062 return 0;
1063
1064
1065 case GENWQE_READ_REG64: {
1066 io = (struct genwqe_reg_io __user *)arg;
1067
1068 if (get_user(reg_offs, &io->num))
1069 return -EFAULT;
1070
1071 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))
1072 return -EINVAL;
1073
1074 val = __genwqe_readq(cd, reg_offs);
1075 put_user(val, &io->val64);
1076 return 0;
1077 }
1078
1079 case GENWQE_WRITE_REG64: {
1080 io = (struct genwqe_reg_io __user *)arg;
1081
1082 if (!capable(CAP_SYS_ADMIN))
1083 return -EPERM;
1084
1085 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1086 return -EPERM;
1087
1088 if (get_user(reg_offs, &io->num))
1089 return -EFAULT;
1090
1091 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x7))
1092 return -EINVAL;
1093
1094 if (get_user(val, &io->val64))
1095 return -EFAULT;
1096
1097 __genwqe_writeq(cd, reg_offs, val);
1098 return 0;
1099 }
1100
1101 case GENWQE_READ_REG32: {
1102 io = (struct genwqe_reg_io __user *)arg;
1103
1104 if (get_user(reg_offs, &io->num))
1105 return -EFAULT;
1106
1107 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))
1108 return -EINVAL;
1109
1110 val = __genwqe_readl(cd, reg_offs);
1111 put_user(val, &io->val64);
1112 return 0;
1113 }
1114
1115 case GENWQE_WRITE_REG32: {
1116 io = (struct genwqe_reg_io __user *)arg;
1117
1118 if (!capable(CAP_SYS_ADMIN))
1119 return -EPERM;
1120
1121 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1122 return -EPERM;
1123
1124 if (get_user(reg_offs, &io->num))
1125 return -EFAULT;
1126
1127 if ((reg_offs >= cd->mmio_len) || (reg_offs & 0x3))
1128 return -EINVAL;
1129
1130 if (get_user(val, &io->val64))
1131 return -EFAULT;
1132
1133 __genwqe_writel(cd, reg_offs, val);
1134 return 0;
1135 }
1136
1137
1138 case GENWQE_SLU_UPDATE: {
1139 struct genwqe_bitstream load;
1140
1141 if (!genwqe_is_privileged(cd))
1142 return -EPERM;
1143
1144 if ((filp->f_flags & O_ACCMODE) == O_RDONLY)
1145 return -EPERM;
1146
1147 if (copy_from_user(&load, (void __user *)arg,
1148 sizeof(load)))
1149 return -EFAULT;
1150
1151 rc = do_flash_update(cfile, &load);
1152
1153 if (copy_to_user((void __user *)arg, &load, sizeof(load)))
1154 return -EFAULT;
1155
1156 return rc;
1157 }
1158
1159 case GENWQE_SLU_READ: {
1160 struct genwqe_bitstream load;
1161
1162 if (!genwqe_is_privileged(cd))
1163 return -EPERM;
1164
1165 if (genwqe_flash_readback_fails(cd))
1166 return -ENOSPC;
1167
1168 if (copy_from_user(&load, (void __user *)arg, sizeof(load)))
1169 return -EFAULT;
1170
1171 rc = do_flash_read(cfile, &load);
1172
1173 if (copy_to_user((void __user *)arg, &load, sizeof(load)))
1174 return -EFAULT;
1175
1176 return rc;
1177 }
1178
1179
1180 case GENWQE_PIN_MEM: {
1181 struct genwqe_mem m;
1182
1183 if (copy_from_user(&m, (void __user *)arg, sizeof(m)))
1184 return -EFAULT;
1185
1186 return genwqe_pin_mem(cfile, &m);
1187 }
1188
1189 case GENWQE_UNPIN_MEM: {
1190 struct genwqe_mem m;
1191
1192 if (copy_from_user(&m, (void __user *)arg, sizeof(m)))
1193 return -EFAULT;
1194
1195 return genwqe_unpin_mem(cfile, &m);
1196 }
1197
1198
1199 case GENWQE_EXECUTE_DDCB:
1200 return do_execute_ddcb(cfile, arg, 0);
1201
1202 case GENWQE_EXECUTE_RAW_DDCB: {
1203
1204 if (!capable(CAP_SYS_ADMIN))
1205 return -EPERM;
1206
1207 return do_execute_ddcb(cfile, arg, 1);
1208 }
1209
1210 default:
1211 return -EINVAL;
1212 }
1213
1214 return rc;
1215}
1216
1217#if defined(CONFIG_COMPAT)
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229static long genwqe_compat_ioctl(struct file *filp, unsigned int cmd,
1230 unsigned long arg)
1231{
1232 return genwqe_ioctl(filp, cmd, arg);
1233}
1234#endif
1235
1236static const struct file_operations genwqe_fops = {
1237 .owner = THIS_MODULE,
1238 .open = genwqe_open,
1239 .fasync = genwqe_fasync,
1240 .mmap = genwqe_mmap,
1241 .unlocked_ioctl = genwqe_ioctl,
1242#if defined(CONFIG_COMPAT)
1243 .compat_ioctl = genwqe_compat_ioctl,
1244#endif
1245 .release = genwqe_release,
1246};
1247
1248static int genwqe_device_initialized(struct genwqe_dev *cd)
1249{
1250 return cd->dev != NULL;
1251}
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261int genwqe_device_create(struct genwqe_dev *cd)
1262{
1263 int rc;
1264 struct pci_dev *pci_dev = cd->pci_dev;
1265
1266
1267
1268
1269
1270
1271
1272 rc = alloc_chrdev_region(&cd->devnum_genwqe, 0,
1273 GENWQE_MAX_MINOR, GENWQE_DEVNAME);
1274 if (rc < 0) {
1275 dev_err(&pci_dev->dev, "err: alloc_chrdev_region failed\n");
1276 goto err_dev;
1277 }
1278
1279 cdev_init(&cd->cdev_genwqe, &genwqe_fops);
1280 cd->cdev_genwqe.owner = THIS_MODULE;
1281
1282 rc = cdev_add(&cd->cdev_genwqe, cd->devnum_genwqe, 1);
1283 if (rc < 0) {
1284 dev_err(&pci_dev->dev, "err: cdev_add failed\n");
1285 goto err_add;
1286 }
1287
1288
1289
1290
1291
1292 cd->dev = device_create_with_groups(cd->class_genwqe,
1293 &cd->pci_dev->dev,
1294 cd->devnum_genwqe, cd,
1295 genwqe_attribute_groups,
1296 GENWQE_DEVNAME "%u_card",
1297 cd->card_idx);
1298 if (IS_ERR(cd->dev)) {
1299 rc = PTR_ERR(cd->dev);
1300 goto err_cdev;
1301 }
1302
1303 rc = genwqe_init_debugfs(cd);
1304 if (rc != 0)
1305 goto err_debugfs;
1306
1307 return 0;
1308
1309 err_debugfs:
1310 device_destroy(cd->class_genwqe, cd->devnum_genwqe);
1311 err_cdev:
1312 cdev_del(&cd->cdev_genwqe);
1313 err_add:
1314 unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);
1315 err_dev:
1316 cd->dev = NULL;
1317 return rc;
1318}
1319
1320static int genwqe_inform_and_stop_processes(struct genwqe_dev *cd)
1321{
1322 int rc;
1323 unsigned int i;
1324 struct pci_dev *pci_dev = cd->pci_dev;
1325
1326 if (!genwqe_open_files(cd))
1327 return 0;
1328
1329 dev_warn(&pci_dev->dev, "[%s] send SIGIO and wait ...\n", __func__);
1330
1331 rc = genwqe_kill_fasync(cd, SIGIO);
1332 if (rc > 0) {
1333
1334 for (i = 0; (i < genwqe_kill_timeout) &&
1335 genwqe_open_files(cd); i++) {
1336 dev_info(&pci_dev->dev, " %d sec ...", i);
1337
1338 cond_resched();
1339 msleep(1000);
1340 }
1341
1342
1343 if (!genwqe_open_files(cd))
1344 return 0;
1345
1346 dev_warn(&pci_dev->dev,
1347 "[%s] send SIGKILL and wait ...\n", __func__);
1348
1349 rc = genwqe_force_sig(cd, SIGKILL);
1350 if (rc) {
1351
1352 for (i = 0; (i < genwqe_kill_timeout) &&
1353 genwqe_open_files(cd); i++) {
1354 dev_warn(&pci_dev->dev, " %d sec ...", i);
1355
1356 cond_resched();
1357 msleep(1000);
1358 }
1359 }
1360 }
1361 return 0;
1362}
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373int genwqe_device_remove(struct genwqe_dev *cd)
1374{
1375 int rc;
1376 struct pci_dev *pci_dev = cd->pci_dev;
1377
1378 if (!genwqe_device_initialized(cd))
1379 return 1;
1380
1381 genwqe_inform_and_stop_processes(cd);
1382
1383
1384
1385
1386
1387
1388
1389 rc = atomic_read(&cd->cdev_genwqe.kobj.kref.refcount);
1390 if (rc != 1) {
1391 dev_err(&pci_dev->dev,
1392 "[%s] err: cdev_genwqe...refcount=%d\n", __func__, rc);
1393 panic("Fatal err: cannot free resources with pending references!");
1394 }
1395
1396 genqwe_exit_debugfs(cd);
1397 device_destroy(cd->class_genwqe, cd->devnum_genwqe);
1398 cdev_del(&cd->cdev_genwqe);
1399 unregister_chrdev_region(cd->devnum_genwqe, GENWQE_MAX_MINOR);
1400 cd->dev = NULL;
1401
1402 return 0;
1403}
1404