1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/pci.h>
35#include <linux/poll.h>
36#include <linux/cdev.h>
37#include <linux/swap.h>
38#include <linux/export.h>
39#include <linux/vmalloc.h>
40#include <linux/slab.h>
41#include <linux/highmem.h>
42#include <linux/io.h>
43#include <linux/jiffies.h>
44#include <linux/cpu.h>
45#include <linux/uio.h>
46#include <asm/pgtable.h>
47
48#include "ipath_kernel.h"
49#include "ipath_common.h"
50#include "ipath_user_sdma.h"
51
52static int ipath_open(struct inode *, struct file *);
53static int ipath_close(struct inode *, struct file *);
54static ssize_t ipath_write(struct file *, const char __user *, size_t,
55 loff_t *);
56static ssize_t ipath_write_iter(struct kiocb *, struct iov_iter *from);
57static unsigned int ipath_poll(struct file *, struct poll_table_struct *);
58static int ipath_mmap(struct file *, struct vm_area_struct *);
59
60
61
62
63
64
65static const struct file_operations ipath_file_ops = {
66 .owner = THIS_MODULE,
67 .write = ipath_write,
68 .write_iter = ipath_write_iter,
69 .open = ipath_open,
70 .release = ipath_close,
71 .poll = ipath_poll,
72 .mmap = ipath_mmap,
73 .llseek = noop_llseek,
74};
75
76
77
78
79
80
81
82static u64 cvt_kvaddr(void *p)
83{
84 struct page *page;
85 u64 paddr = 0;
86
87 page = vmalloc_to_page(p);
88 if (page)
89 paddr = page_to_pfn(page) << PAGE_SHIFT;
90
91 return paddr;
92}
93
94static int ipath_get_base_info(struct file *fp,
95 void __user *ubase, size_t ubase_size)
96{
97 struct ipath_portdata *pd = port_fp(fp);
98 int ret = 0;
99 struct ipath_base_info *kinfo = NULL;
100 struct ipath_devdata *dd = pd->port_dd;
101 unsigned subport_cnt;
102 int shared, master;
103 size_t sz;
104
105 subport_cnt = pd->port_subport_cnt;
106 if (!subport_cnt) {
107 shared = 0;
108 master = 0;
109 subport_cnt = 1;
110 } else {
111 shared = 1;
112 master = !subport_fp(fp);
113 }
114
115 sz = sizeof(*kinfo);
116
117 if (!shared)
118 sz -= 7 * sizeof(u64);
119 if (ubase_size < sz) {
120 ipath_cdbg(PROC,
121 "Base size %zu, need %zu (version mismatch?)\n",
122 ubase_size, sz);
123 ret = -EINVAL;
124 goto bail;
125 }
126
127 kinfo = kzalloc(sizeof(*kinfo), GFP_KERNEL);
128 if (kinfo == NULL) {
129 ret = -ENOMEM;
130 goto bail;
131 }
132
133 ret = dd->ipath_f_get_base_info(pd, kinfo);
134 if (ret < 0)
135 goto bail;
136
137 kinfo->spi_rcvhdr_cnt = dd->ipath_rcvhdrcnt;
138 kinfo->spi_rcvhdrent_size = dd->ipath_rcvhdrentsize;
139 kinfo->spi_tidegrcnt = dd->ipath_rcvegrcnt;
140 kinfo->spi_rcv_egrbufsize = dd->ipath_rcvegrbufsize;
141
142
143
144 kinfo->spi_rcv_egrbuftotlen =
145 pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size;
146 kinfo->spi_rcv_egrperchunk = pd->port_rcvegrbufs_perchunk;
147 kinfo->spi_rcv_egrchunksize = kinfo->spi_rcv_egrbuftotlen /
148 pd->port_rcvegrbuf_chunks;
149 kinfo->spi_tidcnt = dd->ipath_rcvtidcnt / subport_cnt;
150 if (master)
151 kinfo->spi_tidcnt += dd->ipath_rcvtidcnt % subport_cnt;
152
153
154
155
156 kinfo->spi_nports = dd->ipath_cfgports;
157
158 kinfo->spi_unit = dd->ipath_unit;
159
160 kinfo->spi_tid_maxsize = PAGE_SIZE;
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177 kinfo->spi_rcvhdr_base = (u64) pd->port_rcvhdrq_phys;
178 kinfo->spi_rcvhdr_tailaddr = (u64) pd->port_rcvhdrqtailaddr_phys;
179 kinfo->spi_rcv_egrbufs = (u64) pd->port_rcvegr_phys;
180 kinfo->spi_pioavailaddr = (u64) dd->ipath_pioavailregs_phys;
181 kinfo->spi_status = (u64) kinfo->spi_pioavailaddr +
182 (void *) dd->ipath_statusp -
183 (void *) dd->ipath_pioavailregs_dma;
184 if (!shared) {
185 kinfo->spi_piocnt = pd->port_piocnt;
186 kinfo->spi_piobufbase = (u64) pd->port_piobufs;
187 kinfo->__spi_uregbase = (u64) dd->ipath_uregbase +
188 dd->ipath_ureg_align * pd->port_port;
189 } else if (master) {
190 kinfo->spi_piocnt = (pd->port_piocnt / subport_cnt) +
191 (pd->port_piocnt % subport_cnt);
192
193 kinfo->spi_piobufbase = (u64) pd->port_piobufs +
194 dd->ipath_palign *
195 (pd->port_piocnt - kinfo->spi_piocnt);
196 } else {
197 unsigned slave = subport_fp(fp) - 1;
198
199 kinfo->spi_piocnt = pd->port_piocnt / subport_cnt;
200 kinfo->spi_piobufbase = (u64) pd->port_piobufs +
201 dd->ipath_palign * kinfo->spi_piocnt * slave;
202 }
203
204 if (shared) {
205 kinfo->spi_port_uregbase = (u64) dd->ipath_uregbase +
206 dd->ipath_ureg_align * pd->port_port;
207 kinfo->spi_port_rcvegrbuf = kinfo->spi_rcv_egrbufs;
208 kinfo->spi_port_rcvhdr_base = kinfo->spi_rcvhdr_base;
209 kinfo->spi_port_rcvhdr_tailaddr = kinfo->spi_rcvhdr_tailaddr;
210
211 kinfo->__spi_uregbase = cvt_kvaddr(pd->subport_uregbase +
212 PAGE_SIZE * subport_fp(fp));
213
214 kinfo->spi_rcvhdr_base = cvt_kvaddr(pd->subport_rcvhdr_base +
215 pd->port_rcvhdrq_size * subport_fp(fp));
216 kinfo->spi_rcvhdr_tailaddr = 0;
217 kinfo->spi_rcv_egrbufs = cvt_kvaddr(pd->subport_rcvegrbuf +
218 pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size *
219 subport_fp(fp));
220
221 kinfo->spi_subport_uregbase =
222 cvt_kvaddr(pd->subport_uregbase);
223 kinfo->spi_subport_rcvegrbuf =
224 cvt_kvaddr(pd->subport_rcvegrbuf);
225 kinfo->spi_subport_rcvhdr_base =
226 cvt_kvaddr(pd->subport_rcvhdr_base);
227 ipath_cdbg(PROC, "port %u flags %x %llx %llx %llx\n",
228 kinfo->spi_port, kinfo->spi_runtime_flags,
229 (unsigned long long) kinfo->spi_subport_uregbase,
230 (unsigned long long) kinfo->spi_subport_rcvegrbuf,
231 (unsigned long long) kinfo->spi_subport_rcvhdr_base);
232 }
233
234
235
236
237
238
239 kinfo->spi_pioindex = (kinfo->spi_piobufbase -
240 (dd->ipath_piobufbase & 0xffffffff)) / dd->ipath_palign;
241 kinfo->spi_pioalign = dd->ipath_palign;
242
243 kinfo->spi_qpair = IPATH_KD_QP;
244
245
246
247
248
249 kinfo->spi_piosize = dd->ipath_piosize2k - 2 * sizeof(u32);
250 kinfo->spi_mtu = dd->ipath_ibmaxlen;
251 kinfo->spi_port = pd->port_port;
252 kinfo->spi_subport = subport_fp(fp);
253 kinfo->spi_sw_version = IPATH_KERN_SWVERSION;
254 kinfo->spi_hw_version = dd->ipath_revision;
255
256 if (master) {
257 kinfo->spi_runtime_flags |= IPATH_RUNTIME_MASTER;
258 }
259
260 sz = (ubase_size < sizeof(*kinfo)) ? ubase_size : sizeof(*kinfo);
261 if (copy_to_user(ubase, kinfo, sz))
262 ret = -EFAULT;
263
264bail:
265 kfree(kinfo);
266 return ret;
267}
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295static int ipath_tid_update(struct ipath_portdata *pd, struct file *fp,
296 const struct ipath_tid_info *ti)
297{
298 int ret = 0, ntids;
299 u32 tid, porttid, cnt, i, tidcnt, tidoff;
300 u16 *tidlist;
301 struct ipath_devdata *dd = pd->port_dd;
302 u64 physaddr;
303 unsigned long vaddr;
304 u64 __iomem *tidbase;
305 unsigned long tidmap[8];
306 struct page **pagep = NULL;
307 unsigned subport = subport_fp(fp);
308
309 if (!dd->ipath_pageshadow) {
310 ret = -ENOMEM;
311 goto done;
312 }
313
314 cnt = ti->tidcnt;
315 if (!cnt) {
316 ipath_dbg("After copyin, tidcnt 0, tidlist %llx\n",
317 (unsigned long long) ti->tidlist);
318
319
320
321 ret = -EFAULT;
322 goto done;
323 }
324 porttid = pd->port_port * dd->ipath_rcvtidcnt;
325 if (!pd->port_subport_cnt) {
326 tidcnt = dd->ipath_rcvtidcnt;
327 tid = pd->port_tidcursor;
328 tidoff = 0;
329 } else if (!subport) {
330 tidcnt = (dd->ipath_rcvtidcnt / pd->port_subport_cnt) +
331 (dd->ipath_rcvtidcnt % pd->port_subport_cnt);
332 tidoff = dd->ipath_rcvtidcnt - tidcnt;
333 porttid += tidoff;
334 tid = tidcursor_fp(fp);
335 } else {
336 tidcnt = dd->ipath_rcvtidcnt / pd->port_subport_cnt;
337 tidoff = tidcnt * (subport - 1);
338 porttid += tidoff;
339 tid = tidcursor_fp(fp);
340 }
341 if (cnt > tidcnt) {
342
343 dev_info(&dd->pcidev->dev, "Process tried to allocate %u "
344 "TIDs, only trying max (%u)\n", cnt, tidcnt);
345 cnt = tidcnt;
346 }
347 pagep = &((struct page **) pd->port_tid_pg_list)[tidoff];
348 tidlist = &((u16 *) &pagep[dd->ipath_rcvtidcnt])[tidoff];
349
350 memset(tidmap, 0, sizeof(tidmap));
351
352 ntids = tidcnt;
353 tidbase = (u64 __iomem *) (((char __iomem *) dd->ipath_kregbase) +
354 dd->ipath_rcvtidbase +
355 porttid * sizeof(*tidbase));
356
357 ipath_cdbg(VERBOSE, "Port%u %u tids, cursor %u, tidbase %p\n",
358 pd->port_port, cnt, tid, tidbase);
359
360
361 vaddr = ti->tidvaddr;
362 if (!access_ok(VERIFY_WRITE, (void __user *) vaddr,
363 cnt * PAGE_SIZE)) {
364 ipath_dbg("Fail vaddr %p, %u pages, !access_ok\n",
365 (void *)vaddr, cnt);
366 ret = -EFAULT;
367 goto done;
368 }
369 ret = ipath_get_user_pages(vaddr, cnt, pagep);
370 if (ret) {
371 if (ret == -EBUSY) {
372 ipath_dbg("Failed to lock addr %p, %u pages "
373 "(already locked)\n",
374 (void *) vaddr, cnt);
375
376
377
378
379
380
381 ret = 0;
382 } else {
383 dev_info(&dd->pcidev->dev,
384 "Failed to lock addr %p, %u pages: "
385 "errno %d\n", (void *) vaddr, cnt, -ret);
386 goto done;
387 }
388 }
389 for (i = 0; i < cnt; i++, vaddr += PAGE_SIZE) {
390 for (; ntids--; tid++) {
391 if (tid == tidcnt)
392 tid = 0;
393 if (!dd->ipath_pageshadow[porttid + tid])
394 break;
395 }
396 if (ntids < 0) {
397
398
399
400
401
402 ipath_dbg("Not enough free TIDs for %u pages "
403 "(index %d), failing\n", cnt, i);
404 i--;
405 ret = -ENOMEM;
406 break;
407 }
408 tidlist[i] = tid + tidoff;
409 ipath_cdbg(VERBOSE, "Updating idx %u to TID %u, "
410 "vaddr %lx\n", i, tid + tidoff, vaddr);
411
412 dd->ipath_pageshadow[porttid + tid] = pagep[i];
413 dd->ipath_physshadow[porttid + tid] = ipath_map_page(
414 dd->pcidev, pagep[i], 0, PAGE_SIZE,
415 PCI_DMA_FROMDEVICE);
416
417
418
419 __set_bit(tid, tidmap);
420 physaddr = dd->ipath_physshadow[porttid + tid];
421 ipath_stats.sps_pagelocks++;
422 ipath_cdbg(VERBOSE,
423 "TID %u, vaddr %lx, physaddr %llx pgp %p\n",
424 tid, vaddr, (unsigned long long) physaddr,
425 pagep[i]);
426 dd->ipath_f_put_tid(dd, &tidbase[tid], RCVHQ_RCV_TYPE_EXPECTED,
427 physaddr);
428
429
430
431
432 tid++;
433 }
434
435 if (ret) {
436 u32 limit;
437 cleanup:
438
439 ipath_dbg("After failure (ret=%d), undo %d of %d entries\n",
440 -ret, i, cnt);
441
442 limit = sizeof(tidmap) * BITS_PER_BYTE;
443 if (limit > tidcnt)
444
445 limit = tidcnt;
446 tid = find_first_bit((const unsigned long *)tidmap, limit);
447 for (; tid < limit; tid++) {
448 if (!test_bit(tid, tidmap))
449 continue;
450 if (dd->ipath_pageshadow[porttid + tid]) {
451 ipath_cdbg(VERBOSE, "Freeing TID %u\n",
452 tid);
453 dd->ipath_f_put_tid(dd, &tidbase[tid],
454 RCVHQ_RCV_TYPE_EXPECTED,
455 dd->ipath_tidinvalid);
456 pci_unmap_page(dd->pcidev,
457 dd->ipath_physshadow[porttid + tid],
458 PAGE_SIZE, PCI_DMA_FROMDEVICE);
459 dd->ipath_pageshadow[porttid + tid] = NULL;
460 ipath_stats.sps_pageunlocks++;
461 }
462 }
463 ipath_release_user_pages(pagep, cnt);
464 } else {
465
466
467
468
469
470 if (copy_to_user((void __user *)
471 (unsigned long) ti->tidlist,
472 tidlist, cnt * sizeof(*tidlist))) {
473 ret = -EFAULT;
474 goto cleanup;
475 }
476 if (copy_to_user((void __user *) (unsigned long) ti->tidmap,
477 tidmap, sizeof tidmap)) {
478 ret = -EFAULT;
479 goto cleanup;
480 }
481 if (tid == tidcnt)
482 tid = 0;
483 if (!pd->port_subport_cnt)
484 pd->port_tidcursor = tid;
485 else
486 tidcursor_fp(fp) = tid;
487 }
488
489done:
490 if (ret)
491 ipath_dbg("Failed to map %u TID pages, failing with %d\n",
492 ti->tidcnt, -ret);
493 return ret;
494}
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513static int ipath_tid_free(struct ipath_portdata *pd, unsigned subport,
514 const struct ipath_tid_info *ti)
515{
516 int ret = 0;
517 u32 tid, porttid, cnt, limit, tidcnt;
518 struct ipath_devdata *dd = pd->port_dd;
519 u64 __iomem *tidbase;
520 unsigned long tidmap[8];
521
522 if (!dd->ipath_pageshadow) {
523 ret = -ENOMEM;
524 goto done;
525 }
526
527 if (copy_from_user(tidmap, (void __user *)(unsigned long)ti->tidmap,
528 sizeof tidmap)) {
529 ret = -EFAULT;
530 goto done;
531 }
532
533 porttid = pd->port_port * dd->ipath_rcvtidcnt;
534 if (!pd->port_subport_cnt)
535 tidcnt = dd->ipath_rcvtidcnt;
536 else if (!subport) {
537 tidcnt = (dd->ipath_rcvtidcnt / pd->port_subport_cnt) +
538 (dd->ipath_rcvtidcnt % pd->port_subport_cnt);
539 porttid += dd->ipath_rcvtidcnt - tidcnt;
540 } else {
541 tidcnt = dd->ipath_rcvtidcnt / pd->port_subport_cnt;
542 porttid += tidcnt * (subport - 1);
543 }
544 tidbase = (u64 __iomem *) ((char __iomem *)(dd->ipath_kregbase) +
545 dd->ipath_rcvtidbase +
546 porttid * sizeof(*tidbase));
547
548 limit = sizeof(tidmap) * BITS_PER_BYTE;
549 if (limit > tidcnt)
550
551 limit = tidcnt;
552 tid = find_first_bit(tidmap, limit);
553 ipath_cdbg(VERBOSE, "Port%u free %u tids; first bit (max=%d) "
554 "set is %d, porttid %u\n", pd->port_port, ti->tidcnt,
555 limit, tid, porttid);
556 for (cnt = 0; tid < limit; tid++) {
557
558
559
560
561
562
563
564 if (!test_bit(tid, tidmap))
565 continue;
566 cnt++;
567 if (dd->ipath_pageshadow[porttid + tid]) {
568 struct page *p;
569 p = dd->ipath_pageshadow[porttid + tid];
570 dd->ipath_pageshadow[porttid + tid] = NULL;
571 ipath_cdbg(VERBOSE, "PID %u freeing TID %u\n",
572 pid_nr(pd->port_pid), tid);
573 dd->ipath_f_put_tid(dd, &tidbase[tid],
574 RCVHQ_RCV_TYPE_EXPECTED,
575 dd->ipath_tidinvalid);
576 pci_unmap_page(dd->pcidev,
577 dd->ipath_physshadow[porttid + tid],
578 PAGE_SIZE, PCI_DMA_FROMDEVICE);
579 ipath_release_user_pages(&p, 1);
580 ipath_stats.sps_pageunlocks++;
581 } else
582 ipath_dbg("Unused tid %u, ignoring\n", tid);
583 }
584 if (cnt != ti->tidcnt)
585 ipath_dbg("passed in tidcnt %d, only %d bits set in map\n",
586 ti->tidcnt, cnt);
587done:
588 if (ret)
589 ipath_dbg("Failed to unmap %u TID pages, failing with %d\n",
590 ti->tidcnt, -ret);
591 return ret;
592}
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610static int ipath_set_part_key(struct ipath_portdata *pd, u16 key)
611{
612 struct ipath_devdata *dd = pd->port_dd;
613 int i, any = 0, pidx = -1;
614 u16 lkey = key & 0x7FFF;
615 int ret;
616
617 if (lkey == (IPATH_DEFAULT_P_KEY & 0x7FFF)) {
618
619 ret = 0;
620 goto bail;
621 }
622
623 ipath_cdbg(VERBOSE, "p%u try to set pkey %hx, current keys "
624 "%hx:%x %hx:%x %hx:%x %hx:%x\n",
625 pd->port_port, key, dd->ipath_pkeys[0],
626 atomic_read(&dd->ipath_pkeyrefs[0]), dd->ipath_pkeys[1],
627 atomic_read(&dd->ipath_pkeyrefs[1]), dd->ipath_pkeys[2],
628 atomic_read(&dd->ipath_pkeyrefs[2]), dd->ipath_pkeys[3],
629 atomic_read(&dd->ipath_pkeyrefs[3]));
630
631 if (!lkey) {
632 ipath_cdbg(PROC, "p%u tries to set key 0, not allowed\n",
633 pd->port_port);
634 ret = -EINVAL;
635 goto bail;
636 }
637
638
639
640
641
642
643
644 key |= 0x8000;
645
646 for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
647 if (!pd->port_pkeys[i] && pidx == -1)
648 pidx = i;
649 if (pd->port_pkeys[i] == key) {
650 ipath_cdbg(VERBOSE, "p%u tries to set same pkey "
651 "(%x) more than once\n",
652 pd->port_port, key);
653 ret = -EEXIST;
654 goto bail;
655 }
656 }
657 if (pidx == -1) {
658 ipath_dbg("All pkeys for port %u already in use, "
659 "can't set %x\n", pd->port_port, key);
660 ret = -EBUSY;
661 goto bail;
662 }
663 for (any = i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
664 if (!dd->ipath_pkeys[i]) {
665 any++;
666 continue;
667 }
668 if (dd->ipath_pkeys[i] == key) {
669 atomic_t *pkrefs = &dd->ipath_pkeyrefs[i];
670
671 if (atomic_inc_return(pkrefs) > 1) {
672 pd->port_pkeys[pidx] = key;
673 ipath_cdbg(VERBOSE, "p%u set key %x "
674 "matches #%d, count now %d\n",
675 pd->port_port, key, i,
676 atomic_read(pkrefs));
677 ret = 0;
678 goto bail;
679 } else {
680
681
682
683 atomic_dec(pkrefs);
684 ipath_cdbg(VERBOSE, "Lost race, count was "
685 "0, after dec, it's %d\n",
686 atomic_read(pkrefs));
687 any++;
688 }
689 }
690 if ((dd->ipath_pkeys[i] & 0x7FFF) == lkey) {
691
692
693
694
695
696 ret = -EEXIST;
697 goto bail;
698 }
699 }
700 if (!any) {
701 ipath_dbg("port %u, all pkeys already in use, "
702 "can't set %x\n", pd->port_port, key);
703 ret = -EBUSY;
704 goto bail;
705 }
706 for (any = i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
707 if (!dd->ipath_pkeys[i] &&
708 atomic_inc_return(&dd->ipath_pkeyrefs[i]) == 1) {
709 u64 pkey;
710
711
712 ipath_stats.sps_pkeys[i] = lkey;
713 pd->port_pkeys[pidx] = dd->ipath_pkeys[i] = key;
714 pkey =
715 (u64) dd->ipath_pkeys[0] |
716 ((u64) dd->ipath_pkeys[1] << 16) |
717 ((u64) dd->ipath_pkeys[2] << 32) |
718 ((u64) dd->ipath_pkeys[3] << 48);
719 ipath_cdbg(PROC, "p%u set key %x in #%d, "
720 "portidx %d, new pkey reg %llx\n",
721 pd->port_port, key, i, pidx,
722 (unsigned long long) pkey);
723 ipath_write_kreg(
724 dd, dd->ipath_kregs->kr_partitionkey, pkey);
725
726 ret = 0;
727 goto bail;
728 }
729 }
730 ipath_dbg("port %u, all pkeys already in use 2nd pass, "
731 "can't set %x\n", pd->port_port, key);
732 ret = -EBUSY;
733
734bail:
735 return ret;
736}
737
738
739
740
741
742
743
744
745
746
747
748static int ipath_manage_rcvq(struct ipath_portdata *pd, unsigned subport,
749 int start_stop)
750{
751 struct ipath_devdata *dd = pd->port_dd;
752
753 ipath_cdbg(PROC, "%sabling rcv for unit %u port %u:%u\n",
754 start_stop ? "en" : "dis", dd->ipath_unit,
755 pd->port_port, subport);
756 if (subport)
757 goto bail;
758
759 if (start_stop) {
760
761
762
763
764
765
766
767
768
769
770
771
772 if (pd->port_rcvhdrtail_kvaddr)
773 ipath_clear_rcvhdrtail(pd);
774 set_bit(dd->ipath_r_portenable_shift + pd->port_port,
775 &dd->ipath_rcvctrl);
776 } else
777 clear_bit(dd->ipath_r_portenable_shift + pd->port_port,
778 &dd->ipath_rcvctrl);
779 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
780 dd->ipath_rcvctrl);
781
782 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
783 if (start_stop) {
784
785
786
787
788
789
790
791 ipath_read_ureg32(dd, ur_rcvhdrtail, pd->port_port);
792 }
793
794bail:
795 return 0;
796}
797
798static void ipath_clean_part_key(struct ipath_portdata *pd,
799 struct ipath_devdata *dd)
800{
801 int i, j, pchanged = 0;
802 u64 oldpkey;
803
804
805 oldpkey = (u64) dd->ipath_pkeys[0] |
806 ((u64) dd->ipath_pkeys[1] << 16) |
807 ((u64) dd->ipath_pkeys[2] << 32) |
808 ((u64) dd->ipath_pkeys[3] << 48);
809
810 for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
811 if (!pd->port_pkeys[i])
812 continue;
813 ipath_cdbg(VERBOSE, "look for key[%d] %hx in pkeys\n", i,
814 pd->port_pkeys[i]);
815 for (j = 0; j < ARRAY_SIZE(dd->ipath_pkeys); j++) {
816
817 if ((dd->ipath_pkeys[j] & 0x7fff) !=
818 (pd->port_pkeys[i] & 0x7fff))
819 continue;
820 if (atomic_dec_and_test(&dd->ipath_pkeyrefs[j])) {
821 ipath_cdbg(VERBOSE, "p%u clear key "
822 "%x matches #%d\n",
823 pd->port_port,
824 pd->port_pkeys[i], j);
825 ipath_stats.sps_pkeys[j] =
826 dd->ipath_pkeys[j] = 0;
827 pchanged++;
828 }
829 else ipath_cdbg(
830 VERBOSE, "p%u key %x matches #%d, "
831 "but ref still %d\n", pd->port_port,
832 pd->port_pkeys[i], j,
833 atomic_read(&dd->ipath_pkeyrefs[j]));
834 break;
835 }
836 pd->port_pkeys[i] = 0;
837 }
838 if (pchanged) {
839 u64 pkey = (u64) dd->ipath_pkeys[0] |
840 ((u64) dd->ipath_pkeys[1] << 16) |
841 ((u64) dd->ipath_pkeys[2] << 32) |
842 ((u64) dd->ipath_pkeys[3] << 48);
843 ipath_cdbg(VERBOSE, "p%u old pkey reg %llx, "
844 "new pkey reg %llx\n", pd->port_port,
845 (unsigned long long) oldpkey,
846 (unsigned long long) pkey);
847 ipath_write_kreg(dd, dd->ipath_kregs->kr_partitionkey,
848 pkey);
849 }
850}
851
852
853
854
855
856
857
858static void init_user_egr_sizes(struct ipath_portdata *pd)
859{
860 struct ipath_devdata *dd = pd->port_dd;
861 unsigned egrperchunk, egrcnt, size;
862
863
864
865
866
867
868
869
870
871
872
873
874 size = 0x8000;
875 egrperchunk = size / dd->ipath_rcvegrbufsize;
876 egrcnt = dd->ipath_rcvegrcnt;
877 pd->port_rcvegrbuf_chunks = (egrcnt + egrperchunk - 1) / egrperchunk;
878 pd->port_rcvegrbufs_perchunk = egrperchunk;
879 pd->port_rcvegrbuf_size = size;
880}
881
882
883
884
885
886
887
888
889
890
891
892
893
894static int ipath_create_user_egr(struct ipath_portdata *pd)
895{
896 struct ipath_devdata *dd = pd->port_dd;
897 unsigned e, egrcnt, egrperchunk, chunk, egrsize, egroff;
898 size_t size;
899 int ret;
900 gfp_t gfp_flags;
901
902
903
904
905
906
907
908 gfp_flags = __GFP_WAIT | __GFP_IO | __GFP_COMP;
909
910 egrcnt = dd->ipath_rcvegrcnt;
911
912 egroff = (pd->port_port - 1) * egrcnt + dd->ipath_p0_rcvegrcnt;
913 egrsize = dd->ipath_rcvegrbufsize;
914 ipath_cdbg(VERBOSE, "Allocating %d egr buffers, at egrtid "
915 "offset %x, egrsize %u\n", egrcnt, egroff, egrsize);
916
917 chunk = pd->port_rcvegrbuf_chunks;
918 egrperchunk = pd->port_rcvegrbufs_perchunk;
919 size = pd->port_rcvegrbuf_size;
920 pd->port_rcvegrbuf = kmalloc(chunk * sizeof(pd->port_rcvegrbuf[0]),
921 GFP_KERNEL);
922 if (!pd->port_rcvegrbuf) {
923 ret = -ENOMEM;
924 goto bail;
925 }
926 pd->port_rcvegrbuf_phys =
927 kmalloc(chunk * sizeof(pd->port_rcvegrbuf_phys[0]),
928 GFP_KERNEL);
929 if (!pd->port_rcvegrbuf_phys) {
930 ret = -ENOMEM;
931 goto bail_rcvegrbuf;
932 }
933 for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
934
935 pd->port_rcvegrbuf[e] = dma_alloc_coherent(
936 &dd->pcidev->dev, size, &pd->port_rcvegrbuf_phys[e],
937 gfp_flags);
938
939 if (!pd->port_rcvegrbuf[e]) {
940 ret = -ENOMEM;
941 goto bail_rcvegrbuf_phys;
942 }
943 }
944
945 pd->port_rcvegr_phys = pd->port_rcvegrbuf_phys[0];
946
947 for (e = chunk = 0; chunk < pd->port_rcvegrbuf_chunks; chunk++) {
948 dma_addr_t pa = pd->port_rcvegrbuf_phys[chunk];
949 unsigned i;
950
951 for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) {
952 dd->ipath_f_put_tid(dd, e + egroff +
953 (u64 __iomem *)
954 ((char __iomem *)
955 dd->ipath_kregbase +
956 dd->ipath_rcvegrbase),
957 RCVHQ_RCV_TYPE_EAGER, pa);
958 pa += egrsize;
959 }
960 cond_resched();
961 }
962
963 ret = 0;
964 goto bail;
965
966bail_rcvegrbuf_phys:
967 for (e = 0; e < pd->port_rcvegrbuf_chunks &&
968 pd->port_rcvegrbuf[e]; e++) {
969 dma_free_coherent(&dd->pcidev->dev, size,
970 pd->port_rcvegrbuf[e],
971 pd->port_rcvegrbuf_phys[e]);
972
973 }
974 kfree(pd->port_rcvegrbuf_phys);
975 pd->port_rcvegrbuf_phys = NULL;
976bail_rcvegrbuf:
977 kfree(pd->port_rcvegrbuf);
978 pd->port_rcvegrbuf = NULL;
979bail:
980 return ret;
981}
982
983
984
985static int ipath_mmap_mem(struct vm_area_struct *vma,
986 struct ipath_portdata *pd, unsigned len, int write_ok,
987 void *kvaddr, char *what)
988{
989 struct ipath_devdata *dd = pd->port_dd;
990 unsigned long pfn;
991 int ret;
992
993 if ((vma->vm_end - vma->vm_start) > len) {
994 dev_info(&dd->pcidev->dev,
995 "FAIL on %s: len %lx > %x\n", what,
996 vma->vm_end - vma->vm_start, len);
997 ret = -EFAULT;
998 goto bail;
999 }
1000
1001 if (!write_ok) {
1002 if (vma->vm_flags & VM_WRITE) {
1003 dev_info(&dd->pcidev->dev,
1004 "%s must be mapped readonly\n", what);
1005 ret = -EPERM;
1006 goto bail;
1007 }
1008
1009
1010 vma->vm_flags &= ~VM_MAYWRITE;
1011 }
1012
1013 pfn = virt_to_phys(kvaddr) >> PAGE_SHIFT;
1014 ret = remap_pfn_range(vma, vma->vm_start, pfn,
1015 len, vma->vm_page_prot);
1016 if (ret)
1017 dev_info(&dd->pcidev->dev, "%s port%u mmap of %lx, %x "
1018 "bytes r%c failed: %d\n", what, pd->port_port,
1019 pfn, len, write_ok?'w':'o', ret);
1020 else
1021 ipath_cdbg(VERBOSE, "%s port%u mmaped %lx, %x bytes "
1022 "r%c\n", what, pd->port_port, pfn, len,
1023 write_ok?'w':'o');
1024bail:
1025 return ret;
1026}
1027
1028static int mmap_ureg(struct vm_area_struct *vma, struct ipath_devdata *dd,
1029 u64 ureg)
1030{
1031 unsigned long phys;
1032 int ret;
1033
1034
1035
1036
1037
1038
1039 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
1040 dev_info(&dd->pcidev->dev, "FAIL mmap userreg: reqlen "
1041 "%lx > PAGE\n", vma->vm_end - vma->vm_start);
1042 ret = -EFAULT;
1043 } else {
1044 phys = dd->ipath_physaddr + ureg;
1045 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1046
1047 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
1048 ret = io_remap_pfn_range(vma, vma->vm_start,
1049 phys >> PAGE_SHIFT,
1050 vma->vm_end - vma->vm_start,
1051 vma->vm_page_prot);
1052 }
1053 return ret;
1054}
1055
1056static int mmap_piobufs(struct vm_area_struct *vma,
1057 struct ipath_devdata *dd,
1058 struct ipath_portdata *pd,
1059 unsigned piobufs, unsigned piocnt)
1060{
1061 unsigned long phys;
1062 int ret;
1063
1064
1065
1066
1067
1068
1069
1070 if ((vma->vm_end - vma->vm_start) > (piocnt * dd->ipath_palign)) {
1071 dev_info(&dd->pcidev->dev, "FAIL mmap piobufs: "
1072 "reqlen %lx > PAGE\n",
1073 vma->vm_end - vma->vm_start);
1074 ret = -EINVAL;
1075 goto bail;
1076 }
1077
1078 phys = dd->ipath_physaddr + piobufs;
1079
1080#if defined(__powerpc__)
1081
1082 pgprot_val(vma->vm_page_prot) |= _PAGE_NO_CACHE;
1083 pgprot_val(vma->vm_page_prot) |= _PAGE_WRITETHRU;
1084 pgprot_val(vma->vm_page_prot) &= ~_PAGE_GUARDED;
1085#endif
1086
1087
1088
1089
1090
1091 vma->vm_flags &= ~VM_MAYREAD;
1092 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
1093
1094 ret = io_remap_pfn_range(vma, vma->vm_start, phys >> PAGE_SHIFT,
1095 vma->vm_end - vma->vm_start,
1096 vma->vm_page_prot);
1097bail:
1098 return ret;
1099}
1100
1101static int mmap_rcvegrbufs(struct vm_area_struct *vma,
1102 struct ipath_portdata *pd)
1103{
1104 struct ipath_devdata *dd = pd->port_dd;
1105 unsigned long start, size;
1106 size_t total_size, i;
1107 unsigned long pfn;
1108 int ret;
1109
1110 size = pd->port_rcvegrbuf_size;
1111 total_size = pd->port_rcvegrbuf_chunks * size;
1112 if ((vma->vm_end - vma->vm_start) > total_size) {
1113 dev_info(&dd->pcidev->dev, "FAIL on egr bufs: "
1114 "reqlen %lx > actual %lx\n",
1115 vma->vm_end - vma->vm_start,
1116 (unsigned long) total_size);
1117 ret = -EINVAL;
1118 goto bail;
1119 }
1120
1121 if (vma->vm_flags & VM_WRITE) {
1122 dev_info(&dd->pcidev->dev, "Can't map eager buffers as "
1123 "writable (flags=%lx)\n", vma->vm_flags);
1124 ret = -EPERM;
1125 goto bail;
1126 }
1127
1128 vma->vm_flags &= ~VM_MAYWRITE;
1129
1130 start = vma->vm_start;
1131
1132 for (i = 0; i < pd->port_rcvegrbuf_chunks; i++, start += size) {
1133 pfn = virt_to_phys(pd->port_rcvegrbuf[i]) >> PAGE_SHIFT;
1134 ret = remap_pfn_range(vma, start, pfn, size,
1135 vma->vm_page_prot);
1136 if (ret < 0)
1137 goto bail;
1138 }
1139 ret = 0;
1140
1141bail:
1142 return ret;
1143}
1144
1145
1146
1147
1148static int ipath_file_vma_fault(struct vm_area_struct *vma,
1149 struct vm_fault *vmf)
1150{
1151 struct page *page;
1152
1153 page = vmalloc_to_page((void *)(vmf->pgoff << PAGE_SHIFT));
1154 if (!page)
1155 return VM_FAULT_SIGBUS;
1156 get_page(page);
1157 vmf->page = page;
1158
1159 return 0;
1160}
1161
1162static const struct vm_operations_struct ipath_file_vm_ops = {
1163 .fault = ipath_file_vma_fault,
1164};
1165
1166static int mmap_kvaddr(struct vm_area_struct *vma, u64 pgaddr,
1167 struct ipath_portdata *pd, unsigned subport)
1168{
1169 unsigned long len;
1170 struct ipath_devdata *dd;
1171 void *addr;
1172 size_t size;
1173 int ret = 0;
1174
1175
1176 if (!pd->port_subport_cnt)
1177 goto bail;
1178
1179 dd = pd->port_dd;
1180 size = pd->port_rcvegrbuf_chunks * pd->port_rcvegrbuf_size;
1181
1182
1183
1184
1185
1186
1187 if (pgaddr == cvt_kvaddr(pd->subport_uregbase)) {
1188 addr = pd->subport_uregbase;
1189 size = PAGE_SIZE * pd->port_subport_cnt;
1190 } else if (pgaddr == cvt_kvaddr(pd->subport_rcvhdr_base)) {
1191 addr = pd->subport_rcvhdr_base;
1192 size = pd->port_rcvhdrq_size * pd->port_subport_cnt;
1193 } else if (pgaddr == cvt_kvaddr(pd->subport_rcvegrbuf)) {
1194 addr = pd->subport_rcvegrbuf;
1195 size *= pd->port_subport_cnt;
1196 } else if (pgaddr == cvt_kvaddr(pd->subport_uregbase +
1197 PAGE_SIZE * subport)) {
1198 addr = pd->subport_uregbase + PAGE_SIZE * subport;
1199 size = PAGE_SIZE;
1200 } else if (pgaddr == cvt_kvaddr(pd->subport_rcvhdr_base +
1201 pd->port_rcvhdrq_size * subport)) {
1202 addr = pd->subport_rcvhdr_base +
1203 pd->port_rcvhdrq_size * subport;
1204 size = pd->port_rcvhdrq_size;
1205 } else if (pgaddr == cvt_kvaddr(pd->subport_rcvegrbuf +
1206 size * subport)) {
1207 addr = pd->subport_rcvegrbuf + size * subport;
1208
1209 if (vma->vm_flags & VM_WRITE) {
1210 dev_info(&dd->pcidev->dev,
1211 "Can't map eager buffers as "
1212 "writable (flags=%lx)\n", vma->vm_flags);
1213 ret = -EPERM;
1214 goto bail;
1215 }
1216
1217
1218
1219
1220 vma->vm_flags &= ~VM_MAYWRITE;
1221 } else {
1222 goto bail;
1223 }
1224 len = vma->vm_end - vma->vm_start;
1225 if (len > size) {
1226 ipath_cdbg(MM, "FAIL: reqlen %lx > %zx\n", len, size);
1227 ret = -EINVAL;
1228 goto bail;
1229 }
1230
1231 vma->vm_pgoff = (unsigned long) addr >> PAGE_SHIFT;
1232 vma->vm_ops = &ipath_file_vm_ops;
1233 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
1234 ret = 1;
1235
1236bail:
1237 return ret;
1238}
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250static int ipath_mmap(struct file *fp, struct vm_area_struct *vma)
1251{
1252 struct ipath_portdata *pd;
1253 struct ipath_devdata *dd;
1254 u64 pgaddr, ureg;
1255 unsigned piobufs, piocnt;
1256 int ret;
1257
1258 pd = port_fp(fp);
1259 if (!pd) {
1260 ret = -EINVAL;
1261 goto bail;
1262 }
1263 dd = pd->port_dd;
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273 pgaddr = vma->vm_pgoff << PAGE_SHIFT;
1274
1275
1276
1277
1278
1279 if (!pgaddr) {
1280 ret = -EINVAL;
1281 goto bail;
1282 }
1283
1284 ipath_cdbg(MM, "pgaddr %llx vm_start=%lx len %lx port %u:%u:%u\n",
1285 (unsigned long long) pgaddr, vma->vm_start,
1286 vma->vm_end - vma->vm_start, dd->ipath_unit,
1287 pd->port_port, subport_fp(fp));
1288
1289
1290
1291
1292
1293
1294 ret = mmap_kvaddr(vma, pgaddr, pd, subport_fp(fp));
1295 if (ret) {
1296 if (ret > 0)
1297 ret = 0;
1298 goto bail;
1299 }
1300
1301 ureg = dd->ipath_uregbase + dd->ipath_ureg_align * pd->port_port;
1302 if (!pd->port_subport_cnt) {
1303
1304 piocnt = pd->port_piocnt;
1305 piobufs = pd->port_piobufs;
1306 } else if (!subport_fp(fp)) {
1307
1308 piocnt = (pd->port_piocnt / pd->port_subport_cnt) +
1309 (pd->port_piocnt % pd->port_subport_cnt);
1310 piobufs = pd->port_piobufs +
1311 dd->ipath_palign * (pd->port_piocnt - piocnt);
1312 } else {
1313 unsigned slave = subport_fp(fp) - 1;
1314
1315
1316 piocnt = pd->port_piocnt / pd->port_subport_cnt;
1317 piobufs = pd->port_piobufs + dd->ipath_palign * piocnt * slave;
1318 }
1319
1320 if (pgaddr == ureg)
1321 ret = mmap_ureg(vma, dd, ureg);
1322 else if (pgaddr == piobufs)
1323 ret = mmap_piobufs(vma, dd, pd, piobufs, piocnt);
1324 else if (pgaddr == dd->ipath_pioavailregs_phys)
1325
1326 ret = ipath_mmap_mem(vma, pd, PAGE_SIZE, 0,
1327 (void *) dd->ipath_pioavailregs_dma,
1328 "pioavail registers");
1329 else if (pgaddr == pd->port_rcvegr_phys)
1330 ret = mmap_rcvegrbufs(vma, pd);
1331 else if (pgaddr == (u64) pd->port_rcvhdrq_phys)
1332
1333
1334
1335
1336
1337 ret = ipath_mmap_mem(vma, pd, pd->port_rcvhdrq_size, 1,
1338 pd->port_rcvhdrq,
1339 "rcvhdrq");
1340 else if (pgaddr == (u64) pd->port_rcvhdrqtailaddr_phys)
1341
1342 ret = ipath_mmap_mem(vma, pd, PAGE_SIZE, 0,
1343 pd->port_rcvhdrtail_kvaddr,
1344 "rcvhdrq tail");
1345 else
1346 ret = -EINVAL;
1347
1348 vma->vm_private_data = NULL;
1349
1350 if (ret < 0)
1351 dev_info(&dd->pcidev->dev,
1352 "Failure %d on off %llx len %lx\n",
1353 -ret, (unsigned long long)pgaddr,
1354 vma->vm_end - vma->vm_start);
1355bail:
1356 return ret;
1357}
1358
1359static unsigned ipath_poll_hdrqfull(struct ipath_portdata *pd)
1360{
1361 unsigned pollflag = 0;
1362
1363 if ((pd->poll_type & IPATH_POLL_TYPE_OVERFLOW) &&
1364 pd->port_hdrqfull != pd->port_hdrqfull_poll) {
1365 pollflag |= POLLIN | POLLRDNORM;
1366 pd->port_hdrqfull_poll = pd->port_hdrqfull;
1367 }
1368
1369 return pollflag;
1370}
1371
1372static unsigned int ipath_poll_urgent(struct ipath_portdata *pd,
1373 struct file *fp,
1374 struct poll_table_struct *pt)
1375{
1376 unsigned pollflag = 0;
1377 struct ipath_devdata *dd;
1378
1379 dd = pd->port_dd;
1380
1381
1382 rmb();
1383 pollflag = ipath_poll_hdrqfull(pd);
1384
1385 if (pd->port_urgent != pd->port_urgent_poll) {
1386 pollflag |= POLLIN | POLLRDNORM;
1387 pd->port_urgent_poll = pd->port_urgent;
1388 }
1389
1390 if (!pollflag) {
1391
1392 set_bit(IPATH_PORT_WAITING_URG, &pd->port_flag);
1393
1394 wmb();
1395 poll_wait(fp, &pd->port_wait, pt);
1396 }
1397
1398 return pollflag;
1399}
1400
1401static unsigned int ipath_poll_next(struct ipath_portdata *pd,
1402 struct file *fp,
1403 struct poll_table_struct *pt)
1404{
1405 u32 head;
1406 u32 tail;
1407 unsigned pollflag = 0;
1408 struct ipath_devdata *dd;
1409
1410 dd = pd->port_dd;
1411
1412
1413 rmb();
1414 pollflag = ipath_poll_hdrqfull(pd);
1415
1416 head = ipath_read_ureg32(dd, ur_rcvhdrhead, pd->port_port);
1417 if (pd->port_rcvhdrtail_kvaddr)
1418 tail = ipath_get_rcvhdrtail(pd);
1419 else
1420 tail = ipath_read_ureg32(dd, ur_rcvhdrtail, pd->port_port);
1421
1422 if (head != tail)
1423 pollflag |= POLLIN | POLLRDNORM;
1424 else {
1425
1426 set_bit(IPATH_PORT_WAITING_RCV, &pd->port_flag);
1427
1428 wmb();
1429
1430 set_bit(pd->port_port + dd->ipath_r_intravail_shift,
1431 &dd->ipath_rcvctrl);
1432
1433 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
1434 dd->ipath_rcvctrl);
1435
1436 if (dd->ipath_rhdrhead_intr_off)
1437 ipath_write_ureg(dd, ur_rcvhdrhead,
1438 dd->ipath_rhdrhead_intr_off | head,
1439 pd->port_port);
1440
1441 poll_wait(fp, &pd->port_wait, pt);
1442 }
1443
1444 return pollflag;
1445}
1446
1447static unsigned int ipath_poll(struct file *fp,
1448 struct poll_table_struct *pt)
1449{
1450 struct ipath_portdata *pd;
1451 unsigned pollflag;
1452
1453 pd = port_fp(fp);
1454 if (!pd)
1455 pollflag = 0;
1456 else if (pd->poll_type & IPATH_POLL_TYPE_URGENT)
1457 pollflag = ipath_poll_urgent(pd, fp, pt);
1458 else
1459 pollflag = ipath_poll_next(pd, fp, pt);
1460
1461 return pollflag;
1462}
1463
1464static int ipath_supports_subports(int user_swmajor, int user_swminor)
1465{
1466
1467 return (user_swmajor > 1) || (user_swminor >= 3);
1468}
1469
1470static int ipath_compatible_subports(int user_swmajor, int user_swminor)
1471{
1472
1473 if (IPATH_USER_SWMAJOR != user_swmajor) {
1474
1475 return 0;
1476 }
1477 if (IPATH_USER_SWMAJOR == 1) {
1478 switch (IPATH_USER_SWMINOR) {
1479 case 0:
1480 case 1:
1481 case 2:
1482
1483 return 0;
1484 case 3:
1485
1486 return user_swminor == 3;
1487 default:
1488
1489 return user_swminor >= 4;
1490 }
1491 }
1492
1493 return 0;
1494}
1495
1496static int init_subports(struct ipath_devdata *dd,
1497 struct ipath_portdata *pd,
1498 const struct ipath_user_info *uinfo)
1499{
1500 int ret = 0;
1501 unsigned num_subports;
1502 size_t size;
1503
1504
1505
1506
1507
1508 if (uinfo->spu_subport_cnt <= 0)
1509 goto bail;
1510
1511
1512 if (ipath_supports_subports(IPATH_USER_SWMAJOR, IPATH_USER_SWMINOR) &&
1513 !ipath_compatible_subports(IPATH_USER_SWMAJOR,
1514 IPATH_USER_SWMINOR)) {
1515 dev_info(&dd->pcidev->dev,
1516 "Inconsistent ipath_compatible_subports()\n");
1517 goto bail;
1518 }
1519
1520
1521 if (!ipath_compatible_subports(uinfo->spu_userversion >> 16,
1522 uinfo->spu_userversion & 0xffff)) {
1523 dev_info(&dd->pcidev->dev,
1524 "Mismatched user version (%d.%d) and driver "
1525 "version (%d.%d) while port sharing. Ensure "
1526 "that driver and library are from the same "
1527 "release.\n",
1528 (int) (uinfo->spu_userversion >> 16),
1529 (int) (uinfo->spu_userversion & 0xffff),
1530 IPATH_USER_SWMAJOR,
1531 IPATH_USER_SWMINOR);
1532 goto bail;
1533 }
1534 if (uinfo->spu_subport_cnt > INFINIPATH_MAX_SUBPORT) {
1535 ret = -EINVAL;
1536 goto bail;
1537 }
1538
1539 num_subports = uinfo->spu_subport_cnt;
1540 pd->subport_uregbase = vzalloc(PAGE_SIZE * num_subports);
1541 if (!pd->subport_uregbase) {
1542 ret = -ENOMEM;
1543 goto bail;
1544 }
1545
1546 size = ALIGN(dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize *
1547 sizeof(u32), PAGE_SIZE) * num_subports;
1548 pd->subport_rcvhdr_base = vzalloc(size);
1549 if (!pd->subport_rcvhdr_base) {
1550 ret = -ENOMEM;
1551 goto bail_ureg;
1552 }
1553
1554 pd->subport_rcvegrbuf = vzalloc(pd->port_rcvegrbuf_chunks *
1555 pd->port_rcvegrbuf_size *
1556 num_subports);
1557 if (!pd->subport_rcvegrbuf) {
1558 ret = -ENOMEM;
1559 goto bail_rhdr;
1560 }
1561
1562 pd->port_subport_cnt = uinfo->spu_subport_cnt;
1563 pd->port_subport_id = uinfo->spu_subport_id;
1564 pd->active_slaves = 1;
1565 set_bit(IPATH_PORT_MASTER_UNINIT, &pd->port_flag);
1566 goto bail;
1567
1568bail_rhdr:
1569 vfree(pd->subport_rcvhdr_base);
1570bail_ureg:
1571 vfree(pd->subport_uregbase);
1572 pd->subport_uregbase = NULL;
1573bail:
1574 return ret;
1575}
1576
1577static int try_alloc_port(struct ipath_devdata *dd, int port,
1578 struct file *fp,
1579 const struct ipath_user_info *uinfo)
1580{
1581 struct ipath_portdata *pd;
1582 int ret;
1583
1584 if (!(pd = dd->ipath_pd[port])) {
1585 void *ptmp;
1586
1587 pd = kzalloc(sizeof(struct ipath_portdata), GFP_KERNEL);
1588
1589
1590
1591
1592
1593
1594 ptmp = kmalloc(dd->ipath_rcvtidcnt * sizeof(u16) +
1595 dd->ipath_rcvtidcnt * sizeof(struct page **),
1596 GFP_KERNEL);
1597 if (!pd || !ptmp) {
1598 ipath_dev_err(dd, "Unable to allocate portdata "
1599 "memory, failing open\n");
1600 ret = -ENOMEM;
1601 kfree(pd);
1602 kfree(ptmp);
1603 goto bail;
1604 }
1605 dd->ipath_pd[port] = pd;
1606 dd->ipath_pd[port]->port_port = port;
1607 dd->ipath_pd[port]->port_dd = dd;
1608 dd->ipath_pd[port]->port_tid_pg_list = ptmp;
1609 init_waitqueue_head(&dd->ipath_pd[port]->port_wait);
1610 }
1611 if (!pd->port_cnt) {
1612 pd->userversion = uinfo->spu_userversion;
1613 init_user_egr_sizes(pd);
1614 if ((ret = init_subports(dd, pd, uinfo)) != 0)
1615 goto bail;
1616 ipath_cdbg(PROC, "%s[%u] opened unit:port %u:%u\n",
1617 current->comm, current->pid, dd->ipath_unit,
1618 port);
1619 pd->port_cnt = 1;
1620 port_fp(fp) = pd;
1621 pd->port_pid = get_pid(task_pid(current));
1622 strlcpy(pd->port_comm, current->comm, sizeof(pd->port_comm));
1623 ipath_stats.sps_ports++;
1624 ret = 0;
1625 } else
1626 ret = -EBUSY;
1627
1628bail:
1629 return ret;
1630}
1631
1632static inline int usable(struct ipath_devdata *dd)
1633{
1634 return dd &&
1635 (dd->ipath_flags & IPATH_PRESENT) &&
1636 dd->ipath_kregbase &&
1637 dd->ipath_lid &&
1638 !(dd->ipath_flags & (IPATH_LINKDOWN | IPATH_DISABLED
1639 | IPATH_LINKUNK));
1640}
1641
1642static int find_free_port(int unit, struct file *fp,
1643 const struct ipath_user_info *uinfo)
1644{
1645 struct ipath_devdata *dd = ipath_lookup(unit);
1646 int ret, i;
1647
1648 if (!dd) {
1649 ret = -ENODEV;
1650 goto bail;
1651 }
1652
1653 if (!usable(dd)) {
1654 ret = -ENETDOWN;
1655 goto bail;
1656 }
1657
1658 for (i = 1; i < dd->ipath_cfgports; i++) {
1659 ret = try_alloc_port(dd, i, fp, uinfo);
1660 if (ret != -EBUSY)
1661 goto bail;
1662 }
1663 ret = -EBUSY;
1664
1665bail:
1666 return ret;
1667}
1668
1669static int find_best_unit(struct file *fp,
1670 const struct ipath_user_info *uinfo)
1671{
1672 int ret = 0, i, prefunit = -1, devmax;
1673 int maxofallports, npresent, nup;
1674 int ndev;
1675
1676 devmax = ipath_count_units(&npresent, &nup, &maxofallports);
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694 if (!cpumask_empty(tsk_cpus_allowed(current)) &&
1695 !cpumask_full(tsk_cpus_allowed(current))) {
1696 int ncpus = num_online_cpus(), curcpu = -1, nset = 0;
1697 get_online_cpus();
1698 for_each_online_cpu(i)
1699 if (cpumask_test_cpu(i, tsk_cpus_allowed(current))) {
1700 ipath_cdbg(PROC, "%s[%u] affinity set for "
1701 "cpu %d/%d\n", current->comm,
1702 current->pid, i, ncpus);
1703 curcpu = i;
1704 nset++;
1705 }
1706 put_online_cpus();
1707 if (curcpu != -1 && nset != ncpus) {
1708 if (npresent) {
1709 prefunit = curcpu / (ncpus / npresent);
1710 ipath_cdbg(PROC,"%s[%u] %d chips, %d cpus, "
1711 "%d cpus/chip, select unit %d\n",
1712 current->comm, current->pid,
1713 npresent, ncpus, ncpus / npresent,
1714 prefunit);
1715 }
1716 }
1717 }
1718
1719
1720
1721
1722
1723
1724 if (prefunit != -1)
1725 devmax = prefunit + 1;
1726recheck:
1727 for (i = 1; i < maxofallports; i++) {
1728 for (ndev = prefunit != -1 ? prefunit : 0; ndev < devmax;
1729 ndev++) {
1730 struct ipath_devdata *dd = ipath_lookup(ndev);
1731
1732 if (!usable(dd))
1733 continue;
1734 if (i >= dd->ipath_cfgports)
1735
1736
1737
1738
1739 continue;
1740 ret = try_alloc_port(dd, i, fp, uinfo);
1741 if (!ret)
1742 goto done;
1743 }
1744 }
1745
1746 if (npresent) {
1747 if (nup == 0) {
1748 ret = -ENETDOWN;
1749 ipath_dbg("No ports available (none initialized "
1750 "and ready)\n");
1751 } else {
1752 if (prefunit > 0) {
1753
1754 ipath_cdbg(PROC,
1755 "%s[%u] no ports on prefunit "
1756 "%d, clear and re-check\n",
1757 current->comm, current->pid,
1758 prefunit);
1759 devmax = ipath_count_units(NULL, NULL,
1760 NULL);
1761 prefunit = -1;
1762 goto recheck;
1763 }
1764 ret = -EBUSY;
1765 ipath_dbg("No ports available\n");
1766 }
1767 } else {
1768 ret = -ENXIO;
1769 ipath_dbg("No boards found\n");
1770 }
1771
1772done:
1773 return ret;
1774}
1775
1776static int find_shared_port(struct file *fp,
1777 const struct ipath_user_info *uinfo)
1778{
1779 int devmax, ndev, i;
1780 int ret = 0;
1781
1782 devmax = ipath_count_units(NULL, NULL, NULL);
1783
1784 for (ndev = 0; ndev < devmax; ndev++) {
1785 struct ipath_devdata *dd = ipath_lookup(ndev);
1786
1787 if (!usable(dd))
1788 continue;
1789 for (i = 1; i < dd->ipath_cfgports; i++) {
1790 struct ipath_portdata *pd = dd->ipath_pd[i];
1791
1792
1793 if (!pd || !pd->port_cnt)
1794 continue;
1795
1796 if (pd->port_subport_id != uinfo->spu_subport_id)
1797 continue;
1798
1799 if (pd->port_subport_cnt != uinfo->spu_subport_cnt ||
1800 pd->userversion != uinfo->spu_userversion ||
1801 pd->port_cnt >= pd->port_subport_cnt) {
1802 ret = -EINVAL;
1803 goto done;
1804 }
1805 port_fp(fp) = pd;
1806 subport_fp(fp) = pd->port_cnt++;
1807 pd->port_subpid[subport_fp(fp)] =
1808 get_pid(task_pid(current));
1809 tidcursor_fp(fp) = 0;
1810 pd->active_slaves |= 1 << subport_fp(fp);
1811 ipath_cdbg(PROC,
1812 "%s[%u] %u sharing %s[%u] unit:port %u:%u\n",
1813 current->comm, current->pid,
1814 subport_fp(fp),
1815 pd->port_comm, pid_nr(pd->port_pid),
1816 dd->ipath_unit, pd->port_port);
1817 ret = 1;
1818 goto done;
1819 }
1820 }
1821
1822done:
1823 return ret;
1824}
1825
1826static int ipath_open(struct inode *in, struct file *fp)
1827{
1828
1829 fp->private_data = kzalloc(sizeof(struct ipath_filedata), GFP_KERNEL);
1830 return fp->private_data ? 0 : -ENOMEM;
1831}
1832
1833
1834static int ipath_assign_port(struct file *fp,
1835 const struct ipath_user_info *uinfo)
1836{
1837 int ret;
1838 int i_minor;
1839 unsigned swmajor, swminor;
1840
1841
1842 if (port_fp(fp)) {
1843 ret = -EINVAL;
1844 goto done;
1845 }
1846
1847
1848 swmajor = uinfo->spu_userversion >> 16;
1849 if (swmajor != IPATH_USER_SWMAJOR) {
1850 ipath_dbg("User major version %d not same as driver "
1851 "major %d\n", uinfo->spu_userversion >> 16,
1852 IPATH_USER_SWMAJOR);
1853 ret = -ENODEV;
1854 goto done;
1855 }
1856
1857 swminor = uinfo->spu_userversion & 0xffff;
1858 if (swminor != IPATH_USER_SWMINOR)
1859 ipath_dbg("User minor version %d not same as driver "
1860 "minor %d\n", swminor, IPATH_USER_SWMINOR);
1861
1862 mutex_lock(&ipath_mutex);
1863
1864 if (ipath_compatible_subports(swmajor, swminor) &&
1865 uinfo->spu_subport_cnt &&
1866 (ret = find_shared_port(fp, uinfo))) {
1867 if (ret > 0)
1868 ret = 0;
1869 goto done_chk_sdma;
1870 }
1871
1872 i_minor = iminor(file_inode(fp)) - IPATH_USER_MINOR_BASE;
1873 ipath_cdbg(VERBOSE, "open on dev %lx (minor %d)\n",
1874 (long)file_inode(fp)->i_rdev, i_minor);
1875
1876 if (i_minor)
1877 ret = find_free_port(i_minor - 1, fp, uinfo);
1878 else
1879 ret = find_best_unit(fp, uinfo);
1880
1881done_chk_sdma:
1882 if (!ret) {
1883 struct ipath_filedata *fd = fp->private_data;
1884 const struct ipath_portdata *pd = fd->pd;
1885 const struct ipath_devdata *dd = pd->port_dd;
1886
1887 fd->pq = ipath_user_sdma_queue_create(&dd->pcidev->dev,
1888 dd->ipath_unit,
1889 pd->port_port,
1890 fd->subport);
1891
1892 if (!fd->pq)
1893 ret = -ENOMEM;
1894 }
1895
1896 mutex_unlock(&ipath_mutex);
1897
1898done:
1899 return ret;
1900}
1901
1902
1903static int ipath_do_user_init(struct file *fp,
1904 const struct ipath_user_info *uinfo)
1905{
1906 int ret;
1907 struct ipath_portdata *pd = port_fp(fp);
1908 struct ipath_devdata *dd;
1909 u32 head32;
1910
1911
1912 if (subport_fp(fp)) {
1913 ret = wait_event_interruptible(pd->port_wait,
1914 !test_bit(IPATH_PORT_MASTER_UNINIT, &pd->port_flag));
1915 goto done;
1916 }
1917
1918 dd = pd->port_dd;
1919
1920 if (uinfo->spu_rcvhdrsize) {
1921 ret = ipath_setrcvhdrsize(dd, uinfo->spu_rcvhdrsize);
1922 if (ret)
1923 goto done;
1924 }
1925
1926
1927
1928
1929 if (pd->port_port <= dd->ipath_ports_extrabuf)
1930 pd->port_piocnt = dd->ipath_pbufsport + 1;
1931 else
1932 pd->port_piocnt = dd->ipath_pbufsport;
1933
1934
1935 if (pd->port_port <= dd->ipath_ports_extrabuf)
1936 pd->port_pio_base = (dd->ipath_pbufsport + 1)
1937 * (pd->port_port - 1);
1938 else
1939 pd->port_pio_base = dd->ipath_ports_extrabuf +
1940 dd->ipath_pbufsport * (pd->port_port - 1);
1941 pd->port_piobufs = dd->ipath_piobufbase +
1942 pd->port_pio_base * dd->ipath_palign;
1943 ipath_cdbg(VERBOSE, "piobuf base for port %u is 0x%x, piocnt %u,"
1944 " first pio %u\n", pd->port_port, pd->port_piobufs,
1945 pd->port_piocnt, pd->port_pio_base);
1946 ipath_chg_pioavailkernel(dd, pd->port_pio_base, pd->port_piocnt, 0);
1947
1948
1949
1950
1951
1952
1953
1954 ret = ipath_create_rcvhdrq(dd, pd);
1955 if (!ret)
1956 ret = ipath_create_user_egr(pd);
1957 if (ret)
1958 goto done;
1959
1960
1961
1962
1963
1964
1965 head32 = ipath_read_ureg32(dd, ur_rcvegrindextail, pd->port_port);
1966 ipath_write_ureg(dd, ur_rcvegrindexhead, head32, pd->port_port);
1967 pd->port_lastrcvhdrqtail = -1;
1968 ipath_cdbg(VERBOSE, "Wrote port%d egrhead %x from tail regs\n",
1969 pd->port_port, head32);
1970 pd->port_tidcursor = 0;
1971
1972
1973 pd->port_urgent = 0;
1974 pd->port_urgent_poll = 0;
1975 pd->port_hdrqfull_poll = pd->port_hdrqfull;
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988 set_bit(dd->ipath_r_portenable_shift + pd->port_port,
1989 &dd->ipath_rcvctrl);
1990 if (!(dd->ipath_flags & IPATH_NODMA_RTAIL)) {
1991 if (pd->port_rcvhdrtail_kvaddr)
1992 ipath_clear_rcvhdrtail(pd);
1993 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
1994 dd->ipath_rcvctrl &
1995 ~(1ULL << dd->ipath_r_tailupd_shift));
1996 }
1997 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
1998 dd->ipath_rcvctrl);
1999
2000 if (pd->port_subport_cnt) {
2001 clear_bit(IPATH_PORT_MASTER_UNINIT, &pd->port_flag);
2002 wake_up(&pd->port_wait);
2003 }
2004done:
2005 return ret;
2006}
2007
2008
2009
2010
2011
2012
2013
2014
2015static void unlock_expected_tids(struct ipath_portdata *pd)
2016{
2017 struct ipath_devdata *dd = pd->port_dd;
2018 int port_tidbase = pd->port_port * dd->ipath_rcvtidcnt;
2019 int i, cnt = 0, maxtid = port_tidbase + dd->ipath_rcvtidcnt;
2020
2021 ipath_cdbg(VERBOSE, "Port %u unlocking any locked expTID pages\n",
2022 pd->port_port);
2023 for (i = port_tidbase; i < maxtid; i++) {
2024 struct page *ps = dd->ipath_pageshadow[i];
2025
2026 if (!ps)
2027 continue;
2028
2029 dd->ipath_pageshadow[i] = NULL;
2030 pci_unmap_page(dd->pcidev, dd->ipath_physshadow[i],
2031 PAGE_SIZE, PCI_DMA_FROMDEVICE);
2032 ipath_release_user_pages_on_close(&ps, 1);
2033 cnt++;
2034 ipath_stats.sps_pageunlocks++;
2035 }
2036 if (cnt)
2037 ipath_cdbg(VERBOSE, "Port %u locked %u expTID entries\n",
2038 pd->port_port, cnt);
2039
2040 if (ipath_stats.sps_pagelocks || ipath_stats.sps_pageunlocks)
2041 ipath_cdbg(VERBOSE, "%llu pages locked, %llu unlocked\n",
2042 (unsigned long long) ipath_stats.sps_pagelocks,
2043 (unsigned long long)
2044 ipath_stats.sps_pageunlocks);
2045}
2046
2047static int ipath_close(struct inode *in, struct file *fp)
2048{
2049 int ret = 0;
2050 struct ipath_filedata *fd;
2051 struct ipath_portdata *pd;
2052 struct ipath_devdata *dd;
2053 unsigned long flags;
2054 unsigned port;
2055 struct pid *pid;
2056
2057 ipath_cdbg(VERBOSE, "close on dev %lx, private data %p\n",
2058 (long)in->i_rdev, fp->private_data);
2059
2060 mutex_lock(&ipath_mutex);
2061
2062 fd = fp->private_data;
2063 fp->private_data = NULL;
2064 pd = fd->pd;
2065 if (!pd) {
2066 mutex_unlock(&ipath_mutex);
2067 goto bail;
2068 }
2069
2070 dd = pd->port_dd;
2071
2072
2073 ipath_user_sdma_queue_drain(dd, fd->pq);
2074 ipath_user_sdma_queue_destroy(fd->pq);
2075
2076 if (--pd->port_cnt) {
2077
2078
2079
2080
2081
2082 pd->active_slaves &= ~(1 << fd->subport);
2083 put_pid(pd->port_subpid[fd->subport]);
2084 pd->port_subpid[fd->subport] = NULL;
2085 mutex_unlock(&ipath_mutex);
2086 goto bail;
2087 }
2088
2089 spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
2090 port = pd->port_port;
2091 dd->ipath_pd[port] = NULL;
2092 pid = pd->port_pid;
2093 pd->port_pid = NULL;
2094 spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
2095
2096 if (pd->port_rcvwait_to || pd->port_piowait_to
2097 || pd->port_rcvnowait || pd->port_pionowait) {
2098 ipath_cdbg(VERBOSE, "port%u, %u rcv, %u pio wait timeo; "
2099 "%u rcv %u, pio already\n",
2100 pd->port_port, pd->port_rcvwait_to,
2101 pd->port_piowait_to, pd->port_rcvnowait,
2102 pd->port_pionowait);
2103 pd->port_rcvwait_to = pd->port_piowait_to =
2104 pd->port_rcvnowait = pd->port_pionowait = 0;
2105 }
2106 if (pd->port_flag) {
2107 ipath_cdbg(PROC, "port %u port_flag set: 0x%lx\n",
2108 pd->port_port, pd->port_flag);
2109 pd->port_flag = 0;
2110 }
2111
2112 if (dd->ipath_kregbase) {
2113
2114 clear_bit(dd->ipath_r_portenable_shift + port,
2115 &dd->ipath_rcvctrl);
2116 clear_bit(pd->port_port + dd->ipath_r_intravail_shift,
2117 &dd->ipath_rcvctrl);
2118 ipath_write_kreg( dd, dd->ipath_kregs->kr_rcvctrl,
2119 dd->ipath_rcvctrl);
2120
2121
2122 (void)ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
2123
2124
2125 ipath_clean_part_key(pd, dd);
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135 ipath_write_kreg_port(dd,
2136 dd->ipath_kregs->kr_rcvhdrtailaddr, port,
2137 dd->ipath_dummy_hdrq_phys);
2138 ipath_write_kreg_port(dd, dd->ipath_kregs->kr_rcvhdraddr,
2139 pd->port_port, dd->ipath_dummy_hdrq_phys);
2140
2141 ipath_disarm_piobufs(dd, pd->port_pio_base, pd->port_piocnt);
2142 ipath_chg_pioavailkernel(dd, pd->port_pio_base,
2143 pd->port_piocnt, 1);
2144
2145 dd->ipath_f_clear_tids(dd, pd->port_port);
2146
2147 if (dd->ipath_pageshadow)
2148 unlock_expected_tids(pd);
2149 ipath_stats.sps_ports--;
2150 ipath_cdbg(PROC, "%s[%u] closed port %u:%u\n",
2151 pd->port_comm, pid_nr(pid),
2152 dd->ipath_unit, port);
2153 }
2154
2155 put_pid(pid);
2156 mutex_unlock(&ipath_mutex);
2157 ipath_free_pddata(dd, pd);
2158
2159bail:
2160 kfree(fd);
2161 return ret;
2162}
2163
2164static int ipath_port_info(struct ipath_portdata *pd, u16 subport,
2165 struct ipath_port_info __user *uinfo)
2166{
2167 struct ipath_port_info info;
2168 int nup;
2169 int ret;
2170 size_t sz;
2171
2172 (void) ipath_count_units(NULL, &nup, NULL);
2173 info.num_active = nup;
2174 info.unit = pd->port_dd->ipath_unit;
2175 info.port = pd->port_port;
2176 info.subport = subport;
2177
2178 if (ipath_supports_subports(pd->userversion >> 16,
2179 pd->userversion & 0xffff)) {
2180
2181 info.num_ports = pd->port_dd->ipath_cfgports - 1;
2182 info.num_subports = pd->port_subport_cnt;
2183 sz = sizeof(info);
2184 } else
2185 sz = sizeof(info) - 2 * sizeof(u16);
2186
2187 if (copy_to_user(uinfo, &info, sz)) {
2188 ret = -EFAULT;
2189 goto bail;
2190 }
2191 ret = 0;
2192
2193bail:
2194 return ret;
2195}
2196
2197static int ipath_get_slave_info(struct ipath_portdata *pd,
2198 void __user *slave_mask_addr)
2199{
2200 int ret = 0;
2201
2202 if (copy_to_user(slave_mask_addr, &pd->active_slaves, sizeof(u32)))
2203 ret = -EFAULT;
2204 return ret;
2205}
2206
2207static int ipath_sdma_get_inflight(struct ipath_user_sdma_queue *pq,
2208 u32 __user *inflightp)
2209{
2210 const u32 val = ipath_user_sdma_inflight_counter(pq);
2211
2212 if (put_user(val, inflightp))
2213 return -EFAULT;
2214
2215 return 0;
2216}
2217
2218static int ipath_sdma_get_complete(struct ipath_devdata *dd,
2219 struct ipath_user_sdma_queue *pq,
2220 u32 __user *completep)
2221{
2222 u32 val;
2223 int err;
2224
2225 err = ipath_user_sdma_make_progress(dd, pq);
2226 if (err < 0)
2227 return err;
2228
2229 val = ipath_user_sdma_complete_counter(pq);
2230 if (put_user(val, completep))
2231 return -EFAULT;
2232
2233 return 0;
2234}
2235
2236static ssize_t ipath_write(struct file *fp, const char __user *data,
2237 size_t count, loff_t *off)
2238{
2239 const struct ipath_cmd __user *ucmd;
2240 struct ipath_portdata *pd;
2241 const void __user *src;
2242 size_t consumed, copy;
2243 struct ipath_cmd cmd;
2244 ssize_t ret = 0;
2245 void *dest;
2246
2247 if (count < sizeof(cmd.type)) {
2248 ret = -EINVAL;
2249 goto bail;
2250 }
2251
2252 ucmd = (const struct ipath_cmd __user *) data;
2253
2254 if (copy_from_user(&cmd.type, &ucmd->type, sizeof(cmd.type))) {
2255 ret = -EFAULT;
2256 goto bail;
2257 }
2258
2259 consumed = sizeof(cmd.type);
2260
2261 switch (cmd.type) {
2262 case IPATH_CMD_ASSIGN_PORT:
2263 case __IPATH_CMD_USER_INIT:
2264 case IPATH_CMD_USER_INIT:
2265 copy = sizeof(cmd.cmd.user_info);
2266 dest = &cmd.cmd.user_info;
2267 src = &ucmd->cmd.user_info;
2268 break;
2269 case IPATH_CMD_RECV_CTRL:
2270 copy = sizeof(cmd.cmd.recv_ctrl);
2271 dest = &cmd.cmd.recv_ctrl;
2272 src = &ucmd->cmd.recv_ctrl;
2273 break;
2274 case IPATH_CMD_PORT_INFO:
2275 copy = sizeof(cmd.cmd.port_info);
2276 dest = &cmd.cmd.port_info;
2277 src = &ucmd->cmd.port_info;
2278 break;
2279 case IPATH_CMD_TID_UPDATE:
2280 case IPATH_CMD_TID_FREE:
2281 copy = sizeof(cmd.cmd.tid_info);
2282 dest = &cmd.cmd.tid_info;
2283 src = &ucmd->cmd.tid_info;
2284 break;
2285 case IPATH_CMD_SET_PART_KEY:
2286 copy = sizeof(cmd.cmd.part_key);
2287 dest = &cmd.cmd.part_key;
2288 src = &ucmd->cmd.part_key;
2289 break;
2290 case __IPATH_CMD_SLAVE_INFO:
2291 copy = sizeof(cmd.cmd.slave_mask_addr);
2292 dest = &cmd.cmd.slave_mask_addr;
2293 src = &ucmd->cmd.slave_mask_addr;
2294 break;
2295 case IPATH_CMD_PIOAVAILUPD:
2296 copy = 0;
2297 src = NULL;
2298 dest = NULL;
2299 break;
2300 case IPATH_CMD_POLL_TYPE:
2301 copy = sizeof(cmd.cmd.poll_type);
2302 dest = &cmd.cmd.poll_type;
2303 src = &ucmd->cmd.poll_type;
2304 break;
2305 case IPATH_CMD_ARMLAUNCH_CTRL:
2306 copy = sizeof(cmd.cmd.armlaunch_ctrl);
2307 dest = &cmd.cmd.armlaunch_ctrl;
2308 src = &ucmd->cmd.armlaunch_ctrl;
2309 break;
2310 case IPATH_CMD_SDMA_INFLIGHT:
2311 copy = sizeof(cmd.cmd.sdma_inflight);
2312 dest = &cmd.cmd.sdma_inflight;
2313 src = &ucmd->cmd.sdma_inflight;
2314 break;
2315 case IPATH_CMD_SDMA_COMPLETE:
2316 copy = sizeof(cmd.cmd.sdma_complete);
2317 dest = &cmd.cmd.sdma_complete;
2318 src = &ucmd->cmd.sdma_complete;
2319 break;
2320 default:
2321 ret = -EINVAL;
2322 goto bail;
2323 }
2324
2325 if (copy) {
2326 if ((count - consumed) < copy) {
2327 ret = -EINVAL;
2328 goto bail;
2329 }
2330
2331 if (copy_from_user(dest, src, copy)) {
2332 ret = -EFAULT;
2333 goto bail;
2334 }
2335
2336 consumed += copy;
2337 }
2338
2339 pd = port_fp(fp);
2340 if (!pd && cmd.type != __IPATH_CMD_USER_INIT &&
2341 cmd.type != IPATH_CMD_ASSIGN_PORT) {
2342 ret = -EINVAL;
2343 goto bail;
2344 }
2345
2346 switch (cmd.type) {
2347 case IPATH_CMD_ASSIGN_PORT:
2348 ret = ipath_assign_port(fp, &cmd.cmd.user_info);
2349 if (ret)
2350 goto bail;
2351 break;
2352 case __IPATH_CMD_USER_INIT:
2353
2354 ret = ipath_assign_port(fp, &cmd.cmd.user_info);
2355 if (ret)
2356 goto bail;
2357
2358 case IPATH_CMD_USER_INIT:
2359 ret = ipath_do_user_init(fp, &cmd.cmd.user_info);
2360 if (ret)
2361 goto bail;
2362 ret = ipath_get_base_info(
2363 fp, (void __user *) (unsigned long)
2364 cmd.cmd.user_info.spu_base_info,
2365 cmd.cmd.user_info.spu_base_info_size);
2366 break;
2367 case IPATH_CMD_RECV_CTRL:
2368 ret = ipath_manage_rcvq(pd, subport_fp(fp), cmd.cmd.recv_ctrl);
2369 break;
2370 case IPATH_CMD_PORT_INFO:
2371 ret = ipath_port_info(pd, subport_fp(fp),
2372 (struct ipath_port_info __user *)
2373 (unsigned long) cmd.cmd.port_info);
2374 break;
2375 case IPATH_CMD_TID_UPDATE:
2376 ret = ipath_tid_update(pd, fp, &cmd.cmd.tid_info);
2377 break;
2378 case IPATH_CMD_TID_FREE:
2379 ret = ipath_tid_free(pd, subport_fp(fp), &cmd.cmd.tid_info);
2380 break;
2381 case IPATH_CMD_SET_PART_KEY:
2382 ret = ipath_set_part_key(pd, cmd.cmd.part_key);
2383 break;
2384 case __IPATH_CMD_SLAVE_INFO:
2385 ret = ipath_get_slave_info(pd,
2386 (void __user *) (unsigned long)
2387 cmd.cmd.slave_mask_addr);
2388 break;
2389 case IPATH_CMD_PIOAVAILUPD:
2390 ipath_force_pio_avail_update(pd->port_dd);
2391 break;
2392 case IPATH_CMD_POLL_TYPE:
2393 pd->poll_type = cmd.cmd.poll_type;
2394 break;
2395 case IPATH_CMD_ARMLAUNCH_CTRL:
2396 if (cmd.cmd.armlaunch_ctrl)
2397 ipath_enable_armlaunch(pd->port_dd);
2398 else
2399 ipath_disable_armlaunch(pd->port_dd);
2400 break;
2401 case IPATH_CMD_SDMA_INFLIGHT:
2402 ret = ipath_sdma_get_inflight(user_sdma_queue_fp(fp),
2403 (u32 __user *) (unsigned long)
2404 cmd.cmd.sdma_inflight);
2405 break;
2406 case IPATH_CMD_SDMA_COMPLETE:
2407 ret = ipath_sdma_get_complete(pd->port_dd,
2408 user_sdma_queue_fp(fp),
2409 (u32 __user *) (unsigned long)
2410 cmd.cmd.sdma_complete);
2411 break;
2412 }
2413
2414 if (ret >= 0)
2415 ret = consumed;
2416
2417bail:
2418 return ret;
2419}
2420
2421static ssize_t ipath_write_iter(struct kiocb *iocb, struct iov_iter *from)
2422{
2423 struct file *filp = iocb->ki_filp;
2424 struct ipath_filedata *fp = filp->private_data;
2425 struct ipath_portdata *pd = port_fp(filp);
2426 struct ipath_user_sdma_queue *pq = fp->pq;
2427
2428 if (!iter_is_iovec(from) || !from->nr_segs)
2429 return -EINVAL;
2430
2431 return ipath_user_sdma_writev(pd->port_dd, pq, from->iov, from->nr_segs);
2432}
2433
2434static struct class *ipath_class;
2435
2436static int init_cdev(int minor, char *name, const struct file_operations *fops,
2437 struct cdev **cdevp, struct device **devp)
2438{
2439 const dev_t dev = MKDEV(IPATH_MAJOR, minor);
2440 struct cdev *cdev = NULL;
2441 struct device *device = NULL;
2442 int ret;
2443
2444 cdev = cdev_alloc();
2445 if (!cdev) {
2446 printk(KERN_ERR IPATH_DRV_NAME
2447 ": Could not allocate cdev for minor %d, %s\n",
2448 minor, name);
2449 ret = -ENOMEM;
2450 goto done;
2451 }
2452
2453 cdev->owner = THIS_MODULE;
2454 cdev->ops = fops;
2455 kobject_set_name(&cdev->kobj, name);
2456
2457 ret = cdev_add(cdev, dev, 1);
2458 if (ret < 0) {
2459 printk(KERN_ERR IPATH_DRV_NAME
2460 ": Could not add cdev for minor %d, %s (err %d)\n",
2461 minor, name, -ret);
2462 goto err_cdev;
2463 }
2464
2465 device = device_create(ipath_class, NULL, dev, NULL, name);
2466
2467 if (IS_ERR(device)) {
2468 ret = PTR_ERR(device);
2469 printk(KERN_ERR IPATH_DRV_NAME ": Could not create "
2470 "device for minor %d, %s (err %d)\n",
2471 minor, name, -ret);
2472 goto err_cdev;
2473 }
2474
2475 goto done;
2476
2477err_cdev:
2478 cdev_del(cdev);
2479 cdev = NULL;
2480
2481done:
2482 if (ret >= 0) {
2483 *cdevp = cdev;
2484 *devp = device;
2485 } else {
2486 *cdevp = NULL;
2487 *devp = NULL;
2488 }
2489
2490 return ret;
2491}
2492
2493int ipath_cdev_init(int minor, char *name, const struct file_operations *fops,
2494 struct cdev **cdevp, struct device **devp)
2495{
2496 return init_cdev(minor, name, fops, cdevp, devp);
2497}
2498
2499static void cleanup_cdev(struct cdev **cdevp,
2500 struct device **devp)
2501{
2502 struct device *dev = *devp;
2503
2504 if (dev) {
2505 device_unregister(dev);
2506 *devp = NULL;
2507 }
2508
2509 if (*cdevp) {
2510 cdev_del(*cdevp);
2511 *cdevp = NULL;
2512 }
2513}
2514
2515void ipath_cdev_cleanup(struct cdev **cdevp,
2516 struct device **devp)
2517{
2518 cleanup_cdev(cdevp, devp);
2519}
2520
2521static struct cdev *wildcard_cdev;
2522static struct device *wildcard_dev;
2523
2524static const dev_t dev = MKDEV(IPATH_MAJOR, 0);
2525
2526static int user_init(void)
2527{
2528 int ret;
2529
2530 ret = register_chrdev_region(dev, IPATH_NMINORS, IPATH_DRV_NAME);
2531 if (ret < 0) {
2532 printk(KERN_ERR IPATH_DRV_NAME ": Could not register "
2533 "chrdev region (err %d)\n", -ret);
2534 goto done;
2535 }
2536
2537 ipath_class = class_create(THIS_MODULE, IPATH_DRV_NAME);
2538
2539 if (IS_ERR(ipath_class)) {
2540 ret = PTR_ERR(ipath_class);
2541 printk(KERN_ERR IPATH_DRV_NAME ": Could not create "
2542 "device class (err %d)\n", -ret);
2543 goto bail;
2544 }
2545
2546 goto done;
2547bail:
2548 unregister_chrdev_region(dev, IPATH_NMINORS);
2549done:
2550 return ret;
2551}
2552
2553static void user_cleanup(void)
2554{
2555 if (ipath_class) {
2556 class_destroy(ipath_class);
2557 ipath_class = NULL;
2558 }
2559
2560 unregister_chrdev_region(dev, IPATH_NMINORS);
2561}
2562
2563static atomic_t user_count = ATOMIC_INIT(0);
2564static atomic_t user_setup = ATOMIC_INIT(0);
2565
2566int ipath_user_add(struct ipath_devdata *dd)
2567{
2568 char name[10];
2569 int ret;
2570
2571 if (atomic_inc_return(&user_count) == 1) {
2572 ret = user_init();
2573 if (ret < 0) {
2574 ipath_dev_err(dd, "Unable to set up user support: "
2575 "error %d\n", -ret);
2576 goto bail;
2577 }
2578 ret = init_cdev(0, "ipath", &ipath_file_ops, &wildcard_cdev,
2579 &wildcard_dev);
2580 if (ret < 0) {
2581 ipath_dev_err(dd, "Could not create wildcard "
2582 "minor: error %d\n", -ret);
2583 goto bail_user;
2584 }
2585
2586 atomic_set(&user_setup, 1);
2587 }
2588
2589 snprintf(name, sizeof(name), "ipath%d", dd->ipath_unit);
2590
2591 ret = init_cdev(dd->ipath_unit + 1, name, &ipath_file_ops,
2592 &dd->user_cdev, &dd->user_dev);
2593 if (ret < 0)
2594 ipath_dev_err(dd, "Could not create user minor %d, %s\n",
2595 dd->ipath_unit + 1, name);
2596
2597 goto bail;
2598
2599bail_user:
2600 user_cleanup();
2601bail:
2602 return ret;
2603}
2604
2605void ipath_user_remove(struct ipath_devdata *dd)
2606{
2607 cleanup_cdev(&dd->user_cdev, &dd->user_dev);
2608
2609 if (atomic_dec_return(&user_count) == 0) {
2610 if (atomic_read(&user_setup) == 0)
2611 goto bail;
2612
2613 cleanup_cdev(&wildcard_cdev, &wildcard_dev);
2614 user_cleanup();
2615
2616 atomic_set(&user_setup, 0);
2617 }
2618bail:
2619 return;
2620}
2621