1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#include <linux/types.h>
23#include <linux/kernel.h>
24#include <linux/module.h>
25#include <linux/spinlock.h>
26#include <linux/slab.h>
27#include <linux/init.h>
28#include <linux/mm.h>
29#include <linux/string.h>
30#include <linux/pci.h>
31#include <linux/proc_fs.h>
32#include <linux/seq_file.h>
33#include <linux/acpi.h>
34#include <linux/efi.h>
35#include <linux/nodemask.h>
36#include <linux/bitops.h>
37#include <linux/crash_dump.h>
38#include <linux/iommu-helper.h>
39#include <linux/dma-mapping.h>
40#include <linux/prefetch.h>
41
42#include <asm/delay.h>
43#include <asm/io.h>
44#include <asm/page.h>
45#include <asm/dma.h>
46#include <asm/system.h>
47
48#include <asm/acpi-ext.h>
49
50extern int swiotlb_late_init_with_default_size (size_t size);
51
52#define PFX "IOC: "
53
54
55
56
57
58#undef PDIR_SEARCH_TIMING
59
60
61
62
63
64
65
66
67
68
69#define ALLOW_IOV_BYPASS
70
71
72
73
74
75
76
77
78#undef ALLOW_IOV_BYPASS_SG
79
80
81
82
83
84
85
86
87
88
89#undef FULL_VALID_PDIR
90
91#define ENABLE_MARK_CLEAN
92
93
94
95
96
97
98
99#undef DEBUG_SBA_INIT
100#undef DEBUG_SBA_RUN
101#undef DEBUG_SBA_RUN_SG
102#undef DEBUG_SBA_RESOURCE
103#undef ASSERT_PDIR_SANITY
104#undef DEBUG_LARGE_SG_ENTRIES
105#undef DEBUG_BYPASS
106
107#if defined(FULL_VALID_PDIR) && defined(ASSERT_PDIR_SANITY)
108#error FULL_VALID_PDIR and ASSERT_PDIR_SANITY are mutually exclusive
109#endif
110
111#define SBA_INLINE __inline__
112
113
114#ifdef DEBUG_SBA_INIT
115#define DBG_INIT(x...) printk(x)
116#else
117#define DBG_INIT(x...)
118#endif
119
120#ifdef DEBUG_SBA_RUN
121#define DBG_RUN(x...) printk(x)
122#else
123#define DBG_RUN(x...)
124#endif
125
126#ifdef DEBUG_SBA_RUN_SG
127#define DBG_RUN_SG(x...) printk(x)
128#else
129#define DBG_RUN_SG(x...)
130#endif
131
132
133#ifdef DEBUG_SBA_RESOURCE
134#define DBG_RES(x...) printk(x)
135#else
136#define DBG_RES(x...)
137#endif
138
139#ifdef DEBUG_BYPASS
140#define DBG_BYPASS(x...) printk(x)
141#else
142#define DBG_BYPASS(x...)
143#endif
144
145#ifdef ASSERT_PDIR_SANITY
146#define ASSERT(expr) \
147 if(!(expr)) { \
148 printk( "\n" __FILE__ ":%d: Assertion " #expr " failed!\n",__LINE__); \
149 panic(#expr); \
150 }
151#else
152#define ASSERT(expr)
153#endif
154
155
156
157
158
159
160
161
162#define DELAYED_RESOURCE_CNT 64
163
164#define PCI_DEVICE_ID_HP_SX2000_IOC 0x12ec
165
166#define ZX1_IOC_ID ((PCI_DEVICE_ID_HP_ZX1_IOC << 16) | PCI_VENDOR_ID_HP)
167#define ZX2_IOC_ID ((PCI_DEVICE_ID_HP_ZX2_IOC << 16) | PCI_VENDOR_ID_HP)
168#define REO_IOC_ID ((PCI_DEVICE_ID_HP_REO_IOC << 16) | PCI_VENDOR_ID_HP)
169#define SX1000_IOC_ID ((PCI_DEVICE_ID_HP_SX1000_IOC << 16) | PCI_VENDOR_ID_HP)
170#define SX2000_IOC_ID ((PCI_DEVICE_ID_HP_SX2000_IOC << 16) | PCI_VENDOR_ID_HP)
171
172#define ZX1_IOC_OFFSET 0x1000
173
174#define IOC_FUNC_ID 0x000
175#define IOC_FCLASS 0x008
176#define IOC_IBASE 0x300
177#define IOC_IMASK 0x308
178#define IOC_PCOM 0x310
179#define IOC_TCNFG 0x318
180#define IOC_PDIR_BASE 0x320
181
182#define IOC_ROPE0_CFG 0x500
183#define IOC_ROPE_AO 0x10
184
185
186
187#define ZX1_SBA_IOMMU_COOKIE 0x0000badbadc0ffeeUL
188
189
190
191
192
193
194
195
196
197
198
199static unsigned long iovp_size;
200static unsigned long iovp_shift;
201static unsigned long iovp_mask;
202
203struct ioc {
204 void __iomem *ioc_hpa;
205 char *res_map;
206 u64 *pdir_base;
207 unsigned long ibase;
208 unsigned long imask;
209
210 unsigned long *res_hint;
211 unsigned long dma_mask;
212 spinlock_t res_lock;
213
214 unsigned int res_bitshift;
215 unsigned int res_size;
216#ifdef CONFIG_NUMA
217 unsigned int node;
218#endif
219#if DELAYED_RESOURCE_CNT > 0
220 spinlock_t saved_lock;
221
222 int saved_cnt;
223 struct sba_dma_pair {
224 dma_addr_t iova;
225 size_t size;
226 } saved[DELAYED_RESOURCE_CNT];
227#endif
228
229#ifdef PDIR_SEARCH_TIMING
230#define SBA_SEARCH_SAMPLE 0x100
231 unsigned long avg_search[SBA_SEARCH_SAMPLE];
232 unsigned long avg_idx;
233#endif
234
235
236 struct ioc *next;
237 acpi_handle handle;
238 const char *name;
239 unsigned int func_id;
240 unsigned int rev;
241 u32 iov_size;
242 unsigned int pdir_size;
243 struct pci_dev *sac_only_dev;
244};
245
246static struct ioc *ioc_list;
247static int reserve_sba_gart = 1;
248
249static SBA_INLINE void sba_mark_invalid(struct ioc *, dma_addr_t, size_t);
250static SBA_INLINE void sba_free_range(struct ioc *, dma_addr_t, size_t);
251
252#define sba_sg_address(sg) sg_virt((sg))
253
254#ifdef FULL_VALID_PDIR
255static u64 prefetch_spill_page;
256#endif
257
258#ifdef CONFIG_PCI
259# define GET_IOC(dev) (((dev)->bus == &pci_bus_type) \
260 ? ((struct ioc *) PCI_CONTROLLER(to_pci_dev(dev))->iommu) : NULL)
261#else
262# define GET_IOC(dev) NULL
263#endif
264
265
266
267
268
269
270
271
272#define DMA_CHUNK_SIZE (BITS_PER_LONG*iovp_size)
273
274#define ROUNDUP(x,y) ((x + ((y)-1)) & ~((y)-1))
275
276
277
278
279
280
281
282
283#define READ_REG(addr) __raw_readq(addr)
284#define WRITE_REG(val, addr) __raw_writeq(val, addr)
285
286#ifdef DEBUG_SBA_INIT
287
288
289
290
291
292
293
294static void
295sba_dump_tlb(char *hpa)
296{
297 DBG_INIT("IO TLB at 0x%p\n", (void *)hpa);
298 DBG_INIT("IOC_IBASE : %016lx\n", READ_REG(hpa+IOC_IBASE));
299 DBG_INIT("IOC_IMASK : %016lx\n", READ_REG(hpa+IOC_IMASK));
300 DBG_INIT("IOC_TCNFG : %016lx\n", READ_REG(hpa+IOC_TCNFG));
301 DBG_INIT("IOC_PDIR_BASE: %016lx\n", READ_REG(hpa+IOC_PDIR_BASE));
302 DBG_INIT("\n");
303}
304#endif
305
306
307#ifdef ASSERT_PDIR_SANITY
308
309
310
311
312
313
314
315
316
317static void
318sba_dump_pdir_entry(struct ioc *ioc, char *msg, uint pide)
319{
320
321 u64 *ptr = &ioc->pdir_base[pide & ~(BITS_PER_LONG - 1)];
322 unsigned long *rptr = (unsigned long *) &ioc->res_map[(pide >>3) & -sizeof(unsigned long)];
323 uint rcnt;
324
325 printk(KERN_DEBUG "SBA: %s rp %p bit %d rval 0x%lx\n",
326 msg, rptr, pide & (BITS_PER_LONG - 1), *rptr);
327
328 rcnt = 0;
329 while (rcnt < BITS_PER_LONG) {
330 printk(KERN_DEBUG "%s %2d %p %016Lx\n",
331 (rcnt == (pide & (BITS_PER_LONG - 1)))
332 ? " -->" : " ",
333 rcnt, ptr, (unsigned long long) *ptr );
334 rcnt++;
335 ptr++;
336 }
337 printk(KERN_DEBUG "%s", msg);
338}
339
340
341
342
343
344
345
346
347
348static int
349sba_check_pdir(struct ioc *ioc, char *msg)
350{
351 u64 *rptr_end = (u64 *) &(ioc->res_map[ioc->res_size]);
352 u64 *rptr = (u64 *) ioc->res_map;
353 u64 *pptr = ioc->pdir_base;
354 uint pide = 0;
355
356 while (rptr < rptr_end) {
357 u64 rval;
358 int rcnt;
359
360 rval = *rptr;
361 rcnt = 64;
362
363 while (rcnt) {
364
365 u32 pde = ((u32)((*pptr >> (63)) & 0x1));
366 if ((rval & 0x1) ^ pde)
367 {
368
369
370
371
372 sba_dump_pdir_entry(ioc, msg, pide);
373 return(1);
374 }
375 rcnt--;
376 rval >>= 1;
377 pptr++;
378 pide++;
379 }
380 rptr++;
381 }
382
383 return 0;
384}
385
386
387
388
389
390
391
392
393
394
395static void
396sba_dump_sg( struct ioc *ioc, struct scatterlist *startsg, int nents)
397{
398 while (nents-- > 0) {
399 printk(KERN_DEBUG " %d : DMA %08lx/%05x CPU %p\n", nents,
400 startsg->dma_address, startsg->dma_length,
401 sba_sg_address(startsg));
402 startsg = sg_next(startsg);
403 }
404}
405
406static void
407sba_check_sg( struct ioc *ioc, struct scatterlist *startsg, int nents)
408{
409 struct scatterlist *the_sg = startsg;
410 int the_nents = nents;
411
412 while (the_nents-- > 0) {
413 if (sba_sg_address(the_sg) == 0x0UL)
414 sba_dump_sg(NULL, startsg, nents);
415 the_sg = sg_next(the_sg);
416 }
417}
418
419#endif
420
421
422
423
424
425
426
427
428
429
430
431
432
433#define PAGES_PER_RANGE 1
434
435
436#define SBA_IOVA(ioc,iovp,offset) ((ioc->ibase) | (iovp) | (offset))
437#define SBA_IOVP(ioc,iova) ((iova) & ~(ioc->ibase))
438
439#define PDIR_ENTRY_SIZE sizeof(u64)
440
441#define PDIR_INDEX(iovp) ((iovp)>>iovp_shift)
442
443#define RESMAP_MASK(n) ~(~0UL << (n))
444#define RESMAP_IDX_MASK (sizeof(unsigned long) - 1)
445
446
447
448
449
450
451
452
453static SBA_INLINE int
454get_iovp_order (unsigned long size)
455{
456 long double d = size - 1;
457 long order;
458
459 order = ia64_getf_exp(d);
460 order = order - iovp_shift - 0xffff + 1;
461 if (order < 0)
462 order = 0;
463 return order;
464}
465
466static unsigned long ptr_to_pide(struct ioc *ioc, unsigned long *res_ptr,
467 unsigned int bitshiftcnt)
468{
469 return (((unsigned long)res_ptr - (unsigned long)ioc->res_map) << 3)
470 + bitshiftcnt;
471}
472
473
474
475
476
477
478
479
480
481
482
483static SBA_INLINE unsigned long
484sba_search_bitmap(struct ioc *ioc, struct device *dev,
485 unsigned long bits_wanted, int use_hint)
486{
487 unsigned long *res_ptr;
488 unsigned long *res_end = (unsigned long *) &(ioc->res_map[ioc->res_size]);
489 unsigned long flags, pide = ~0UL, tpide;
490 unsigned long boundary_size;
491 unsigned long shift;
492 int ret;
493
494 ASSERT(((unsigned long) ioc->res_hint & (sizeof(unsigned long) - 1UL)) == 0);
495 ASSERT(res_ptr < res_end);
496
497 boundary_size = (unsigned long long)dma_get_seg_boundary(dev) + 1;
498 boundary_size = ALIGN(boundary_size, 1ULL << iovp_shift) >> iovp_shift;
499
500 BUG_ON(ioc->ibase & ~iovp_mask);
501 shift = ioc->ibase >> iovp_shift;
502
503 spin_lock_irqsave(&ioc->res_lock, flags);
504
505
506 if (likely(use_hint)) {
507 res_ptr = ioc->res_hint;
508 } else {
509 res_ptr = (ulong *)ioc->res_map;
510 ioc->res_bitshift = 0;
511 }
512
513
514
515
516
517
518
519 bits_wanted = 1UL << get_iovp_order(bits_wanted << iovp_shift);
520
521 if (likely(bits_wanted == 1)) {
522 unsigned int bitshiftcnt;
523 for(; res_ptr < res_end ; res_ptr++) {
524 if (likely(*res_ptr != ~0UL)) {
525 bitshiftcnt = ffz(*res_ptr);
526 *res_ptr |= (1UL << bitshiftcnt);
527 pide = ptr_to_pide(ioc, res_ptr, bitshiftcnt);
528 ioc->res_bitshift = bitshiftcnt + bits_wanted;
529 goto found_it;
530 }
531 }
532 goto not_found;
533
534 }
535
536 if (likely(bits_wanted <= BITS_PER_LONG/2)) {
537
538
539
540
541
542
543 unsigned long o = 1 << get_iovp_order(bits_wanted << iovp_shift);
544 uint bitshiftcnt = ROUNDUP(ioc->res_bitshift, o);
545 unsigned long mask, base_mask;
546
547 base_mask = RESMAP_MASK(bits_wanted);
548 mask = base_mask << bitshiftcnt;
549
550 DBG_RES("%s() o %ld %p", __func__, o, res_ptr);
551 for(; res_ptr < res_end ; res_ptr++)
552 {
553 DBG_RES(" %p %lx %lx\n", res_ptr, mask, *res_ptr);
554 ASSERT(0 != mask);
555 for (; mask ; mask <<= o, bitshiftcnt += o) {
556 tpide = ptr_to_pide(ioc, res_ptr, bitshiftcnt);
557 ret = iommu_is_span_boundary(tpide, bits_wanted,
558 shift,
559 boundary_size);
560 if ((0 == ((*res_ptr) & mask)) && !ret) {
561 *res_ptr |= mask;
562 pide = tpide;
563 ioc->res_bitshift = bitshiftcnt + bits_wanted;
564 goto found_it;
565 }
566 }
567
568 bitshiftcnt = 0;
569 mask = base_mask;
570
571 }
572
573 } else {
574 int qwords, bits, i;
575 unsigned long *end;
576
577 qwords = bits_wanted >> 6;
578 bits = bits_wanted - (qwords * BITS_PER_LONG);
579
580 end = res_end - qwords;
581
582 for (; res_ptr < end; res_ptr++) {
583 tpide = ptr_to_pide(ioc, res_ptr, 0);
584 ret = iommu_is_span_boundary(tpide, bits_wanted,
585 shift, boundary_size);
586 if (ret)
587 goto next_ptr;
588 for (i = 0 ; i < qwords ; i++) {
589 if (res_ptr[i] != 0)
590 goto next_ptr;
591 }
592 if (bits && res_ptr[i] && (__ffs(res_ptr[i]) < bits))
593 continue;
594
595
596 for (i = 0 ; i < qwords ; i++)
597 res_ptr[i] = ~0UL;
598 res_ptr[i] |= RESMAP_MASK(bits);
599
600 pide = tpide;
601 res_ptr += qwords;
602 ioc->res_bitshift = bits;
603 goto found_it;
604next_ptr:
605 ;
606 }
607 }
608
609not_found:
610 prefetch(ioc->res_map);
611 ioc->res_hint = (unsigned long *) ioc->res_map;
612 ioc->res_bitshift = 0;
613 spin_unlock_irqrestore(&ioc->res_lock, flags);
614 return (pide);
615
616found_it:
617 ioc->res_hint = res_ptr;
618 spin_unlock_irqrestore(&ioc->res_lock, flags);
619 return (pide);
620}
621
622
623
624
625
626
627
628
629
630
631static int
632sba_alloc_range(struct ioc *ioc, struct device *dev, size_t size)
633{
634 unsigned int pages_needed = size >> iovp_shift;
635#ifdef PDIR_SEARCH_TIMING
636 unsigned long itc_start;
637#endif
638 unsigned long pide;
639
640 ASSERT(pages_needed);
641 ASSERT(0 == (size & ~iovp_mask));
642
643#ifdef PDIR_SEARCH_TIMING
644 itc_start = ia64_get_itc();
645#endif
646
647
648
649 pide = sba_search_bitmap(ioc, dev, pages_needed, 1);
650 if (unlikely(pide >= (ioc->res_size << 3))) {
651 pide = sba_search_bitmap(ioc, dev, pages_needed, 0);
652 if (unlikely(pide >= (ioc->res_size << 3))) {
653#if DELAYED_RESOURCE_CNT > 0
654 unsigned long flags;
655
656
657
658
659
660
661 spin_lock_irqsave(&ioc->saved_lock, flags);
662 if (ioc->saved_cnt > 0) {
663 struct sba_dma_pair *d;
664 int cnt = ioc->saved_cnt;
665
666 d = &(ioc->saved[ioc->saved_cnt - 1]);
667
668 spin_lock(&ioc->res_lock);
669 while (cnt--) {
670 sba_mark_invalid(ioc, d->iova, d->size);
671 sba_free_range(ioc, d->iova, d->size);
672 d--;
673 }
674 ioc->saved_cnt = 0;
675 READ_REG(ioc->ioc_hpa+IOC_PCOM);
676 spin_unlock(&ioc->res_lock);
677 }
678 spin_unlock_irqrestore(&ioc->saved_lock, flags);
679
680 pide = sba_search_bitmap(ioc, dev, pages_needed, 0);
681 if (unlikely(pide >= (ioc->res_size << 3))) {
682 printk(KERN_WARNING "%s: I/O MMU @ %p is"
683 "out of mapping resources, %u %u %lx\n",
684 __func__, ioc->ioc_hpa, ioc->res_size,
685 pages_needed, dma_get_seg_boundary(dev));
686 return -1;
687 }
688#else
689 printk(KERN_WARNING "%s: I/O MMU @ %p is"
690 "out of mapping resources, %u %u %lx\n",
691 __func__, ioc->ioc_hpa, ioc->res_size,
692 pages_needed, dma_get_seg_boundary(dev));
693 return -1;
694#endif
695 }
696 }
697
698#ifdef PDIR_SEARCH_TIMING
699 ioc->avg_search[ioc->avg_idx++] = (ia64_get_itc() - itc_start) / pages_needed;
700 ioc->avg_idx &= SBA_SEARCH_SAMPLE - 1;
701#endif
702
703 prefetchw(&(ioc->pdir_base[pide]));
704
705#ifdef ASSERT_PDIR_SANITY
706
707 if(0x00 != ((u8 *) ioc->pdir_base)[pide*PDIR_ENTRY_SIZE + 7]) {
708 sba_dump_pdir_entry(ioc, "sba_search_bitmap() botched it?", pide);
709 }
710#endif
711
712 DBG_RES("%s(%x) %d -> %lx hint %x/%x\n",
713 __func__, size, pages_needed, pide,
714 (uint) ((unsigned long) ioc->res_hint - (unsigned long) ioc->res_map),
715 ioc->res_bitshift );
716
717 return (pide);
718}
719
720
721
722
723
724
725
726
727
728
729static SBA_INLINE void
730sba_free_range(struct ioc *ioc, dma_addr_t iova, size_t size)
731{
732 unsigned long iovp = SBA_IOVP(ioc, iova);
733 unsigned int pide = PDIR_INDEX(iovp);
734 unsigned int ridx = pide >> 3;
735 unsigned long *res_ptr = (unsigned long *) &((ioc)->res_map[ridx & ~RESMAP_IDX_MASK]);
736 int bits_not_wanted = size >> iovp_shift;
737 unsigned long m;
738
739
740 bits_not_wanted = 1UL << get_iovp_order(bits_not_wanted << iovp_shift);
741 for (; bits_not_wanted > 0 ; res_ptr++) {
742
743 if (unlikely(bits_not_wanted > BITS_PER_LONG)) {
744
745
746 *res_ptr = 0UL;
747 bits_not_wanted -= BITS_PER_LONG;
748 pide += BITS_PER_LONG;
749
750 } else {
751
752
753 m = RESMAP_MASK(bits_not_wanted) << (pide & (BITS_PER_LONG - 1));
754 bits_not_wanted = 0;
755
756 DBG_RES("%s( ,%x,%x) %x/%lx %x %p %lx\n", __func__, (uint) iova, size,
757 bits_not_wanted, m, pide, res_ptr, *res_ptr);
758
759 ASSERT(m != 0);
760 ASSERT(bits_not_wanted);
761 ASSERT((*res_ptr & m) == m);
762 *res_ptr &= ~m;
763 }
764 }
765}
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799#if 1
800#define sba_io_pdir_entry(pdir_ptr, vba) *pdir_ptr = ((vba & ~0xE000000000000FFFULL) \
801 | 0x8000000000000000ULL)
802#else
803void SBA_INLINE
804sba_io_pdir_entry(u64 *pdir_ptr, unsigned long vba)
805{
806 *pdir_ptr = ((vba & ~0xE000000000000FFFULL) | 0x80000000000000FFULL);
807}
808#endif
809
810#ifdef ENABLE_MARK_CLEAN
811
812
813
814
815
816static void
817mark_clean (void *addr, size_t size)
818{
819 unsigned long pg_addr, end;
820
821 pg_addr = PAGE_ALIGN((unsigned long) addr);
822 end = (unsigned long) addr + size;
823 while (pg_addr + PAGE_SIZE <= end) {
824 struct page *page = virt_to_page((void *)pg_addr);
825 set_bit(PG_arch_1, &page->flags);
826 pg_addr += PAGE_SIZE;
827 }
828}
829#endif
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847static SBA_INLINE void
848sba_mark_invalid(struct ioc *ioc, dma_addr_t iova, size_t byte_cnt)
849{
850 u32 iovp = (u32) SBA_IOVP(ioc,iova);
851
852 int off = PDIR_INDEX(iovp);
853
854
855 ASSERT(byte_cnt > 0);
856 ASSERT(0 == (byte_cnt & ~iovp_mask));
857
858#ifdef ASSERT_PDIR_SANITY
859
860 if (!(ioc->pdir_base[off] >> 60)) {
861 sba_dump_pdir_entry(ioc,"sba_mark_invalid()", PDIR_INDEX(iovp));
862 }
863#endif
864
865 if (byte_cnt <= iovp_size)
866 {
867 ASSERT(off < ioc->pdir_size);
868
869 iovp |= iovp_shift;
870
871#ifndef FULL_VALID_PDIR
872
873
874
875
876
877
878 ioc->pdir_base[off] &= ~(0x80000000000000FFULL);
879#else
880
881
882
883
884
885 ioc->pdir_base[off] = (0x80000000000000FFULL | prefetch_spill_page);
886#endif
887 } else {
888 u32 t = get_iovp_order(byte_cnt) + iovp_shift;
889
890 iovp |= t;
891 ASSERT(t <= 31);
892
893 do {
894
895 ASSERT(ioc->pdir_base[off] >> 63);
896#ifndef FULL_VALID_PDIR
897
898 ioc->pdir_base[off] &= ~(0x80000000000000FFULL);
899#else
900 ioc->pdir_base[off] = (0x80000000000000FFULL | prefetch_spill_page);
901#endif
902 off++;
903 byte_cnt -= iovp_size;
904 } while (byte_cnt > 0);
905 }
906
907 WRITE_REG(iovp | ioc->ibase, ioc->ioc_hpa+IOC_PCOM);
908}
909
910
911
912
913
914
915
916
917
918
919
920static dma_addr_t sba_map_page(struct device *dev, struct page *page,
921 unsigned long poff, size_t size,
922 enum dma_data_direction dir,
923 struct dma_attrs *attrs)
924{
925 struct ioc *ioc;
926 void *addr = page_address(page) + poff;
927 dma_addr_t iovp;
928 dma_addr_t offset;
929 u64 *pdir_start;
930 int pide;
931#ifdef ASSERT_PDIR_SANITY
932 unsigned long flags;
933#endif
934#ifdef ALLOW_IOV_BYPASS
935 unsigned long pci_addr = virt_to_phys(addr);
936#endif
937
938#ifdef ALLOW_IOV_BYPASS
939 ASSERT(to_pci_dev(dev)->dma_mask);
940
941
942
943 if (likely((pci_addr & ~to_pci_dev(dev)->dma_mask) == 0)) {
944
945
946
947
948 DBG_BYPASS("sba_map_single_attrs() bypass mask/addr: "
949 "0x%lx/0x%lx\n",
950 to_pci_dev(dev)->dma_mask, pci_addr);
951 return pci_addr;
952 }
953#endif
954 ioc = GET_IOC(dev);
955 ASSERT(ioc);
956
957 prefetch(ioc->res_hint);
958
959 ASSERT(size > 0);
960 ASSERT(size <= DMA_CHUNK_SIZE);
961
962
963 offset = ((dma_addr_t) (long) addr) & ~iovp_mask;
964
965
966 size = (size + offset + ~iovp_mask) & iovp_mask;
967
968#ifdef ASSERT_PDIR_SANITY
969 spin_lock_irqsave(&ioc->res_lock, flags);
970 if (sba_check_pdir(ioc,"Check before sba_map_single_attrs()"))
971 panic("Sanity check failed");
972 spin_unlock_irqrestore(&ioc->res_lock, flags);
973#endif
974
975 pide = sba_alloc_range(ioc, dev, size);
976 if (pide < 0)
977 return 0;
978
979 iovp = (dma_addr_t) pide << iovp_shift;
980
981 DBG_RUN("%s() 0x%p -> 0x%lx\n", __func__, addr, (long) iovp | offset);
982
983 pdir_start = &(ioc->pdir_base[pide]);
984
985 while (size > 0) {
986 ASSERT(((u8 *)pdir_start)[7] == 0);
987 sba_io_pdir_entry(pdir_start, (unsigned long) addr);
988
989 DBG_RUN(" pdir 0x%p %lx\n", pdir_start, *pdir_start);
990
991 addr += iovp_size;
992 size -= iovp_size;
993 pdir_start++;
994 }
995
996 wmb();
997
998
999#ifdef ASSERT_PDIR_SANITY
1000 spin_lock_irqsave(&ioc->res_lock, flags);
1001 sba_check_pdir(ioc,"Check after sba_map_single_attrs()");
1002 spin_unlock_irqrestore(&ioc->res_lock, flags);
1003#endif
1004 return SBA_IOVA(ioc, iovp, offset);
1005}
1006
1007static dma_addr_t sba_map_single_attrs(struct device *dev, void *addr,
1008 size_t size, enum dma_data_direction dir,
1009 struct dma_attrs *attrs)
1010{
1011 return sba_map_page(dev, virt_to_page(addr),
1012 (unsigned long)addr & ~PAGE_MASK, size, dir, attrs);
1013}
1014
1015#ifdef ENABLE_MARK_CLEAN
1016static SBA_INLINE void
1017sba_mark_clean(struct ioc *ioc, dma_addr_t iova, size_t size)
1018{
1019 u32 iovp = (u32) SBA_IOVP(ioc,iova);
1020 int off = PDIR_INDEX(iovp);
1021 void *addr;
1022
1023 if (size <= iovp_size) {
1024 addr = phys_to_virt(ioc->pdir_base[off] &
1025 ~0xE000000000000FFFULL);
1026 mark_clean(addr, size);
1027 } else {
1028 do {
1029 addr = phys_to_virt(ioc->pdir_base[off] &
1030 ~0xE000000000000FFFULL);
1031 mark_clean(addr, min(size, iovp_size));
1032 off++;
1033 size -= iovp_size;
1034 } while (size > 0);
1035 }
1036}
1037#endif
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049static void sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size,
1050 enum dma_data_direction dir, struct dma_attrs *attrs)
1051{
1052 struct ioc *ioc;
1053#if DELAYED_RESOURCE_CNT > 0
1054 struct sba_dma_pair *d;
1055#endif
1056 unsigned long flags;
1057 dma_addr_t offset;
1058
1059 ioc = GET_IOC(dev);
1060 ASSERT(ioc);
1061
1062#ifdef ALLOW_IOV_BYPASS
1063 if (likely((iova & ioc->imask) != ioc->ibase)) {
1064
1065
1066
1067 DBG_BYPASS("sba_unmap_single_attrs() bypass addr: 0x%lx\n",
1068 iova);
1069
1070#ifdef ENABLE_MARK_CLEAN
1071 if (dir == DMA_FROM_DEVICE) {
1072 mark_clean(phys_to_virt(iova), size);
1073 }
1074#endif
1075 return;
1076 }
1077#endif
1078 offset = iova & ~iovp_mask;
1079
1080 DBG_RUN("%s() iovp 0x%lx/%x\n", __func__, (long) iova, size);
1081
1082 iova ^= offset;
1083 size += offset;
1084 size = ROUNDUP(size, iovp_size);
1085
1086#ifdef ENABLE_MARK_CLEAN
1087 if (dir == DMA_FROM_DEVICE)
1088 sba_mark_clean(ioc, iova, size);
1089#endif
1090
1091#if DELAYED_RESOURCE_CNT > 0
1092 spin_lock_irqsave(&ioc->saved_lock, flags);
1093 d = &(ioc->saved[ioc->saved_cnt]);
1094 d->iova = iova;
1095 d->size = size;
1096 if (unlikely(++(ioc->saved_cnt) >= DELAYED_RESOURCE_CNT)) {
1097 int cnt = ioc->saved_cnt;
1098 spin_lock(&ioc->res_lock);
1099 while (cnt--) {
1100 sba_mark_invalid(ioc, d->iova, d->size);
1101 sba_free_range(ioc, d->iova, d->size);
1102 d--;
1103 }
1104 ioc->saved_cnt = 0;
1105 READ_REG(ioc->ioc_hpa+IOC_PCOM);
1106 spin_unlock(&ioc->res_lock);
1107 }
1108 spin_unlock_irqrestore(&ioc->saved_lock, flags);
1109#else
1110 spin_lock_irqsave(&ioc->res_lock, flags);
1111 sba_mark_invalid(ioc, iova, size);
1112 sba_free_range(ioc, iova, size);
1113 READ_REG(ioc->ioc_hpa+IOC_PCOM);
1114 spin_unlock_irqrestore(&ioc->res_lock, flags);
1115#endif
1116}
1117
1118void sba_unmap_single_attrs(struct device *dev, dma_addr_t iova, size_t size,
1119 enum dma_data_direction dir, struct dma_attrs *attrs)
1120{
1121 sba_unmap_page(dev, iova, size, dir, attrs);
1122}
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132static void *
1133sba_alloc_coherent (struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags)
1134{
1135 struct ioc *ioc;
1136 void *addr;
1137
1138 ioc = GET_IOC(dev);
1139 ASSERT(ioc);
1140
1141#ifdef CONFIG_NUMA
1142 {
1143 struct page *page;
1144 page = alloc_pages_exact_node(ioc->node == MAX_NUMNODES ?
1145 numa_node_id() : ioc->node, flags,
1146 get_order(size));
1147
1148 if (unlikely(!page))
1149 return NULL;
1150
1151 addr = page_address(page);
1152 }
1153#else
1154 addr = (void *) __get_free_pages(flags, get_order(size));
1155#endif
1156 if (unlikely(!addr))
1157 return NULL;
1158
1159 memset(addr, 0, size);
1160 *dma_handle = virt_to_phys(addr);
1161
1162#ifdef ALLOW_IOV_BYPASS
1163 ASSERT(dev->coherent_dma_mask);
1164
1165
1166
1167 if (likely((*dma_handle & ~dev->coherent_dma_mask) == 0)) {
1168 DBG_BYPASS("sba_alloc_coherent() bypass mask/addr: 0x%lx/0x%lx\n",
1169 dev->coherent_dma_mask, *dma_handle);
1170
1171 return addr;
1172 }
1173#endif
1174
1175
1176
1177
1178
1179 *dma_handle = sba_map_single_attrs(&ioc->sac_only_dev->dev, addr,
1180 size, 0, NULL);
1181
1182 return addr;
1183}
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195static void sba_free_coherent (struct device *dev, size_t size, void *vaddr,
1196 dma_addr_t dma_handle)
1197{
1198 sba_unmap_single_attrs(dev, dma_handle, size, 0, NULL);
1199 free_pages((unsigned long) vaddr, get_order(size));
1200}
1201
1202
1203
1204
1205
1206
1207
1208#define PIDE_FLAG 0x1UL
1209
1210#ifdef DEBUG_LARGE_SG_ENTRIES
1211int dump_run_sg = 0;
1212#endif
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225static SBA_INLINE int
1226sba_fill_pdir(
1227 struct ioc *ioc,
1228 struct scatterlist *startsg,
1229 int nents)
1230{
1231 struct scatterlist *dma_sg = startsg;
1232 int n_mappings = 0;
1233 u64 *pdirp = NULL;
1234 unsigned long dma_offset = 0;
1235
1236 while (nents-- > 0) {
1237 int cnt = startsg->dma_length;
1238 startsg->dma_length = 0;
1239
1240#ifdef DEBUG_LARGE_SG_ENTRIES
1241 if (dump_run_sg)
1242 printk(" %2d : %08lx/%05x %p\n",
1243 nents, startsg->dma_address, cnt,
1244 sba_sg_address(startsg));
1245#else
1246 DBG_RUN_SG(" %d : %08lx/%05x %p\n",
1247 nents, startsg->dma_address, cnt,
1248 sba_sg_address(startsg));
1249#endif
1250
1251
1252
1253 if (startsg->dma_address & PIDE_FLAG) {
1254 u32 pide = startsg->dma_address & ~PIDE_FLAG;
1255 dma_offset = (unsigned long) pide & ~iovp_mask;
1256 startsg->dma_address = 0;
1257 if (n_mappings)
1258 dma_sg = sg_next(dma_sg);
1259 dma_sg->dma_address = pide | ioc->ibase;
1260 pdirp = &(ioc->pdir_base[pide >> iovp_shift]);
1261 n_mappings++;
1262 }
1263
1264
1265
1266
1267 if (cnt) {
1268 unsigned long vaddr = (unsigned long) sba_sg_address(startsg);
1269 ASSERT(pdirp);
1270
1271
1272
1273
1274 dma_sg->dma_length += cnt;
1275 cnt += dma_offset;
1276 dma_offset=0;
1277 cnt = ROUNDUP(cnt, iovp_size);
1278 do {
1279 sba_io_pdir_entry(pdirp, vaddr);
1280 vaddr += iovp_size;
1281 cnt -= iovp_size;
1282 pdirp++;
1283 } while (cnt > 0);
1284 }
1285 startsg = sg_next(startsg);
1286 }
1287
1288 wmb();
1289
1290#ifdef DEBUG_LARGE_SG_ENTRIES
1291 dump_run_sg = 0;
1292#endif
1293 return(n_mappings);
1294}
1295
1296
1297
1298
1299
1300
1301
1302
1303#define DMA_CONTIG(__X, __Y) \
1304 (((((unsigned long) __X) | ((unsigned long) __Y)) << (BITS_PER_LONG - iovp_shift)) == 0UL)
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321static SBA_INLINE int
1322sba_coalesce_chunks(struct ioc *ioc, struct device *dev,
1323 struct scatterlist *startsg,
1324 int nents)
1325{
1326 struct scatterlist *vcontig_sg;
1327 unsigned long vcontig_len;
1328 unsigned long vcontig_end;
1329 struct scatterlist *dma_sg;
1330 unsigned long dma_offset, dma_len;
1331 int n_mappings = 0;
1332 unsigned int max_seg_size = dma_get_max_seg_size(dev);
1333 int idx;
1334
1335 while (nents > 0) {
1336 unsigned long vaddr = (unsigned long) sba_sg_address(startsg);
1337
1338
1339
1340
1341 dma_sg = vcontig_sg = startsg;
1342 dma_len = vcontig_len = vcontig_end = startsg->length;
1343 vcontig_end += vaddr;
1344 dma_offset = vaddr & ~iovp_mask;
1345
1346
1347 startsg->dma_address = startsg->dma_length = 0;
1348
1349
1350
1351
1352
1353 while (--nents > 0) {
1354 unsigned long vaddr;
1355
1356 startsg = sg_next(startsg);
1357
1358
1359 startsg->dma_address = startsg->dma_length = 0;
1360
1361
1362 ASSERT(startsg->length <= DMA_CHUNK_SIZE);
1363
1364
1365
1366
1367
1368
1369 if (((dma_len + dma_offset + startsg->length + ~iovp_mask) & iovp_mask)
1370 > DMA_CHUNK_SIZE)
1371 break;
1372
1373 if (dma_len + startsg->length > max_seg_size)
1374 break;
1375
1376
1377
1378
1379
1380
1381 vaddr = (unsigned long) sba_sg_address(startsg);
1382 if (vcontig_end == vaddr)
1383 {
1384 vcontig_len += startsg->length;
1385 vcontig_end += startsg->length;
1386 dma_len += startsg->length;
1387 continue;
1388 }
1389
1390#ifdef DEBUG_LARGE_SG_ENTRIES
1391 dump_run_sg = (vcontig_len > iovp_size);
1392#endif
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405 vcontig_sg->dma_length = vcontig_len;
1406
1407 vcontig_sg = startsg;
1408 vcontig_len = startsg->length;
1409
1410
1411
1412
1413
1414 if (DMA_CONTIG(vcontig_end, vaddr))
1415 {
1416 vcontig_end = vcontig_len + vaddr;
1417 dma_len += vcontig_len;
1418 continue;
1419 } else {
1420 break;
1421 }
1422 }
1423
1424
1425
1426
1427
1428
1429 vcontig_sg->dma_length = vcontig_len;
1430 dma_len = (dma_len + dma_offset + ~iovp_mask) & iovp_mask;
1431 ASSERT(dma_len <= DMA_CHUNK_SIZE);
1432 idx = sba_alloc_range(ioc, dev, dma_len);
1433 if (idx < 0) {
1434 dma_sg->dma_length = 0;
1435 return -1;
1436 }
1437 dma_sg->dma_address = (dma_addr_t)(PIDE_FLAG | (idx << iovp_shift)
1438 | dma_offset);
1439 n_mappings++;
1440 }
1441
1442 return n_mappings;
1443}
1444
1445static void sba_unmap_sg_attrs(struct device *dev, struct scatterlist *sglist,
1446 int nents, enum dma_data_direction dir,
1447 struct dma_attrs *attrs);
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458static int sba_map_sg_attrs(struct device *dev, struct scatterlist *sglist,
1459 int nents, enum dma_data_direction dir,
1460 struct dma_attrs *attrs)
1461{
1462 struct ioc *ioc;
1463 int coalesced, filled = 0;
1464#ifdef ASSERT_PDIR_SANITY
1465 unsigned long flags;
1466#endif
1467#ifdef ALLOW_IOV_BYPASS_SG
1468 struct scatterlist *sg;
1469#endif
1470
1471 DBG_RUN_SG("%s() START %d entries\n", __func__, nents);
1472 ioc = GET_IOC(dev);
1473 ASSERT(ioc);
1474
1475#ifdef ALLOW_IOV_BYPASS_SG
1476 ASSERT(to_pci_dev(dev)->dma_mask);
1477 if (likely((ioc->dma_mask & ~to_pci_dev(dev)->dma_mask) == 0)) {
1478 for_each_sg(sglist, sg, nents, filled) {
1479 sg->dma_length = sg->length;
1480 sg->dma_address = virt_to_phys(sba_sg_address(sg));
1481 }
1482 return filled;
1483 }
1484#endif
1485
1486 if (nents == 1) {
1487 sglist->dma_length = sglist->length;
1488 sglist->dma_address = sba_map_single_attrs(dev, sba_sg_address(sglist), sglist->length, dir, attrs);
1489 return 1;
1490 }
1491
1492#ifdef ASSERT_PDIR_SANITY
1493 spin_lock_irqsave(&ioc->res_lock, flags);
1494 if (sba_check_pdir(ioc,"Check before sba_map_sg_attrs()"))
1495 {
1496 sba_dump_sg(ioc, sglist, nents);
1497 panic("Check before sba_map_sg_attrs()");
1498 }
1499 spin_unlock_irqrestore(&ioc->res_lock, flags);
1500#endif
1501
1502 prefetch(ioc->res_hint);
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512 coalesced = sba_coalesce_chunks(ioc, dev, sglist, nents);
1513 if (coalesced < 0) {
1514 sba_unmap_sg_attrs(dev, sglist, nents, dir, attrs);
1515 return 0;
1516 }
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526 filled = sba_fill_pdir(ioc, sglist, nents);
1527
1528#ifdef ASSERT_PDIR_SANITY
1529 spin_lock_irqsave(&ioc->res_lock, flags);
1530 if (sba_check_pdir(ioc,"Check after sba_map_sg_attrs()"))
1531 {
1532 sba_dump_sg(ioc, sglist, nents);
1533 panic("Check after sba_map_sg_attrs()\n");
1534 }
1535 spin_unlock_irqrestore(&ioc->res_lock, flags);
1536#endif
1537
1538 ASSERT(coalesced == filled);
1539 DBG_RUN_SG("%s() DONE %d mappings\n", __func__, filled);
1540
1541 return filled;
1542}
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554static void sba_unmap_sg_attrs(struct device *dev, struct scatterlist *sglist,
1555 int nents, enum dma_data_direction dir,
1556 struct dma_attrs *attrs)
1557{
1558#ifdef ASSERT_PDIR_SANITY
1559 struct ioc *ioc;
1560 unsigned long flags;
1561#endif
1562
1563 DBG_RUN_SG("%s() START %d entries, %p,%x\n",
1564 __func__, nents, sba_sg_address(sglist), sglist->length);
1565
1566#ifdef ASSERT_PDIR_SANITY
1567 ioc = GET_IOC(dev);
1568 ASSERT(ioc);
1569
1570 spin_lock_irqsave(&ioc->res_lock, flags);
1571 sba_check_pdir(ioc,"Check before sba_unmap_sg_attrs()");
1572 spin_unlock_irqrestore(&ioc->res_lock, flags);
1573#endif
1574
1575 while (nents && sglist->dma_length) {
1576
1577 sba_unmap_single_attrs(dev, sglist->dma_address,
1578 sglist->dma_length, dir, attrs);
1579 sglist = sg_next(sglist);
1580 nents--;
1581 }
1582
1583 DBG_RUN_SG("%s() DONE (nents %d)\n", __func__, nents);
1584
1585#ifdef ASSERT_PDIR_SANITY
1586 spin_lock_irqsave(&ioc->res_lock, flags);
1587 sba_check_pdir(ioc,"Check after sba_unmap_sg_attrs()");
1588 spin_unlock_irqrestore(&ioc->res_lock, flags);
1589#endif
1590
1591}
1592
1593
1594
1595
1596
1597
1598
1599static void __init
1600ioc_iova_init(struct ioc *ioc)
1601{
1602 int tcnfg;
1603 int agp_found = 0;
1604 struct pci_dev *device = NULL;
1605#ifdef FULL_VALID_PDIR
1606 unsigned long index;
1607#endif
1608
1609
1610
1611
1612
1613
1614 ioc->ibase = READ_REG(ioc->ioc_hpa + IOC_IBASE) & ~0x1UL;
1615 ioc->imask = READ_REG(ioc->ioc_hpa + IOC_IMASK) | 0xFFFFFFFF00000000UL;
1616
1617 ioc->iov_size = ~ioc->imask + 1;
1618
1619 DBG_INIT("%s() hpa %p IOV base 0x%lx mask 0x%lx (%dMB)\n",
1620 __func__, ioc->ioc_hpa, ioc->ibase, ioc->imask,
1621 ioc->iov_size >> 20);
1622
1623 switch (iovp_size) {
1624 case 4*1024: tcnfg = 0; break;
1625 case 8*1024: tcnfg = 1; break;
1626 case 16*1024: tcnfg = 2; break;
1627 case 64*1024: tcnfg = 3; break;
1628 default:
1629 panic(PFX "Unsupported IOTLB page size %ldK",
1630 iovp_size >> 10);
1631 break;
1632 }
1633 WRITE_REG(tcnfg, ioc->ioc_hpa + IOC_TCNFG);
1634
1635 ioc->pdir_size = (ioc->iov_size / iovp_size) * PDIR_ENTRY_SIZE;
1636 ioc->pdir_base = (void *) __get_free_pages(GFP_KERNEL,
1637 get_order(ioc->pdir_size));
1638 if (!ioc->pdir_base)
1639 panic(PFX "Couldn't allocate I/O Page Table\n");
1640
1641 memset(ioc->pdir_base, 0, ioc->pdir_size);
1642
1643 DBG_INIT("%s() IOV page size %ldK pdir %p size %x\n", __func__,
1644 iovp_size >> 10, ioc->pdir_base, ioc->pdir_size);
1645
1646 ASSERT(ALIGN((unsigned long) ioc->pdir_base, 4*1024) == (unsigned long) ioc->pdir_base);
1647 WRITE_REG(virt_to_phys(ioc->pdir_base), ioc->ioc_hpa + IOC_PDIR_BASE);
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657 for_each_pci_dev(device)
1658 agp_found |= pci_find_capability(device, PCI_CAP_ID_AGP);
1659
1660 if (agp_found && reserve_sba_gart) {
1661 printk(KERN_INFO PFX "reserving %dMb of IOVA space at 0x%lx for agpgart\n",
1662 ioc->iov_size/2 >> 20, ioc->ibase + ioc->iov_size/2);
1663 ioc->pdir_size /= 2;
1664 ((u64 *)ioc->pdir_base)[PDIR_INDEX(ioc->iov_size/2)] = ZX1_SBA_IOMMU_COOKIE;
1665 }
1666#ifdef FULL_VALID_PDIR
1667
1668
1669
1670
1671 if (!prefetch_spill_page) {
1672 char *spill_poison = "SBAIOMMU POISON";
1673 int poison_size = 16;
1674 void *poison_addr, *addr;
1675
1676 addr = (void *)__get_free_pages(GFP_KERNEL, get_order(iovp_size));
1677 if (!addr)
1678 panic(PFX "Couldn't allocate PDIR spill page\n");
1679
1680 poison_addr = addr;
1681 for ( ; (u64) poison_addr < addr + iovp_size; poison_addr += poison_size)
1682 memcpy(poison_addr, spill_poison, poison_size);
1683
1684 prefetch_spill_page = virt_to_phys(addr);
1685
1686 DBG_INIT("%s() prefetch spill addr: 0x%lx\n", __func__, prefetch_spill_page);
1687 }
1688
1689
1690
1691 for (index = 0 ; index < (ioc->pdir_size / PDIR_ENTRY_SIZE) ; index++)
1692 ((u64 *)ioc->pdir_base)[index] = (0x80000000000000FF | prefetch_spill_page);
1693#endif
1694
1695
1696 WRITE_REG(ioc->ibase | (get_iovp_order(ioc->iov_size) + iovp_shift), ioc->ioc_hpa + IOC_PCOM);
1697 READ_REG(ioc->ioc_hpa + IOC_PCOM);
1698
1699
1700 WRITE_REG(ioc->ibase | 1, ioc->ioc_hpa + IOC_IBASE);
1701 READ_REG(ioc->ioc_hpa + IOC_IBASE);
1702}
1703
1704static void __init
1705ioc_resource_init(struct ioc *ioc)
1706{
1707 spin_lock_init(&ioc->res_lock);
1708#if DELAYED_RESOURCE_CNT > 0
1709 spin_lock_init(&ioc->saved_lock);
1710#endif
1711
1712
1713 ioc->res_size = ioc->pdir_size / PDIR_ENTRY_SIZE;
1714 ioc->res_size >>= 3;
1715 DBG_INIT("%s() res_size 0x%x\n", __func__, ioc->res_size);
1716
1717 ioc->res_map = (char *) __get_free_pages(GFP_KERNEL,
1718 get_order(ioc->res_size));
1719 if (!ioc->res_map)
1720 panic(PFX "Couldn't allocate resource map\n");
1721
1722 memset(ioc->res_map, 0, ioc->res_size);
1723
1724 ioc->res_hint = (unsigned long *) ioc->res_map;
1725
1726#ifdef ASSERT_PDIR_SANITY
1727
1728 ioc->res_map[0] = 0x1;
1729 ioc->pdir_base[0] = 0x8000000000000000ULL | ZX1_SBA_IOMMU_COOKIE;
1730#endif
1731#ifdef FULL_VALID_PDIR
1732
1733 ioc->res_map[ioc->res_size - 1] |= 0x80UL;
1734 ioc->pdir_base[(ioc->pdir_size / PDIR_ENTRY_SIZE) - 1] = (0x80000000000000FF
1735 | prefetch_spill_page);
1736#endif
1737
1738 DBG_INIT("%s() res_map %x %p\n", __func__,
1739 ioc->res_size, (void *) ioc->res_map);
1740}
1741
1742static void __init
1743ioc_sac_init(struct ioc *ioc)
1744{
1745 struct pci_dev *sac = NULL;
1746 struct pci_controller *controller = NULL;
1747
1748
1749
1750
1751
1752
1753 sac = kzalloc(sizeof(*sac), GFP_KERNEL);
1754 if (!sac)
1755 panic(PFX "Couldn't allocate struct pci_dev");
1756
1757 controller = kzalloc(sizeof(*controller), GFP_KERNEL);
1758 if (!controller)
1759 panic(PFX "Couldn't allocate struct pci_controller");
1760
1761 controller->iommu = ioc;
1762 sac->sysdata = controller;
1763 sac->dma_mask = 0xFFFFFFFFUL;
1764#ifdef CONFIG_PCI
1765 sac->dev.bus = &pci_bus_type;
1766#endif
1767 ioc->sac_only_dev = sac;
1768}
1769
1770static void __init
1771ioc_zx1_init(struct ioc *ioc)
1772{
1773 unsigned long rope_config;
1774 unsigned int i;
1775
1776 if (ioc->rev < 0x20)
1777 panic(PFX "IOC 2.0 or later required for IOMMU support\n");
1778
1779
1780 ioc->dma_mask = (0x1UL << 39) - 1;
1781
1782
1783
1784
1785
1786
1787
1788 for (i=0; i<(8*8); i+=8) {
1789 rope_config = READ_REG(ioc->ioc_hpa + IOC_ROPE0_CFG + i);
1790 rope_config &= ~IOC_ROPE_AO;
1791 WRITE_REG(rope_config, ioc->ioc_hpa + IOC_ROPE0_CFG + i);
1792 }
1793}
1794
1795typedef void (initfunc)(struct ioc *);
1796
1797struct ioc_iommu {
1798 u32 func_id;
1799 char *name;
1800 initfunc *init;
1801};
1802
1803static struct ioc_iommu ioc_iommu_info[] __initdata = {
1804 { ZX1_IOC_ID, "zx1", ioc_zx1_init },
1805 { ZX2_IOC_ID, "zx2", NULL },
1806 { SX1000_IOC_ID, "sx1000", NULL },
1807 { SX2000_IOC_ID, "sx2000", NULL },
1808};
1809
1810static struct ioc * __init
1811ioc_init(unsigned long hpa, void *handle)
1812{
1813 struct ioc *ioc;
1814 struct ioc_iommu *info;
1815
1816 ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
1817 if (!ioc)
1818 return NULL;
1819
1820 ioc->next = ioc_list;
1821 ioc_list = ioc;
1822
1823 ioc->handle = handle;
1824 ioc->ioc_hpa = ioremap(hpa, 0x1000);
1825
1826 ioc->func_id = READ_REG(ioc->ioc_hpa + IOC_FUNC_ID);
1827 ioc->rev = READ_REG(ioc->ioc_hpa + IOC_FCLASS) & 0xFFUL;
1828 ioc->dma_mask = 0xFFFFFFFFFFFFFFFFUL;
1829
1830 for (info = ioc_iommu_info; info < ioc_iommu_info + ARRAY_SIZE(ioc_iommu_info); info++) {
1831 if (ioc->func_id == info->func_id) {
1832 ioc->name = info->name;
1833 if (info->init)
1834 (info->init)(ioc);
1835 }
1836 }
1837
1838 iovp_size = (1 << iovp_shift);
1839 iovp_mask = ~(iovp_size - 1);
1840
1841 DBG_INIT("%s: PAGE_SIZE %ldK, iovp_size %ldK\n", __func__,
1842 PAGE_SIZE >> 10, iovp_size >> 10);
1843
1844 if (!ioc->name) {
1845 ioc->name = kmalloc(24, GFP_KERNEL);
1846 if (ioc->name)
1847 sprintf((char *) ioc->name, "Unknown (%04x:%04x)",
1848 ioc->func_id & 0xFFFF, (ioc->func_id >> 16) & 0xFFFF);
1849 else
1850 ioc->name = "Unknown";
1851 }
1852
1853 ioc_iova_init(ioc);
1854 ioc_resource_init(ioc);
1855 ioc_sac_init(ioc);
1856
1857 if ((long) ~iovp_mask > (long) ia64_max_iommu_merge_mask)
1858 ia64_max_iommu_merge_mask = ~iovp_mask;
1859
1860 printk(KERN_INFO PFX
1861 "%s %d.%d HPA 0x%lx IOVA space %dMb at 0x%lx\n",
1862 ioc->name, (ioc->rev >> 4) & 0xF, ioc->rev & 0xF,
1863 hpa, ioc->iov_size >> 20, ioc->ibase);
1864
1865 return ioc;
1866}
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879#ifdef CONFIG_PROC_FS
1880static void *
1881ioc_start(struct seq_file *s, loff_t *pos)
1882{
1883 struct ioc *ioc;
1884 loff_t n = *pos;
1885
1886 for (ioc = ioc_list; ioc; ioc = ioc->next)
1887 if (!n--)
1888 return ioc;
1889
1890 return NULL;
1891}
1892
1893static void *
1894ioc_next(struct seq_file *s, void *v, loff_t *pos)
1895{
1896 struct ioc *ioc = v;
1897
1898 ++*pos;
1899 return ioc->next;
1900}
1901
1902static void
1903ioc_stop(struct seq_file *s, void *v)
1904{
1905}
1906
1907static int
1908ioc_show(struct seq_file *s, void *v)
1909{
1910 struct ioc *ioc = v;
1911 unsigned long *res_ptr = (unsigned long *)ioc->res_map;
1912 int i, used = 0;
1913
1914 seq_printf(s, "Hewlett Packard %s IOC rev %d.%d\n",
1915 ioc->name, ((ioc->rev >> 4) & 0xF), (ioc->rev & 0xF));
1916#ifdef CONFIG_NUMA
1917 if (ioc->node != MAX_NUMNODES)
1918 seq_printf(s, "NUMA node : %d\n", ioc->node);
1919#endif
1920 seq_printf(s, "IOVA size : %ld MB\n", ((ioc->pdir_size >> 3) * iovp_size)/(1024*1024));
1921 seq_printf(s, "IOVA page size : %ld kb\n", iovp_size/1024);
1922
1923 for (i = 0; i < (ioc->res_size / sizeof(unsigned long)); ++i, ++res_ptr)
1924 used += hweight64(*res_ptr);
1925
1926 seq_printf(s, "PDIR size : %d entries\n", ioc->pdir_size >> 3);
1927 seq_printf(s, "PDIR used : %d entries\n", used);
1928
1929#ifdef PDIR_SEARCH_TIMING
1930 {
1931 unsigned long i = 0, avg = 0, min, max;
1932 min = max = ioc->avg_search[0];
1933 for (i = 0; i < SBA_SEARCH_SAMPLE; i++) {
1934 avg += ioc->avg_search[i];
1935 if (ioc->avg_search[i] > max) max = ioc->avg_search[i];
1936 if (ioc->avg_search[i] < min) min = ioc->avg_search[i];
1937 }
1938 avg /= SBA_SEARCH_SAMPLE;
1939 seq_printf(s, "Bitmap search : %ld/%ld/%ld (min/avg/max CPU Cycles/IOVA page)\n",
1940 min, avg, max);
1941 }
1942#endif
1943#ifndef ALLOW_IOV_BYPASS
1944 seq_printf(s, "IOVA bypass disabled\n");
1945#endif
1946 return 0;
1947}
1948
1949static const struct seq_operations ioc_seq_ops = {
1950 .start = ioc_start,
1951 .next = ioc_next,
1952 .stop = ioc_stop,
1953 .show = ioc_show
1954};
1955
1956static int
1957ioc_open(struct inode *inode, struct file *file)
1958{
1959 return seq_open(file, &ioc_seq_ops);
1960}
1961
1962static const struct file_operations ioc_fops = {
1963 .open = ioc_open,
1964 .read = seq_read,
1965 .llseek = seq_lseek,
1966 .release = seq_release
1967};
1968
1969static void __init
1970ioc_proc_init(void)
1971{
1972 struct proc_dir_entry *dir;
1973
1974 dir = proc_mkdir("bus/mckinley", NULL);
1975 if (!dir)
1976 return;
1977
1978 proc_create(ioc_list->name, 0, dir, &ioc_fops);
1979}
1980#endif
1981
1982static void
1983sba_connect_bus(struct pci_bus *bus)
1984{
1985 acpi_handle handle, parent;
1986 acpi_status status;
1987 struct ioc *ioc;
1988
1989 if (!PCI_CONTROLLER(bus))
1990 panic(PFX "no sysdata on bus %d!\n", bus->number);
1991
1992 if (PCI_CONTROLLER(bus)->iommu)
1993 return;
1994
1995 handle = PCI_CONTROLLER(bus)->acpi_handle;
1996 if (!handle)
1997 return;
1998
1999
2000
2001
2002
2003
2004 do {
2005 for (ioc = ioc_list; ioc; ioc = ioc->next)
2006 if (ioc->handle == handle) {
2007 PCI_CONTROLLER(bus)->iommu = ioc;
2008 return;
2009 }
2010
2011 status = acpi_get_parent(handle, &parent);
2012 handle = parent;
2013 } while (ACPI_SUCCESS(status));
2014
2015 printk(KERN_WARNING "No IOC for PCI Bus %04x:%02x in ACPI\n", pci_domain_nr(bus), bus->number);
2016}
2017
2018#ifdef CONFIG_NUMA
2019static void __init
2020sba_map_ioc_to_node(struct ioc *ioc, acpi_handle handle)
2021{
2022 unsigned int node;
2023 int pxm;
2024
2025 ioc->node = MAX_NUMNODES;
2026
2027 pxm = acpi_get_pxm(handle);
2028
2029 if (pxm < 0)
2030 return;
2031
2032 node = pxm_to_node(pxm);
2033
2034 if (node >= MAX_NUMNODES || !node_online(node))
2035 return;
2036
2037 ioc->node = node;
2038 return;
2039}
2040#else
2041#define sba_map_ioc_to_node(ioc, handle)
2042#endif
2043
2044static int __init
2045acpi_sba_ioc_add(struct acpi_device *device)
2046{
2047 struct ioc *ioc;
2048 acpi_status status;
2049 u64 hpa, length;
2050 struct acpi_device_info *adi;
2051
2052 status = hp_acpi_csr_space(device->handle, &hpa, &length);
2053 if (ACPI_FAILURE(status))
2054 return 1;
2055
2056 status = acpi_get_object_info(device->handle, &adi);
2057 if (ACPI_FAILURE(status))
2058 return 1;
2059
2060
2061
2062
2063
2064 if (strncmp("HWP0001", adi->hardware_id.string, 7) == 0) {
2065 hpa += ZX1_IOC_OFFSET;
2066
2067 if (!iovp_shift)
2068 iovp_shift = min(PAGE_SHIFT, 16);
2069 }
2070 kfree(adi);
2071
2072
2073
2074
2075
2076 if (!iovp_shift)
2077 iovp_shift = 12;
2078
2079 ioc = ioc_init(hpa, device->handle);
2080 if (!ioc)
2081 return 1;
2082
2083
2084 sba_map_ioc_to_node(ioc, device->handle);
2085 return 0;
2086}
2087
2088static const struct acpi_device_id hp_ioc_iommu_device_ids[] = {
2089 {"HWP0001", 0},
2090 {"HWP0004", 0},
2091 {"", 0},
2092};
2093static struct acpi_driver acpi_sba_ioc_driver = {
2094 .name = "IOC IOMMU Driver",
2095 .ids = hp_ioc_iommu_device_ids,
2096 .ops = {
2097 .add = acpi_sba_ioc_add,
2098 },
2099};
2100
2101extern struct dma_map_ops swiotlb_dma_ops;
2102
2103static int __init
2104sba_init(void)
2105{
2106 if (!ia64_platform_is("hpzx1") && !ia64_platform_is("hpzx1_swiotlb"))
2107 return 0;
2108
2109#if defined(CONFIG_IA64_GENERIC)
2110
2111
2112
2113
2114
2115 if (is_kdump_kernel()) {
2116 dma_ops = &swiotlb_dma_ops;
2117 if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
2118 panic("Unable to initialize software I/O TLB:"
2119 " Try machvec=dig boot option");
2120 machvec_init("dig");
2121 return 0;
2122 }
2123#endif
2124
2125 acpi_bus_register_driver(&acpi_sba_ioc_driver);
2126 if (!ioc_list) {
2127#ifdef CONFIG_IA64_GENERIC
2128
2129
2130
2131
2132 dma_ops = &swiotlb_dma_ops;
2133 if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
2134 panic("Unable to find SBA IOMMU or initialize "
2135 "software I/O TLB: Try machvec=dig boot option");
2136 machvec_init("dig");
2137#else
2138 panic("Unable to find SBA IOMMU: Try a generic or DIG kernel");
2139#endif
2140 return 0;
2141 }
2142
2143#if defined(CONFIG_IA64_GENERIC) || defined(CONFIG_IA64_HP_ZX1_SWIOTLB)
2144
2145
2146
2147
2148
2149 if (ia64_platform_is("hpzx1_swiotlb")) {
2150 extern void hwsw_init(void);
2151
2152 hwsw_init();
2153 }
2154#endif
2155
2156#ifdef CONFIG_PCI
2157 {
2158 struct pci_bus *b = NULL;
2159 while ((b = pci_find_next_bus(b)) != NULL)
2160 sba_connect_bus(b);
2161 }
2162#endif
2163
2164#ifdef CONFIG_PROC_FS
2165 ioc_proc_init();
2166#endif
2167 return 0;
2168}
2169
2170subsys_initcall(sba_init);
2171
2172static int __init
2173nosbagart(char *str)
2174{
2175 reserve_sba_gart = 0;
2176 return 1;
2177}
2178
2179static int sba_dma_supported (struct device *dev, u64 mask)
2180{
2181
2182 return ((mask & 0xFFFFFFFFUL) == 0xFFFFFFFFUL);
2183}
2184
2185static int sba_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
2186{
2187 return 0;
2188}
2189
2190__setup("nosbagart", nosbagart);
2191
2192static int __init
2193sba_page_override(char *str)
2194{
2195 unsigned long page_size;
2196
2197 page_size = memparse(str, &str);
2198 switch (page_size) {
2199 case 4096:
2200 case 8192:
2201 case 16384:
2202 case 65536:
2203 iovp_shift = ffs(page_size) - 1;
2204 break;
2205 default:
2206 printk("%s: unknown/unsupported iommu page size %ld\n",
2207 __func__, page_size);
2208 }
2209
2210 return 1;
2211}
2212
2213__setup("sbapagesize=",sba_page_override);
2214
2215struct dma_map_ops sba_dma_ops = {
2216 .alloc_coherent = sba_alloc_coherent,
2217 .free_coherent = sba_free_coherent,
2218 .map_page = sba_map_page,
2219 .unmap_page = sba_unmap_page,
2220 .map_sg = sba_map_sg_attrs,
2221 .unmap_sg = sba_unmap_sg_attrs,
2222 .sync_single_for_cpu = machvec_dma_sync_single,
2223 .sync_sg_for_cpu = machvec_dma_sync_sg,
2224 .sync_single_for_device = machvec_dma_sync_single,
2225 .sync_sg_for_device = machvec_dma_sync_sg,
2226 .dma_supported = sba_dma_supported,
2227 .mapping_error = sba_dma_mapping_error,
2228};
2229
2230void sba_dma_init(void)
2231{
2232 dma_ops = &sba_dma_ops;
2233}
2234