1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/kernel.h>
22#include <linux/export.h>
23#include <linux/memblock.h>
24#include <linux/slab.h>
25
26#include <asm/cell-regs.h>
27#include <asm/firmware.h>
28#include <asm/prom.h>
29#include <asm/udbg.h>
30#include <asm/lv1call.h>
31#include <asm/setup.h>
32
33#include "platform.h"
34
35#if defined(DEBUG)
36#define DBG udbg_printf
37#else
38#define DBG pr_devel
39#endif
40
41enum {
42#if defined(CONFIG_PS3_DYNAMIC_DMA)
43 USE_DYNAMIC_DMA = 1,
44#else
45 USE_DYNAMIC_DMA = 0,
46#endif
47};
48
49enum {
50 PAGE_SHIFT_4K = 12U,
51 PAGE_SHIFT_64K = 16U,
52 PAGE_SHIFT_16M = 24U,
53};
54
55static unsigned long make_page_sizes(unsigned long a, unsigned long b)
56{
57 return (a << 56) | (b << 48);
58}
59
60enum {
61 ALLOCATE_MEMORY_TRY_ALT_UNIT = 0X04,
62 ALLOCATE_MEMORY_ADDR_ZERO = 0X08,
63};
64
65
66
67enum {
68 HTAB_SIZE_MAX = 20U,
69 HTAB_SIZE_MIN = 18U,
70};
71
72
73
74
75
76
77
78
79
80
81
82
83
84struct mem_region {
85 u64 base;
86 u64 size;
87 unsigned long offset;
88 int destroy;
89};
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109struct map {
110 u64 total;
111 u64 vas_id;
112 u64 htab_size;
113 struct mem_region rm;
114 struct mem_region r1;
115};
116
117#define debug_dump_map(x) _debug_dump_map(x, __func__, __LINE__)
118static void __maybe_unused _debug_dump_map(const struct map *m,
119 const char *func, int line)
120{
121 DBG("%s:%d: map.total = %llxh\n", func, line, m->total);
122 DBG("%s:%d: map.rm.size = %llxh\n", func, line, m->rm.size);
123 DBG("%s:%d: map.vas_id = %llu\n", func, line, m->vas_id);
124 DBG("%s:%d: map.htab_size = %llxh\n", func, line, m->htab_size);
125 DBG("%s:%d: map.r1.base = %llxh\n", func, line, m->r1.base);
126 DBG("%s:%d: map.r1.offset = %lxh\n", func, line, m->r1.offset);
127 DBG("%s:%d: map.r1.size = %llxh\n", func, line, m->r1.size);
128}
129
130static struct map map;
131
132
133
134
135
136
137unsigned long ps3_mm_phys_to_lpar(unsigned long phys_addr)
138{
139 BUG_ON(is_kernel_addr(phys_addr));
140 return (phys_addr < map.rm.size || phys_addr >= map.total)
141 ? phys_addr : phys_addr + map.r1.offset;
142}
143
144EXPORT_SYMBOL(ps3_mm_phys_to_lpar);
145
146
147
148
149
150void __init ps3_mm_vas_create(unsigned long* htab_size)
151{
152 int result;
153 u64 start_address;
154 u64 size;
155 u64 access_right;
156 u64 max_page_size;
157 u64 flags;
158
159 result = lv1_query_logical_partition_address_region_info(0,
160 &start_address, &size, &access_right, &max_page_size,
161 &flags);
162
163 if (result) {
164 DBG("%s:%d: lv1_query_logical_partition_address_region_info "
165 "failed: %s\n", __func__, __LINE__,
166 ps3_result(result));
167 goto fail;
168 }
169
170 if (max_page_size < PAGE_SHIFT_16M) {
171 DBG("%s:%d: bad max_page_size %llxh\n", __func__, __LINE__,
172 max_page_size);
173 goto fail;
174 }
175
176 BUILD_BUG_ON(CONFIG_PS3_HTAB_SIZE > HTAB_SIZE_MAX);
177 BUILD_BUG_ON(CONFIG_PS3_HTAB_SIZE < HTAB_SIZE_MIN);
178
179 result = lv1_construct_virtual_address_space(CONFIG_PS3_HTAB_SIZE,
180 2, make_page_sizes(PAGE_SHIFT_16M, PAGE_SHIFT_64K),
181 &map.vas_id, &map.htab_size);
182
183 if (result) {
184 DBG("%s:%d: lv1_construct_virtual_address_space failed: %s\n",
185 __func__, __LINE__, ps3_result(result));
186 goto fail;
187 }
188
189 result = lv1_select_virtual_address_space(map.vas_id);
190
191 if (result) {
192 DBG("%s:%d: lv1_select_virtual_address_space failed: %s\n",
193 __func__, __LINE__, ps3_result(result));
194 goto fail;
195 }
196
197 *htab_size = map.htab_size;
198
199 debug_dump_map(&map);
200
201 return;
202
203fail:
204 panic("ps3_mm_vas_create failed");
205}
206
207
208
209
210
211void ps3_mm_vas_destroy(void)
212{
213 int result;
214
215 DBG("%s:%d: map.vas_id = %llu\n", __func__, __LINE__, map.vas_id);
216
217 if (map.vas_id) {
218 result = lv1_select_virtual_address_space(0);
219 BUG_ON(result);
220 result = lv1_destruct_virtual_address_space(map.vas_id);
221 BUG_ON(result);
222 map.vas_id = 0;
223 }
224}
225
226static int ps3_mm_get_repository_highmem(struct mem_region *r)
227{
228 int result;
229
230
231
232 result = ps3_repository_read_highmem_info(0, &r->base, &r->size);
233
234 if (result)
235 goto zero_region;
236
237 if (!r->base || !r->size) {
238 result = -1;
239 goto zero_region;
240 }
241
242 r->offset = r->base - map.rm.size;
243
244 DBG("%s:%d: Found high region in repository: %llxh %llxh\n",
245 __func__, __LINE__, r->base, r->size);
246
247 return 0;
248
249zero_region:
250 DBG("%s:%d: No high region in repository.\n", __func__, __LINE__);
251
252 r->size = r->base = r->offset = 0;
253 return result;
254}
255
256static int ps3_mm_set_repository_highmem(const struct mem_region *r)
257{
258
259
260 return r ? ps3_repository_write_highmem_info(0, r->base, r->size) :
261 ps3_repository_write_highmem_info(0, 0, 0);
262}
263
264
265
266
267
268
269
270
271
272
273static int ps3_mm_region_create(struct mem_region *r, unsigned long size)
274{
275 int result;
276 u64 muid;
277
278 r->size = _ALIGN_DOWN(size, 1 << PAGE_SHIFT_16M);
279
280 DBG("%s:%d requested %lxh\n", __func__, __LINE__, size);
281 DBG("%s:%d actual %llxh\n", __func__, __LINE__, r->size);
282 DBG("%s:%d difference %llxh (%lluMB)\n", __func__, __LINE__,
283 size - r->size, (size - r->size) / 1024 / 1024);
284
285 if (r->size == 0) {
286 DBG("%s:%d: size == 0\n", __func__, __LINE__);
287 result = -1;
288 goto zero_region;
289 }
290
291 result = lv1_allocate_memory(r->size, PAGE_SHIFT_16M, 0,
292 ALLOCATE_MEMORY_TRY_ALT_UNIT, &r->base, &muid);
293
294 if (result || r->base < map.rm.size) {
295 DBG("%s:%d: lv1_allocate_memory failed: %s\n",
296 __func__, __LINE__, ps3_result(result));
297 goto zero_region;
298 }
299
300 r->destroy = 1;
301 r->offset = r->base - map.rm.size;
302 return result;
303
304zero_region:
305 r->size = r->base = r->offset = 0;
306 return result;
307}
308
309
310
311
312
313
314static void ps3_mm_region_destroy(struct mem_region *r)
315{
316 int result;
317
318 if (!r->destroy) {
319 pr_info("%s:%d: Not destroying high region: %llxh %llxh\n",
320 __func__, __LINE__, r->base, r->size);
321 return;
322 }
323
324 DBG("%s:%d: r->base = %llxh\n", __func__, __LINE__, r->base);
325
326 if (r->base) {
327 result = lv1_release_memory(r->base);
328 BUG_ON(result);
329 r->size = r->base = r->offset = 0;
330 map.total = map.rm.size;
331 }
332 ps3_mm_set_repository_highmem(NULL);
333}
334
335
336
337
338
339
340
341
342
343
344
345static unsigned long dma_sb_lpar_to_bus(struct ps3_dma_region *r,
346 unsigned long lpar_addr)
347{
348 if (lpar_addr >= map.rm.size)
349 lpar_addr -= map.r1.offset;
350 BUG_ON(lpar_addr < r->offset);
351 BUG_ON(lpar_addr >= r->offset + r->len);
352 return r->bus_addr + lpar_addr - r->offset;
353}
354
355#define dma_dump_region(_a) _dma_dump_region(_a, __func__, __LINE__)
356static void __maybe_unused _dma_dump_region(const struct ps3_dma_region *r,
357 const char *func, int line)
358{
359 DBG("%s:%d: dev %llu:%llu\n", func, line, r->dev->bus_id,
360 r->dev->dev_id);
361 DBG("%s:%d: page_size %u\n", func, line, r->page_size);
362 DBG("%s:%d: bus_addr %lxh\n", func, line, r->bus_addr);
363 DBG("%s:%d: len %lxh\n", func, line, r->len);
364 DBG("%s:%d: offset %lxh\n", func, line, r->offset);
365}
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381struct dma_chunk {
382 struct ps3_dma_region *region;
383 unsigned long lpar_addr;
384 unsigned long bus_addr;
385 unsigned long len;
386 struct list_head link;
387 unsigned int usage_count;
388};
389
390#define dma_dump_chunk(_a) _dma_dump_chunk(_a, __func__, __LINE__)
391static void _dma_dump_chunk (const struct dma_chunk* c, const char* func,
392 int line)
393{
394 DBG("%s:%d: r.dev %llu:%llu\n", func, line,
395 c->region->dev->bus_id, c->region->dev->dev_id);
396 DBG("%s:%d: r.bus_addr %lxh\n", func, line, c->region->bus_addr);
397 DBG("%s:%d: r.page_size %u\n", func, line, c->region->page_size);
398 DBG("%s:%d: r.len %lxh\n", func, line, c->region->len);
399 DBG("%s:%d: r.offset %lxh\n", func, line, c->region->offset);
400 DBG("%s:%d: c.lpar_addr %lxh\n", func, line, c->lpar_addr);
401 DBG("%s:%d: c.bus_addr %lxh\n", func, line, c->bus_addr);
402 DBG("%s:%d: c.len %lxh\n", func, line, c->len);
403}
404
405static struct dma_chunk * dma_find_chunk(struct ps3_dma_region *r,
406 unsigned long bus_addr, unsigned long len)
407{
408 struct dma_chunk *c;
409 unsigned long aligned_bus = _ALIGN_DOWN(bus_addr, 1 << r->page_size);
410 unsigned long aligned_len = _ALIGN_UP(len+bus_addr-aligned_bus,
411 1 << r->page_size);
412
413 list_for_each_entry(c, &r->chunk_list.head, link) {
414
415 if (aligned_bus >= c->bus_addr &&
416 aligned_bus + aligned_len <= c->bus_addr + c->len)
417 return c;
418
419
420 if (aligned_bus + aligned_len <= c->bus_addr)
421 continue;
422
423
424 if (aligned_bus >= c->bus_addr + c->len)
425 continue;
426
427
428 dma_dump_chunk(c);
429 BUG();
430 }
431 return NULL;
432}
433
434static struct dma_chunk *dma_find_chunk_lpar(struct ps3_dma_region *r,
435 unsigned long lpar_addr, unsigned long len)
436{
437 struct dma_chunk *c;
438 unsigned long aligned_lpar = _ALIGN_DOWN(lpar_addr, 1 << r->page_size);
439 unsigned long aligned_len = _ALIGN_UP(len + lpar_addr - aligned_lpar,
440 1 << r->page_size);
441
442 list_for_each_entry(c, &r->chunk_list.head, link) {
443
444 if (c->lpar_addr <= aligned_lpar &&
445 aligned_lpar < c->lpar_addr + c->len) {
446 if (aligned_lpar + aligned_len <= c->lpar_addr + c->len)
447 return c;
448 else {
449 dma_dump_chunk(c);
450 BUG();
451 }
452 }
453
454 if (aligned_lpar + aligned_len <= c->lpar_addr) {
455 continue;
456 }
457
458 if (c->lpar_addr + c->len <= aligned_lpar) {
459 continue;
460 }
461 }
462 return NULL;
463}
464
465static int dma_sb_free_chunk(struct dma_chunk *c)
466{
467 int result = 0;
468
469 if (c->bus_addr) {
470 result = lv1_unmap_device_dma_region(c->region->dev->bus_id,
471 c->region->dev->dev_id, c->bus_addr, c->len);
472 BUG_ON(result);
473 }
474
475 kfree(c);
476 return result;
477}
478
479static int dma_ioc0_free_chunk(struct dma_chunk *c)
480{
481 int result = 0;
482 int iopage;
483 unsigned long offset;
484 struct ps3_dma_region *r = c->region;
485
486 DBG("%s:start\n", __func__);
487 for (iopage = 0; iopage < (c->len >> r->page_size); iopage++) {
488 offset = (1 << r->page_size) * iopage;
489
490 result = lv1_put_iopte(0,
491 c->bus_addr + offset,
492 c->lpar_addr + offset,
493 r->ioid,
494 0);
495 DBG("%s: bus=%#lx, lpar=%#lx, ioid=%d\n", __func__,
496 c->bus_addr + offset,
497 c->lpar_addr + offset,
498 r->ioid);
499
500 if (result) {
501 DBG("%s:%d: lv1_put_iopte failed: %s\n", __func__,
502 __LINE__, ps3_result(result));
503 }
504 }
505 kfree(c);
506 DBG("%s:end\n", __func__);
507 return result;
508}
509
510
511
512
513
514
515
516
517
518
519
520
521static int dma_sb_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,
522 unsigned long len, struct dma_chunk **c_out, u64 iopte_flag)
523{
524 int result;
525 struct dma_chunk *c;
526
527 c = kzalloc(sizeof(*c), GFP_ATOMIC);
528 if (!c) {
529 result = -ENOMEM;
530 goto fail_alloc;
531 }
532
533 c->region = r;
534 c->lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
535 c->bus_addr = dma_sb_lpar_to_bus(r, c->lpar_addr);
536 c->len = len;
537
538 BUG_ON(iopte_flag != 0xf800000000000000UL);
539 result = lv1_map_device_dma_region(c->region->dev->bus_id,
540 c->region->dev->dev_id, c->lpar_addr,
541 c->bus_addr, c->len, iopte_flag);
542 if (result) {
543 DBG("%s:%d: lv1_map_device_dma_region failed: %s\n",
544 __func__, __LINE__, ps3_result(result));
545 goto fail_map;
546 }
547
548 list_add(&c->link, &r->chunk_list.head);
549
550 *c_out = c;
551 return 0;
552
553fail_map:
554 kfree(c);
555fail_alloc:
556 *c_out = NULL;
557 DBG(" <- %s:%d\n", __func__, __LINE__);
558 return result;
559}
560
561static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,
562 unsigned long len, struct dma_chunk **c_out,
563 u64 iopte_flag)
564{
565 int result;
566 struct dma_chunk *c, *last;
567 int iopage, pages;
568 unsigned long offset;
569
570 DBG(KERN_ERR "%s: phy=%#lx, lpar%#lx, len=%#lx\n", __func__,
571 phys_addr, ps3_mm_phys_to_lpar(phys_addr), len);
572 c = kzalloc(sizeof(*c), GFP_ATOMIC);
573 if (!c) {
574 result = -ENOMEM;
575 goto fail_alloc;
576 }
577
578 c->region = r;
579 c->len = len;
580 c->lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
581
582 if (list_empty(&r->chunk_list.head)) {
583
584 c->bus_addr = r->bus_addr;
585 } else {
586
587 last = list_entry(r->chunk_list.head.next,
588 struct dma_chunk, link);
589 c->bus_addr = last->bus_addr + last->len;
590 DBG("%s: last bus=%#lx, len=%#lx\n", __func__,
591 last->bus_addr, last->len);
592 }
593
594
595
596
597 pages = len >> r->page_size;
598 DBG("%s: pgsize=%#x len=%#lx pages=%#x iopteflag=%#llx\n", __func__,
599 r->page_size, r->len, pages, iopte_flag);
600 for (iopage = 0; iopage < pages; iopage++) {
601 offset = (1 << r->page_size) * iopage;
602 result = lv1_put_iopte(0,
603 c->bus_addr + offset,
604 c->lpar_addr + offset,
605 r->ioid,
606 iopte_flag);
607 if (result) {
608 pr_warn("%s:%d: lv1_put_iopte failed: %s\n",
609 __func__, __LINE__, ps3_result(result));
610 goto fail_map;
611 }
612 DBG("%s: pg=%d bus=%#lx, lpar=%#lx, ioid=%#x\n", __func__,
613 iopage, c->bus_addr + offset, c->lpar_addr + offset,
614 r->ioid);
615 }
616
617
618 list_add(&c->link, &r->chunk_list.head);
619
620 *c_out = c;
621 DBG("%s: end\n", __func__);
622 return 0;
623
624fail_map:
625 for (iopage--; 0 <= iopage; iopage--) {
626 lv1_put_iopte(0,
627 c->bus_addr + offset,
628 c->lpar_addr + offset,
629 r->ioid,
630 0);
631 }
632 kfree(c);
633fail_alloc:
634 *c_out = NULL;
635 return result;
636}
637
638
639
640
641
642
643
644
645
646static int dma_sb_region_create(struct ps3_dma_region *r)
647{
648 int result;
649 u64 bus_addr;
650
651 DBG(" -> %s:%d:\n", __func__, __LINE__);
652
653 BUG_ON(!r);
654
655 if (!r->dev->bus_id) {
656 pr_info("%s:%d: %llu:%llu no dma\n", __func__, __LINE__,
657 r->dev->bus_id, r->dev->dev_id);
658 return 0;
659 }
660
661 DBG("%s:%u: len = 0x%lx, page_size = %u, offset = 0x%lx\n", __func__,
662 __LINE__, r->len, r->page_size, r->offset);
663
664 BUG_ON(!r->len);
665 BUG_ON(!r->page_size);
666 BUG_ON(!r->region_ops);
667
668 INIT_LIST_HEAD(&r->chunk_list.head);
669 spin_lock_init(&r->chunk_list.lock);
670
671 result = lv1_allocate_device_dma_region(r->dev->bus_id, r->dev->dev_id,
672 roundup_pow_of_two(r->len), r->page_size, r->region_type,
673 &bus_addr);
674 r->bus_addr = bus_addr;
675
676 if (result) {
677 DBG("%s:%d: lv1_allocate_device_dma_region failed: %s\n",
678 __func__, __LINE__, ps3_result(result));
679 r->len = r->bus_addr = 0;
680 }
681
682 return result;
683}
684
685static int dma_ioc0_region_create(struct ps3_dma_region *r)
686{
687 int result;
688 u64 bus_addr;
689
690 INIT_LIST_HEAD(&r->chunk_list.head);
691 spin_lock_init(&r->chunk_list.lock);
692
693 result = lv1_allocate_io_segment(0,
694 r->len,
695 r->page_size,
696 &bus_addr);
697 r->bus_addr = bus_addr;
698 if (result) {
699 DBG("%s:%d: lv1_allocate_io_segment failed: %s\n",
700 __func__, __LINE__, ps3_result(result));
701 r->len = r->bus_addr = 0;
702 }
703 DBG("%s: len=%#lx, pg=%d, bus=%#lx\n", __func__,
704 r->len, r->page_size, r->bus_addr);
705 return result;
706}
707
708
709
710
711
712
713
714
715
716static int dma_sb_region_free(struct ps3_dma_region *r)
717{
718 int result;
719 struct dma_chunk *c;
720 struct dma_chunk *tmp;
721
722 BUG_ON(!r);
723
724 if (!r->dev->bus_id) {
725 pr_info("%s:%d: %llu:%llu no dma\n", __func__, __LINE__,
726 r->dev->bus_id, r->dev->dev_id);
727 return 0;
728 }
729
730 list_for_each_entry_safe(c, tmp, &r->chunk_list.head, link) {
731 list_del(&c->link);
732 dma_sb_free_chunk(c);
733 }
734
735 result = lv1_free_device_dma_region(r->dev->bus_id, r->dev->dev_id,
736 r->bus_addr);
737
738 if (result)
739 DBG("%s:%d: lv1_free_device_dma_region failed: %s\n",
740 __func__, __LINE__, ps3_result(result));
741
742 r->bus_addr = 0;
743
744 return result;
745}
746
747static int dma_ioc0_region_free(struct ps3_dma_region *r)
748{
749 int result;
750 struct dma_chunk *c, *n;
751
752 DBG("%s: start\n", __func__);
753 list_for_each_entry_safe(c, n, &r->chunk_list.head, link) {
754 list_del(&c->link);
755 dma_ioc0_free_chunk(c);
756 }
757
758 result = lv1_release_io_segment(0, r->bus_addr);
759
760 if (result)
761 DBG("%s:%d: lv1_free_device_dma_region failed: %s\n",
762 __func__, __LINE__, ps3_result(result));
763
764 r->bus_addr = 0;
765 DBG("%s: end\n", __func__);
766
767 return result;
768}
769
770
771
772
773
774
775
776
777
778
779
780
781static int dma_sb_map_area(struct ps3_dma_region *r, unsigned long virt_addr,
782 unsigned long len, dma_addr_t *bus_addr,
783 u64 iopte_flag)
784{
785 int result;
786 unsigned long flags;
787 struct dma_chunk *c;
788 unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr)
789 : virt_addr;
790 unsigned long aligned_phys = _ALIGN_DOWN(phys_addr, 1 << r->page_size);
791 unsigned long aligned_len = _ALIGN_UP(len + phys_addr - aligned_phys,
792 1 << r->page_size);
793 *bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr));
794
795 if (!USE_DYNAMIC_DMA) {
796 unsigned long lpar_addr = ps3_mm_phys_to_lpar(phys_addr);
797 DBG(" -> %s:%d\n", __func__, __LINE__);
798 DBG("%s:%d virt_addr %lxh\n", __func__, __LINE__,
799 virt_addr);
800 DBG("%s:%d phys_addr %lxh\n", __func__, __LINE__,
801 phys_addr);
802 DBG("%s:%d lpar_addr %lxh\n", __func__, __LINE__,
803 lpar_addr);
804 DBG("%s:%d len %lxh\n", __func__, __LINE__, len);
805 DBG("%s:%d bus_addr %llxh (%lxh)\n", __func__, __LINE__,
806 *bus_addr, len);
807 }
808
809 spin_lock_irqsave(&r->chunk_list.lock, flags);
810 c = dma_find_chunk(r, *bus_addr, len);
811
812 if (c) {
813 DBG("%s:%d: reusing mapped chunk", __func__, __LINE__);
814 dma_dump_chunk(c);
815 c->usage_count++;
816 spin_unlock_irqrestore(&r->chunk_list.lock, flags);
817 return 0;
818 }
819
820 result = dma_sb_map_pages(r, aligned_phys, aligned_len, &c, iopte_flag);
821
822 if (result) {
823 *bus_addr = 0;
824 DBG("%s:%d: dma_sb_map_pages failed (%d)\n",
825 __func__, __LINE__, result);
826 spin_unlock_irqrestore(&r->chunk_list.lock, flags);
827 return result;
828 }
829
830 c->usage_count = 1;
831
832 spin_unlock_irqrestore(&r->chunk_list.lock, flags);
833 return result;
834}
835
836static int dma_ioc0_map_area(struct ps3_dma_region *r, unsigned long virt_addr,
837 unsigned long len, dma_addr_t *bus_addr,
838 u64 iopte_flag)
839{
840 int result;
841 unsigned long flags;
842 struct dma_chunk *c;
843 unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr)
844 : virt_addr;
845 unsigned long aligned_phys = _ALIGN_DOWN(phys_addr, 1 << r->page_size);
846 unsigned long aligned_len = _ALIGN_UP(len + phys_addr - aligned_phys,
847 1 << r->page_size);
848
849 DBG(KERN_ERR "%s: vaddr=%#lx, len=%#lx\n", __func__,
850 virt_addr, len);
851 DBG(KERN_ERR "%s: ph=%#lx a_ph=%#lx a_l=%#lx\n", __func__,
852 phys_addr, aligned_phys, aligned_len);
853
854 spin_lock_irqsave(&r->chunk_list.lock, flags);
855 c = dma_find_chunk_lpar(r, ps3_mm_phys_to_lpar(phys_addr), len);
856
857 if (c) {
858
859 BUG();
860 *bus_addr = c->bus_addr + phys_addr - aligned_phys;
861 c->usage_count++;
862 spin_unlock_irqrestore(&r->chunk_list.lock, flags);
863 return 0;
864 }
865
866 result = dma_ioc0_map_pages(r, aligned_phys, aligned_len, &c,
867 iopte_flag);
868
869 if (result) {
870 *bus_addr = 0;
871 DBG("%s:%d: dma_ioc0_map_pages failed (%d)\n",
872 __func__, __LINE__, result);
873 spin_unlock_irqrestore(&r->chunk_list.lock, flags);
874 return result;
875 }
876 *bus_addr = c->bus_addr + phys_addr - aligned_phys;
877 DBG("%s: va=%#lx pa=%#lx a_pa=%#lx bus=%#llx\n", __func__,
878 virt_addr, phys_addr, aligned_phys, *bus_addr);
879 c->usage_count = 1;
880
881 spin_unlock_irqrestore(&r->chunk_list.lock, flags);
882 return result;
883}
884
885
886
887
888
889
890
891
892
893
894static int dma_sb_unmap_area(struct ps3_dma_region *r, dma_addr_t bus_addr,
895 unsigned long len)
896{
897 unsigned long flags;
898 struct dma_chunk *c;
899
900 spin_lock_irqsave(&r->chunk_list.lock, flags);
901 c = dma_find_chunk(r, bus_addr, len);
902
903 if (!c) {
904 unsigned long aligned_bus = _ALIGN_DOWN(bus_addr,
905 1 << r->page_size);
906 unsigned long aligned_len = _ALIGN_UP(len + bus_addr
907 - aligned_bus, 1 << r->page_size);
908 DBG("%s:%d: not found: bus_addr %llxh\n",
909 __func__, __LINE__, bus_addr);
910 DBG("%s:%d: not found: len %lxh\n",
911 __func__, __LINE__, len);
912 DBG("%s:%d: not found: aligned_bus %lxh\n",
913 __func__, __LINE__, aligned_bus);
914 DBG("%s:%d: not found: aligned_len %lxh\n",
915 __func__, __LINE__, aligned_len);
916 BUG();
917 }
918
919 c->usage_count--;
920
921 if (!c->usage_count) {
922 list_del(&c->link);
923 dma_sb_free_chunk(c);
924 }
925
926 spin_unlock_irqrestore(&r->chunk_list.lock, flags);
927 return 0;
928}
929
930static int dma_ioc0_unmap_area(struct ps3_dma_region *r,
931 dma_addr_t bus_addr, unsigned long len)
932{
933 unsigned long flags;
934 struct dma_chunk *c;
935
936 DBG("%s: start a=%#llx l=%#lx\n", __func__, bus_addr, len);
937 spin_lock_irqsave(&r->chunk_list.lock, flags);
938 c = dma_find_chunk(r, bus_addr, len);
939
940 if (!c) {
941 unsigned long aligned_bus = _ALIGN_DOWN(bus_addr,
942 1 << r->page_size);
943 unsigned long aligned_len = _ALIGN_UP(len + bus_addr
944 - aligned_bus,
945 1 << r->page_size);
946 DBG("%s:%d: not found: bus_addr %llxh\n",
947 __func__, __LINE__, bus_addr);
948 DBG("%s:%d: not found: len %lxh\n",
949 __func__, __LINE__, len);
950 DBG("%s:%d: not found: aligned_bus %lxh\n",
951 __func__, __LINE__, aligned_bus);
952 DBG("%s:%d: not found: aligned_len %lxh\n",
953 __func__, __LINE__, aligned_len);
954 BUG();
955 }
956
957 c->usage_count--;
958
959 if (!c->usage_count) {
960 list_del(&c->link);
961 dma_ioc0_free_chunk(c);
962 }
963
964 spin_unlock_irqrestore(&r->chunk_list.lock, flags);
965 DBG("%s: end\n", __func__);
966 return 0;
967}
968
969
970
971
972
973
974
975
976
977static int dma_sb_region_create_linear(struct ps3_dma_region *r)
978{
979 int result;
980 unsigned long virt_addr, len;
981 dma_addr_t tmp;
982
983 if (r->len > 16*1024*1024) {
984
985 if (r->page_size != PS3_DMA_16M) {
986 pr_info("%s:%d: forcing 16M pages for linear map\n",
987 __func__, __LINE__);
988 r->page_size = PS3_DMA_16M;
989 r->len = _ALIGN_UP(r->len, 1 << r->page_size);
990 }
991 }
992
993 result = dma_sb_region_create(r);
994 BUG_ON(result);
995
996 if (r->offset < map.rm.size) {
997
998 virt_addr = map.rm.base + r->offset;
999 len = map.rm.size - r->offset;
1000 if (len > r->len)
1001 len = r->len;
1002 result = dma_sb_map_area(r, virt_addr, len, &tmp,
1003 CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_SO_RW |
1004 CBE_IOPTE_M);
1005 BUG_ON(result);
1006 }
1007
1008 if (r->offset + r->len > map.rm.size) {
1009
1010 virt_addr = map.rm.size;
1011 len = r->len;
1012 if (r->offset >= map.rm.size)
1013 virt_addr += r->offset - map.rm.size;
1014 else
1015 len -= map.rm.size - r->offset;
1016 result = dma_sb_map_area(r, virt_addr, len, &tmp,
1017 CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_SO_RW |
1018 CBE_IOPTE_M);
1019 BUG_ON(result);
1020 }
1021
1022 return result;
1023}
1024
1025
1026
1027
1028
1029
1030
1031
1032static int dma_sb_region_free_linear(struct ps3_dma_region *r)
1033{
1034 int result;
1035 dma_addr_t bus_addr;
1036 unsigned long len, lpar_addr;
1037
1038 if (r->offset < map.rm.size) {
1039
1040 lpar_addr = map.rm.base + r->offset;
1041 len = map.rm.size - r->offset;
1042 if (len > r->len)
1043 len = r->len;
1044 bus_addr = dma_sb_lpar_to_bus(r, lpar_addr);
1045 result = dma_sb_unmap_area(r, bus_addr, len);
1046 BUG_ON(result);
1047 }
1048
1049 if (r->offset + r->len > map.rm.size) {
1050
1051 lpar_addr = map.r1.base;
1052 len = r->len;
1053 if (r->offset >= map.rm.size)
1054 lpar_addr += r->offset - map.rm.size;
1055 else
1056 len -= map.rm.size - r->offset;
1057 bus_addr = dma_sb_lpar_to_bus(r, lpar_addr);
1058 result = dma_sb_unmap_area(r, bus_addr, len);
1059 BUG_ON(result);
1060 }
1061
1062 result = dma_sb_region_free(r);
1063 BUG_ON(result);
1064
1065 return result;
1066}
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080static int dma_sb_map_area_linear(struct ps3_dma_region *r,
1081 unsigned long virt_addr, unsigned long len, dma_addr_t *bus_addr,
1082 u64 iopte_flag)
1083{
1084 unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr)
1085 : virt_addr;
1086 *bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr));
1087 return 0;
1088}
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099static int dma_sb_unmap_area_linear(struct ps3_dma_region *r,
1100 dma_addr_t bus_addr, unsigned long len)
1101{
1102 return 0;
1103};
1104
1105static const struct ps3_dma_region_ops ps3_dma_sb_region_ops = {
1106 .create = dma_sb_region_create,
1107 .free = dma_sb_region_free,
1108 .map = dma_sb_map_area,
1109 .unmap = dma_sb_unmap_area
1110};
1111
1112static const struct ps3_dma_region_ops ps3_dma_sb_region_linear_ops = {
1113 .create = dma_sb_region_create_linear,
1114 .free = dma_sb_region_free_linear,
1115 .map = dma_sb_map_area_linear,
1116 .unmap = dma_sb_unmap_area_linear
1117};
1118
1119static const struct ps3_dma_region_ops ps3_dma_ioc0_region_ops = {
1120 .create = dma_ioc0_region_create,
1121 .free = dma_ioc0_region_free,
1122 .map = dma_ioc0_map_area,
1123 .unmap = dma_ioc0_unmap_area
1124};
1125
1126int ps3_dma_region_init(struct ps3_system_bus_device *dev,
1127 struct ps3_dma_region *r, enum ps3_dma_page_size page_size,
1128 enum ps3_dma_region_type region_type, void *addr, unsigned long len)
1129{
1130 unsigned long lpar_addr;
1131
1132 lpar_addr = addr ? ps3_mm_phys_to_lpar(__pa(addr)) : 0;
1133
1134 r->dev = dev;
1135 r->page_size = page_size;
1136 r->region_type = region_type;
1137 r->offset = lpar_addr;
1138 if (r->offset >= map.rm.size)
1139 r->offset -= map.r1.offset;
1140 r->len = len ? len : _ALIGN_UP(map.total, 1 << r->page_size);
1141
1142 switch (dev->dev_type) {
1143 case PS3_DEVICE_TYPE_SB:
1144 r->region_ops = (USE_DYNAMIC_DMA)
1145 ? &ps3_dma_sb_region_ops
1146 : &ps3_dma_sb_region_linear_ops;
1147 break;
1148 case PS3_DEVICE_TYPE_IOC0:
1149 r->region_ops = &ps3_dma_ioc0_region_ops;
1150 break;
1151 default:
1152 BUG();
1153 return -EINVAL;
1154 }
1155 return 0;
1156}
1157EXPORT_SYMBOL(ps3_dma_region_init);
1158
1159int ps3_dma_region_create(struct ps3_dma_region *r)
1160{
1161 BUG_ON(!r);
1162 BUG_ON(!r->region_ops);
1163 BUG_ON(!r->region_ops->create);
1164 return r->region_ops->create(r);
1165}
1166EXPORT_SYMBOL(ps3_dma_region_create);
1167
1168int ps3_dma_region_free(struct ps3_dma_region *r)
1169{
1170 BUG_ON(!r);
1171 BUG_ON(!r->region_ops);
1172 BUG_ON(!r->region_ops->free);
1173 return r->region_ops->free(r);
1174}
1175EXPORT_SYMBOL(ps3_dma_region_free);
1176
1177int ps3_dma_map(struct ps3_dma_region *r, unsigned long virt_addr,
1178 unsigned long len, dma_addr_t *bus_addr,
1179 u64 iopte_flag)
1180{
1181 return r->region_ops->map(r, virt_addr, len, bus_addr, iopte_flag);
1182}
1183
1184int ps3_dma_unmap(struct ps3_dma_region *r, dma_addr_t bus_addr,
1185 unsigned long len)
1186{
1187 return r->region_ops->unmap(r, bus_addr, len);
1188}
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198void __init ps3_mm_init(void)
1199{
1200 int result;
1201
1202 DBG(" -> %s:%d\n", __func__, __LINE__);
1203
1204 result = ps3_repository_read_mm_info(&map.rm.base, &map.rm.size,
1205 &map.total);
1206
1207 if (result)
1208 panic("ps3_repository_read_mm_info() failed");
1209
1210 map.rm.offset = map.rm.base;
1211 map.vas_id = map.htab_size = 0;
1212
1213
1214
1215 BUG_ON(map.rm.base);
1216 BUG_ON(!map.rm.size);
1217
1218
1219
1220 if (ps3_mm_get_repository_highmem(&map.r1)) {
1221 result = ps3_mm_region_create(&map.r1, map.total - map.rm.size);
1222
1223 if (!result)
1224 ps3_mm_set_repository_highmem(&map.r1);
1225 }
1226
1227
1228 map.total = map.rm.size + map.r1.size;
1229
1230 if (!map.r1.size) {
1231 DBG("%s:%d: No highmem region found\n", __func__, __LINE__);
1232 } else {
1233 DBG("%s:%d: Adding highmem region: %llxh %llxh\n",
1234 __func__, __LINE__, map.rm.size,
1235 map.total - map.rm.size);
1236 memblock_add(map.rm.size, map.total - map.rm.size);
1237 }
1238
1239 DBG(" <- %s:%d\n", __func__, __LINE__);
1240}
1241
1242
1243
1244
1245
1246void ps3_mm_shutdown(void)
1247{
1248 ps3_mm_region_destroy(&map.r1);
1249}
1250