1
2
3
4
5
6
7
8
9
10#include <linux/hardirq.h>
11#include <linux/init.h>
12#include <linux/highmem.h>
13#include <linux/kernel.h>
14#include <linux/linkage.h>
15#include <linux/sched.h>
16#include <linux/smp.h>
17#include <linux/mm.h>
18#include <linux/module.h>
19#include <linux/bitops.h>
20
21#include <asm/bcache.h>
22#include <asm/bootinfo.h>
23#include <asm/cache.h>
24#include <asm/cacheops.h>
25#include <asm/cpu.h>
26#include <asm/cpu-features.h>
27#include <asm/io.h>
28#include <asm/page.h>
29#include <asm/pgtable.h>
30#include <asm/r4kcache.h>
31#include <asm/sections.h>
32#include <asm/system.h>
33#include <asm/mmu_context.h>
34#include <asm/war.h>
35#include <asm/cacheflush.h>
36
37
38
39
40
41
42
43
44
45
46static inline void r4k_on_each_cpu(void (*func) (void *info), void *info,
47 int wait)
48{
49 preempt_disable();
50
51#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
52 smp_call_function(func, info, wait);
53#endif
54 func(info);
55 preempt_enable();
56}
57
58#if defined(CONFIG_MIPS_CMP)
59#define cpu_has_safe_index_cacheops 0
60#else
61#define cpu_has_safe_index_cacheops 1
62#endif
63
64
65
66
67static unsigned long icache_size __read_mostly;
68static unsigned long dcache_size __read_mostly;
69static unsigned long scache_size __read_mostly;
70
71
72
73
74static void cache_noop(void) {}
75
76static struct bcache_ops no_sc_ops = {
77 .bc_enable = (void *)cache_noop,
78 .bc_disable = (void *)cache_noop,
79 .bc_wback_inv = (void *)cache_noop,
80 .bc_inv = (void *)cache_noop
81};
82
83struct bcache_ops *bcops = &no_sc_ops;
84
85#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
86#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
87
88#define R4600_HIT_CACHEOP_WAR_IMPL \
89do { \
90 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \
91 *(volatile unsigned long *)CKSEG1; \
92 if (R4600_V1_HIT_CACHEOP_WAR) \
93 __asm__ __volatile__("nop;nop;nop;nop"); \
94} while (0)
95
96static void (*r4k_blast_dcache_page)(unsigned long addr);
97
98static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
99{
100 R4600_HIT_CACHEOP_WAR_IMPL;
101 blast_dcache32_page(addr);
102}
103
104static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
105{
106 R4600_HIT_CACHEOP_WAR_IMPL;
107 blast_dcache64_page(addr);
108}
109
110static void __cpuinit r4k_blast_dcache_page_setup(void)
111{
112 unsigned long dc_lsize = cpu_dcache_line_size();
113
114 if (dc_lsize == 0)
115 r4k_blast_dcache_page = (void *)cache_noop;
116 else if (dc_lsize == 16)
117 r4k_blast_dcache_page = blast_dcache16_page;
118 else if (dc_lsize == 32)
119 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
120 else if (dc_lsize == 64)
121 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
122}
123
124static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
125
126static void __cpuinit r4k_blast_dcache_page_indexed_setup(void)
127{
128 unsigned long dc_lsize = cpu_dcache_line_size();
129
130 if (dc_lsize == 0)
131 r4k_blast_dcache_page_indexed = (void *)cache_noop;
132 else if (dc_lsize == 16)
133 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
134 else if (dc_lsize == 32)
135 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
136 else if (dc_lsize == 64)
137 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
138}
139
140static void (* r4k_blast_dcache)(void);
141
142static void __cpuinit r4k_blast_dcache_setup(void)
143{
144 unsigned long dc_lsize = cpu_dcache_line_size();
145
146 if (dc_lsize == 0)
147 r4k_blast_dcache = (void *)cache_noop;
148 else if (dc_lsize == 16)
149 r4k_blast_dcache = blast_dcache16;
150 else if (dc_lsize == 32)
151 r4k_blast_dcache = blast_dcache32;
152 else if (dc_lsize == 64)
153 r4k_blast_dcache = blast_dcache64;
154}
155
156
157#define JUMP_TO_ALIGN(order) \
158 __asm__ __volatile__( \
159 "b\t1f\n\t" \
160 ".align\t" #order "\n\t" \
161 "1:\n\t" \
162 )
163#define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10)
164#define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
165
166static inline void blast_r4600_v1_icache32(void)
167{
168 unsigned long flags;
169
170 local_irq_save(flags);
171 blast_icache32();
172 local_irq_restore(flags);
173}
174
175static inline void tx49_blast_icache32(void)
176{
177 unsigned long start = INDEX_BASE;
178 unsigned long end = start + current_cpu_data.icache.waysize;
179 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
180 unsigned long ws_end = current_cpu_data.icache.ways <<
181 current_cpu_data.icache.waybit;
182 unsigned long ws, addr;
183
184 CACHE32_UNROLL32_ALIGN2;
185
186 for (ws = 0; ws < ws_end; ws += ws_inc)
187 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
188 cache32_unroll32(addr|ws, Index_Invalidate_I);
189 CACHE32_UNROLL32_ALIGN;
190
191 for (ws = 0; ws < ws_end; ws += ws_inc)
192 for (addr = start; addr < end; addr += 0x400 * 2)
193 cache32_unroll32(addr|ws, Index_Invalidate_I);
194}
195
196static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
197{
198 unsigned long flags;
199
200 local_irq_save(flags);
201 blast_icache32_page_indexed(page);
202 local_irq_restore(flags);
203}
204
205static inline void tx49_blast_icache32_page_indexed(unsigned long page)
206{
207 unsigned long indexmask = current_cpu_data.icache.waysize - 1;
208 unsigned long start = INDEX_BASE + (page & indexmask);
209 unsigned long end = start + PAGE_SIZE;
210 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
211 unsigned long ws_end = current_cpu_data.icache.ways <<
212 current_cpu_data.icache.waybit;
213 unsigned long ws, addr;
214
215 CACHE32_UNROLL32_ALIGN2;
216
217 for (ws = 0; ws < ws_end; ws += ws_inc)
218 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
219 cache32_unroll32(addr|ws, Index_Invalidate_I);
220 CACHE32_UNROLL32_ALIGN;
221
222 for (ws = 0; ws < ws_end; ws += ws_inc)
223 for (addr = start; addr < end; addr += 0x400 * 2)
224 cache32_unroll32(addr|ws, Index_Invalidate_I);
225}
226
227static void (* r4k_blast_icache_page)(unsigned long addr);
228
229static void __cpuinit r4k_blast_icache_page_setup(void)
230{
231 unsigned long ic_lsize = cpu_icache_line_size();
232
233 if (ic_lsize == 0)
234 r4k_blast_icache_page = (void *)cache_noop;
235 else if (ic_lsize == 16)
236 r4k_blast_icache_page = blast_icache16_page;
237 else if (ic_lsize == 32)
238 r4k_blast_icache_page = blast_icache32_page;
239 else if (ic_lsize == 64)
240 r4k_blast_icache_page = blast_icache64_page;
241}
242
243
244static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
245
246static void __cpuinit r4k_blast_icache_page_indexed_setup(void)
247{
248 unsigned long ic_lsize = cpu_icache_line_size();
249
250 if (ic_lsize == 0)
251 r4k_blast_icache_page_indexed = (void *)cache_noop;
252 else if (ic_lsize == 16)
253 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
254 else if (ic_lsize == 32) {
255 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
256 r4k_blast_icache_page_indexed =
257 blast_icache32_r4600_v1_page_indexed;
258 else if (TX49XX_ICACHE_INDEX_INV_WAR)
259 r4k_blast_icache_page_indexed =
260 tx49_blast_icache32_page_indexed;
261 else
262 r4k_blast_icache_page_indexed =
263 blast_icache32_page_indexed;
264 } else if (ic_lsize == 64)
265 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
266}
267
268static void (* r4k_blast_icache)(void);
269
270static void __cpuinit r4k_blast_icache_setup(void)
271{
272 unsigned long ic_lsize = cpu_icache_line_size();
273
274 if (ic_lsize == 0)
275 r4k_blast_icache = (void *)cache_noop;
276 else if (ic_lsize == 16)
277 r4k_blast_icache = blast_icache16;
278 else if (ic_lsize == 32) {
279 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
280 r4k_blast_icache = blast_r4600_v1_icache32;
281 else if (TX49XX_ICACHE_INDEX_INV_WAR)
282 r4k_blast_icache = tx49_blast_icache32;
283 else
284 r4k_blast_icache = blast_icache32;
285 } else if (ic_lsize == 64)
286 r4k_blast_icache = blast_icache64;
287}
288
289static void (* r4k_blast_scache_page)(unsigned long addr);
290
291static void __cpuinit r4k_blast_scache_page_setup(void)
292{
293 unsigned long sc_lsize = cpu_scache_line_size();
294
295 if (scache_size == 0)
296 r4k_blast_scache_page = (void *)cache_noop;
297 else if (sc_lsize == 16)
298 r4k_blast_scache_page = blast_scache16_page;
299 else if (sc_lsize == 32)
300 r4k_blast_scache_page = blast_scache32_page;
301 else if (sc_lsize == 64)
302 r4k_blast_scache_page = blast_scache64_page;
303 else if (sc_lsize == 128)
304 r4k_blast_scache_page = blast_scache128_page;
305}
306
307static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
308
309static void __cpuinit r4k_blast_scache_page_indexed_setup(void)
310{
311 unsigned long sc_lsize = cpu_scache_line_size();
312
313 if (scache_size == 0)
314 r4k_blast_scache_page_indexed = (void *)cache_noop;
315 else if (sc_lsize == 16)
316 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
317 else if (sc_lsize == 32)
318 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
319 else if (sc_lsize == 64)
320 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
321 else if (sc_lsize == 128)
322 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
323}
324
325static void (* r4k_blast_scache)(void);
326
327static void __cpuinit r4k_blast_scache_setup(void)
328{
329 unsigned long sc_lsize = cpu_scache_line_size();
330
331 if (scache_size == 0)
332 r4k_blast_scache = (void *)cache_noop;
333 else if (sc_lsize == 16)
334 r4k_blast_scache = blast_scache16;
335 else if (sc_lsize == 32)
336 r4k_blast_scache = blast_scache32;
337 else if (sc_lsize == 64)
338 r4k_blast_scache = blast_scache64;
339 else if (sc_lsize == 128)
340 r4k_blast_scache = blast_scache128;
341}
342
343static inline void local_r4k___flush_cache_all(void * args)
344{
345#if defined(CONFIG_CPU_LOONGSON2)
346 r4k_blast_scache();
347 return;
348#endif
349 r4k_blast_dcache();
350 r4k_blast_icache();
351
352 switch (current_cpu_type()) {
353 case CPU_R4000SC:
354 case CPU_R4000MC:
355 case CPU_R4400SC:
356 case CPU_R4400MC:
357 case CPU_R10000:
358 case CPU_R12000:
359 case CPU_R14000:
360 r4k_blast_scache();
361 }
362}
363
364static void r4k___flush_cache_all(void)
365{
366 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1);
367}
368
369static inline int has_valid_asid(const struct mm_struct *mm)
370{
371#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC)
372 int i;
373
374 for_each_online_cpu(i)
375 if (cpu_context(i, mm))
376 return 1;
377
378 return 0;
379#else
380 return cpu_context(smp_processor_id(), mm);
381#endif
382}
383
384static void r4k__flush_cache_vmap(void)
385{
386 r4k_blast_dcache();
387}
388
389static void r4k__flush_cache_vunmap(void)
390{
391 r4k_blast_dcache();
392}
393
394static inline void local_r4k_flush_cache_range(void * args)
395{
396 struct vm_area_struct *vma = args;
397 int exec = vma->vm_flags & VM_EXEC;
398
399 if (!(has_valid_asid(vma->vm_mm)))
400 return;
401
402 r4k_blast_dcache();
403 if (exec)
404 r4k_blast_icache();
405}
406
407static void r4k_flush_cache_range(struct vm_area_struct *vma,
408 unsigned long start, unsigned long end)
409{
410 int exec = vma->vm_flags & VM_EXEC;
411
412 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
413 r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1);
414}
415
416static inline void local_r4k_flush_cache_mm(void * args)
417{
418 struct mm_struct *mm = args;
419
420 if (!has_valid_asid(mm))
421 return;
422
423
424
425
426
427
428
429 if (current_cpu_type() == CPU_R4000SC ||
430 current_cpu_type() == CPU_R4000MC ||
431 current_cpu_type() == CPU_R4400SC ||
432 current_cpu_type() == CPU_R4400MC) {
433 r4k_blast_scache();
434 return;
435 }
436
437 r4k_blast_dcache();
438}
439
440static void r4k_flush_cache_mm(struct mm_struct *mm)
441{
442 if (!cpu_has_dc_aliases)
443 return;
444
445 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1);
446}
447
448struct flush_cache_page_args {
449 struct vm_area_struct *vma;
450 unsigned long addr;
451 unsigned long pfn;
452};
453
454static inline void local_r4k_flush_cache_page(void *args)
455{
456 struct flush_cache_page_args *fcp_args = args;
457 struct vm_area_struct *vma = fcp_args->vma;
458 unsigned long addr = fcp_args->addr;
459 struct page *page = pfn_to_page(fcp_args->pfn);
460 int exec = vma->vm_flags & VM_EXEC;
461 struct mm_struct *mm = vma->vm_mm;
462 int map_coherent = 0;
463 pgd_t *pgdp;
464 pud_t *pudp;
465 pmd_t *pmdp;
466 pte_t *ptep;
467 void *vaddr;
468
469
470
471
472
473 if (!has_valid_asid(mm))
474 return;
475
476 addr &= PAGE_MASK;
477 pgdp = pgd_offset(mm, addr);
478 pudp = pud_offset(pgdp, addr);
479 pmdp = pmd_offset(pudp, addr);
480 ptep = pte_offset(pmdp, addr);
481
482
483
484
485
486 if (!(pte_present(*ptep)))
487 return;
488
489 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
490 vaddr = NULL;
491 else {
492
493
494
495
496 map_coherent = (cpu_has_dc_aliases &&
497 page_mapped(page) && !Page_dcache_dirty(page));
498 if (map_coherent)
499 vaddr = kmap_coherent(page, addr);
500 else
501 vaddr = kmap_atomic(page, KM_USER0);
502 addr = (unsigned long)vaddr;
503 }
504
505 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
506 r4k_blast_dcache_page(addr);
507 if (exec && !cpu_icache_snoops_remote_store)
508 r4k_blast_scache_page(addr);
509 }
510 if (exec) {
511 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
512 int cpu = smp_processor_id();
513
514 if (cpu_context(cpu, mm) != 0)
515 drop_mmu_context(mm, cpu);
516 } else
517 r4k_blast_icache_page(addr);
518 }
519
520 if (vaddr) {
521 if (map_coherent)
522 kunmap_coherent();
523 else
524 kunmap_atomic(vaddr, KM_USER0);
525 }
526}
527
528static void r4k_flush_cache_page(struct vm_area_struct *vma,
529 unsigned long addr, unsigned long pfn)
530{
531 struct flush_cache_page_args args;
532
533 args.vma = vma;
534 args.addr = addr;
535 args.pfn = pfn;
536
537 r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1);
538}
539
540static inline void local_r4k_flush_data_cache_page(void * addr)
541{
542 r4k_blast_dcache_page((unsigned long) addr);
543}
544
545static void r4k_flush_data_cache_page(unsigned long addr)
546{
547 if (in_atomic())
548 local_r4k_flush_data_cache_page((void *)addr);
549 else
550 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr,
551 1);
552}
553
554struct flush_icache_range_args {
555 unsigned long start;
556 unsigned long end;
557};
558
559static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end)
560{
561 if (!cpu_has_ic_fills_f_dc) {
562 if (end - start >= dcache_size) {
563 r4k_blast_dcache();
564 } else {
565 R4600_HIT_CACHEOP_WAR_IMPL;
566 protected_blast_dcache_range(start, end);
567 }
568 }
569
570 if (end - start > icache_size)
571 r4k_blast_icache();
572 else
573 protected_blast_icache_range(start, end);
574}
575
576static inline void local_r4k_flush_icache_range_ipi(void *args)
577{
578 struct flush_icache_range_args *fir_args = args;
579 unsigned long start = fir_args->start;
580 unsigned long end = fir_args->end;
581
582 local_r4k_flush_icache_range(start, end);
583}
584
585static void r4k_flush_icache_range(unsigned long start, unsigned long end)
586{
587 struct flush_icache_range_args args;
588
589 args.start = start;
590 args.end = end;
591
592 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args, 1);
593 instruction_hazard();
594}
595
596#ifdef CONFIG_DMA_NONCOHERENT
597
598static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
599{
600
601 BUG_ON(size == 0);
602
603 if (cpu_has_inclusive_pcaches) {
604 if (size >= scache_size)
605 r4k_blast_scache();
606 else
607 blast_scache_range(addr, addr + size);
608 return;
609 }
610
611
612
613
614
615
616 if (cpu_has_safe_index_cacheops && size >= dcache_size) {
617 r4k_blast_dcache();
618 } else {
619 R4600_HIT_CACHEOP_WAR_IMPL;
620 blast_dcache_range(addr, addr + size);
621 }
622
623 bc_wback_inv(addr, size);
624}
625
626static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
627{
628
629 BUG_ON(size == 0);
630
631 if (cpu_has_inclusive_pcaches) {
632 if (size >= scache_size)
633 r4k_blast_scache();
634 else {
635 unsigned long lsize = cpu_scache_line_size();
636 unsigned long almask = ~(lsize - 1);
637
638
639
640
641
642
643
644
645
646 cache_op(Hit_Writeback_Inv_SD, addr & almask);
647 cache_op(Hit_Writeback_Inv_SD,
648 (addr + size - 1) & almask);
649 blast_inv_scache_range(addr, addr + size);
650 }
651 return;
652 }
653
654 if (cpu_has_safe_index_cacheops && size >= dcache_size) {
655 r4k_blast_dcache();
656 } else {
657 unsigned long lsize = cpu_dcache_line_size();
658 unsigned long almask = ~(lsize - 1);
659
660 R4600_HIT_CACHEOP_WAR_IMPL;
661 cache_op(Hit_Writeback_Inv_D, addr & almask);
662 cache_op(Hit_Writeback_Inv_D, (addr + size - 1) & almask);
663 blast_inv_dcache_range(addr, addr + size);
664 }
665
666 bc_inv(addr, size);
667}
668#endif
669
670
671
672
673
674
675static void local_r4k_flush_cache_sigtramp(void * arg)
676{
677 unsigned long ic_lsize = cpu_icache_line_size();
678 unsigned long dc_lsize = cpu_dcache_line_size();
679 unsigned long sc_lsize = cpu_scache_line_size();
680 unsigned long addr = (unsigned long) arg;
681
682 R4600_HIT_CACHEOP_WAR_IMPL;
683 if (dc_lsize)
684 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
685 if (!cpu_icache_snoops_remote_store && scache_size)
686 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
687 if (ic_lsize)
688 protected_flush_icache_line(addr & ~(ic_lsize - 1));
689 if (MIPS4K_ICACHE_REFILL_WAR) {
690 __asm__ __volatile__ (
691 ".set push\n\t"
692 ".set noat\n\t"
693 ".set mips3\n\t"
694#ifdef CONFIG_32BIT
695 "la $at,1f\n\t"
696#endif
697#ifdef CONFIG_64BIT
698 "dla $at,1f\n\t"
699#endif
700 "cache %0,($at)\n\t"
701 "nop; nop; nop\n"
702 "1:\n\t"
703 ".set pop"
704 :
705 : "i" (Hit_Invalidate_I));
706 }
707 if (MIPS_CACHE_SYNC_WAR)
708 __asm__ __volatile__ ("sync");
709}
710
711static void r4k_flush_cache_sigtramp(unsigned long addr)
712{
713 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1);
714}
715
716static void r4k_flush_icache_all(void)
717{
718 if (cpu_has_vtag_icache)
719 r4k_blast_icache();
720}
721
722static inline void rm7k_erratum31(void)
723{
724 const unsigned long ic_lsize = 32;
725 unsigned long addr;
726
727
728 write_c0_taglo(0);
729 write_c0_taghi(0);
730
731 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
732 __asm__ __volatile__ (
733 ".set push\n\t"
734 ".set noreorder\n\t"
735 ".set mips3\n\t"
736 "cache\t%1, 0(%0)\n\t"
737 "cache\t%1, 0x1000(%0)\n\t"
738 "cache\t%1, 0x2000(%0)\n\t"
739 "cache\t%1, 0x3000(%0)\n\t"
740 "cache\t%2, 0(%0)\n\t"
741 "cache\t%2, 0x1000(%0)\n\t"
742 "cache\t%2, 0x2000(%0)\n\t"
743 "cache\t%2, 0x3000(%0)\n\t"
744 "cache\t%1, 0(%0)\n\t"
745 "cache\t%1, 0x1000(%0)\n\t"
746 "cache\t%1, 0x2000(%0)\n\t"
747 "cache\t%1, 0x3000(%0)\n\t"
748 ".set pop\n"
749 :
750 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
751 }
752}
753
754static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way",
755 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
756};
757
758static void __cpuinit probe_pcache(void)
759{
760 struct cpuinfo_mips *c = ¤t_cpu_data;
761 unsigned int config = read_c0_config();
762 unsigned int prid = read_c0_prid();
763 unsigned long config1;
764 unsigned int lsize;
765
766 switch (c->cputype) {
767 case CPU_R4600:
768 case CPU_R4700:
769 case CPU_R5000:
770 case CPU_NEVADA:
771 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
772 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
773 c->icache.ways = 2;
774 c->icache.waybit = __ffs(icache_size/2);
775
776 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
777 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
778 c->dcache.ways = 2;
779 c->dcache.waybit= __ffs(dcache_size/2);
780
781 c->options |= MIPS_CPU_CACHE_CDEX_P;
782 break;
783
784 case CPU_R5432:
785 case CPU_R5500:
786 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
787 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
788 c->icache.ways = 2;
789 c->icache.waybit= 0;
790
791 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
792 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
793 c->dcache.ways = 2;
794 c->dcache.waybit = 0;
795
796 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH;
797 break;
798
799 case CPU_TX49XX:
800 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
801 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
802 c->icache.ways = 4;
803 c->icache.waybit= 0;
804
805 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
806 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
807 c->dcache.ways = 4;
808 c->dcache.waybit = 0;
809
810 c->options |= MIPS_CPU_CACHE_CDEX_P;
811 c->options |= MIPS_CPU_PREFETCH;
812 break;
813
814 case CPU_R4000PC:
815 case CPU_R4000SC:
816 case CPU_R4000MC:
817 case CPU_R4400PC:
818 case CPU_R4400SC:
819 case CPU_R4400MC:
820 case CPU_R4300:
821 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
822 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
823 c->icache.ways = 1;
824 c->icache.waybit = 0;
825
826 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
827 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
828 c->dcache.ways = 1;
829 c->dcache.waybit = 0;
830
831 c->options |= MIPS_CPU_CACHE_CDEX_P;
832 break;
833
834 case CPU_R10000:
835 case CPU_R12000:
836 case CPU_R14000:
837 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
838 c->icache.linesz = 64;
839 c->icache.ways = 2;
840 c->icache.waybit = 0;
841
842 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
843 c->dcache.linesz = 32;
844 c->dcache.ways = 2;
845 c->dcache.waybit = 0;
846
847 c->options |= MIPS_CPU_PREFETCH;
848 break;
849
850 case CPU_VR4133:
851 write_c0_config(config & ~VR41_CONF_P4K);
852 case CPU_VR4131:
853
854 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
855 c->processor_id == 0x0c82U) {
856 config |= 0x00400000U;
857 if (c->processor_id == 0x0c80U)
858 config |= VR41_CONF_BP;
859 write_c0_config(config);
860 } else
861 c->options |= MIPS_CPU_CACHE_CDEX_P;
862
863 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
864 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
865 c->icache.ways = 2;
866 c->icache.waybit = __ffs(icache_size/2);
867
868 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
869 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
870 c->dcache.ways = 2;
871 c->dcache.waybit = __ffs(dcache_size/2);
872 break;
873
874 case CPU_VR41XX:
875 case CPU_VR4111:
876 case CPU_VR4121:
877 case CPU_VR4122:
878 case CPU_VR4181:
879 case CPU_VR4181A:
880 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
881 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
882 c->icache.ways = 1;
883 c->icache.waybit = 0;
884
885 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
886 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
887 c->dcache.ways = 1;
888 c->dcache.waybit = 0;
889
890 c->options |= MIPS_CPU_CACHE_CDEX_P;
891 break;
892
893 case CPU_RM7000:
894 rm7k_erratum31();
895
896 case CPU_RM9000:
897 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
898 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
899 c->icache.ways = 4;
900 c->icache.waybit = __ffs(icache_size / c->icache.ways);
901
902 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
903 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
904 c->dcache.ways = 4;
905 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
906
907#if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR)
908 c->options |= MIPS_CPU_CACHE_CDEX_P;
909#endif
910 c->options |= MIPS_CPU_PREFETCH;
911 break;
912
913 case CPU_LOONGSON2:
914 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
915 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
916 if (prid & 0x3)
917 c->icache.ways = 4;
918 else
919 c->icache.ways = 2;
920 c->icache.waybit = 0;
921
922 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
923 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
924 if (prid & 0x3)
925 c->dcache.ways = 4;
926 else
927 c->dcache.ways = 2;
928 c->dcache.waybit = 0;
929 break;
930
931 default:
932 if (!(config & MIPS_CONF_M))
933 panic("Don't know how to probe P-caches on this cpu.");
934
935
936
937
938
939 config1 = read_c0_config1();
940
941 if ((lsize = ((config1 >> 19) & 7)))
942 c->icache.linesz = 2 << lsize;
943 else
944 c->icache.linesz = lsize;
945 c->icache.sets = 64 << ((config1 >> 22) & 7);
946 c->icache.ways = 1 + ((config1 >> 16) & 7);
947
948 icache_size = c->icache.sets *
949 c->icache.ways *
950 c->icache.linesz;
951 c->icache.waybit = __ffs(icache_size/c->icache.ways);
952
953 if (config & 0x8)
954 c->icache.flags |= MIPS_CACHE_VTAG;
955
956
957
958
959 c->dcache.flags = 0;
960
961 if ((lsize = ((config1 >> 10) & 7)))
962 c->dcache.linesz = 2 << lsize;
963 else
964 c->dcache.linesz= lsize;
965 c->dcache.sets = 64 << ((config1 >> 13) & 7);
966 c->dcache.ways = 1 + ((config1 >> 7) & 7);
967
968 dcache_size = c->dcache.sets *
969 c->dcache.ways *
970 c->dcache.linesz;
971 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
972
973 c->options |= MIPS_CPU_PREFETCH;
974 break;
975 }
976
977
978
979
980
981
982
983
984
985 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 &&
986 !(config & CONF_SC) && c->icache.linesz != 16 &&
987 PAGE_SIZE <= 0x8000)
988 panic("Improper R4000SC processor configuration detected");
989
990
991 c->icache.waysize = icache_size / c->icache.ways;
992 c->dcache.waysize = dcache_size / c->dcache.ways;
993
994 c->icache.sets = c->icache.linesz ?
995 icache_size / (c->icache.linesz * c->icache.ways) : 0;
996 c->dcache.sets = c->dcache.linesz ?
997 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
998
999
1000
1001
1002
1003
1004
1005 switch (c->cputype) {
1006 case CPU_20KC:
1007 case CPU_25KF:
1008 case CPU_SB1:
1009 case CPU_SB1A:
1010 c->dcache.flags |= MIPS_CACHE_PINDEX;
1011 break;
1012
1013 case CPU_R10000:
1014 case CPU_R12000:
1015 case CPU_R14000:
1016 break;
1017
1018 case CPU_24K:
1019 case CPU_34K:
1020 case CPU_74K:
1021 case CPU_1004K:
1022 if ((read_c0_config7() & (1 << 16))) {
1023
1024
1025 c->dcache.flags |= MIPS_CACHE_PINDEX;
1026 break;
1027 }
1028 default:
1029 if (c->dcache.waysize > PAGE_SIZE)
1030 c->dcache.flags |= MIPS_CACHE_ALIASES;
1031 }
1032
1033 switch (c->cputype) {
1034 case CPU_20KC:
1035
1036
1037
1038
1039 c->icache.flags |= MIPS_CACHE_VTAG;
1040 break;
1041
1042 case CPU_ALCHEMY:
1043 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1044 break;
1045 }
1046
1047#ifdef CONFIG_CPU_LOONGSON2
1048
1049
1050
1051
1052 c->icache.ways = 1;
1053#endif
1054
1055 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1056 icache_size >> 10,
1057 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
1058 way_string[c->icache.ways], c->icache.linesz);
1059
1060 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
1061 dcache_size >> 10, way_string[c->dcache.ways],
1062 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
1063 (c->dcache.flags & MIPS_CACHE_ALIASES) ?
1064 "cache aliases" : "no aliases",
1065 c->dcache.linesz);
1066}
1067
1068
1069
1070
1071
1072
1073
1074static int __cpuinit probe_scache(void)
1075{
1076 unsigned long flags, addr, begin, end, pow2;
1077 unsigned int config = read_c0_config();
1078 struct cpuinfo_mips *c = ¤t_cpu_data;
1079 int tmp;
1080
1081 if (config & CONF_SC)
1082 return 0;
1083
1084 begin = (unsigned long) &_stext;
1085 begin &= ~((4 * 1024 * 1024) - 1);
1086 end = begin + (4 * 1024 * 1024);
1087
1088
1089
1090
1091
1092 local_irq_save(flags);
1093
1094
1095 pow2 = (64 * 1024);
1096 for (addr = begin; addr < end; addr = (begin + pow2)) {
1097 unsigned long *p = (unsigned long *) addr;
1098 __asm__ __volatile__("nop" : : "r" (*p));
1099 pow2 <<= 1;
1100 }
1101
1102
1103 write_c0_taglo(0);
1104 write_c0_taghi(0);
1105 __asm__ __volatile__("nop; nop; nop; nop;");
1106 cache_op(Index_Store_Tag_I, begin);
1107 cache_op(Index_Store_Tag_D, begin);
1108 cache_op(Index_Store_Tag_SD, begin);
1109
1110
1111 pow2 = (128 * 1024);
1112 tmp = 0;
1113 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1114 cache_op(Index_Load_Tag_SD, addr);
1115 __asm__ __volatile__("nop; nop; nop; nop;");
1116 if (!read_c0_taglo())
1117 break;
1118 pow2 <<= 1;
1119 }
1120 local_irq_restore(flags);
1121 addr -= begin;
1122
1123 scache_size = addr;
1124 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1125 c->scache.ways = 1;
1126 c->dcache.waybit = 0;
1127
1128 return 1;
1129}
1130
1131#if defined(CONFIG_CPU_LOONGSON2)
1132static void __init loongson2_sc_init(void)
1133{
1134 struct cpuinfo_mips *c = ¤t_cpu_data;
1135
1136 scache_size = 512*1024;
1137 c->scache.linesz = 32;
1138 c->scache.ways = 4;
1139 c->scache.waybit = 0;
1140 c->scache.waysize = scache_size / (c->scache.ways);
1141 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1142 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1143 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1144
1145 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1146}
1147#endif
1148
1149extern int r5k_sc_init(void);
1150extern int rm7k_sc_init(void);
1151extern int mips_sc_init(void);
1152
1153static void __cpuinit setup_scache(void)
1154{
1155 struct cpuinfo_mips *c = ¤t_cpu_data;
1156 unsigned int config = read_c0_config();
1157 int sc_present = 0;
1158
1159
1160
1161
1162
1163
1164 switch (c->cputype) {
1165 case CPU_R4000SC:
1166 case CPU_R4000MC:
1167 case CPU_R4400SC:
1168 case CPU_R4400MC:
1169 sc_present = run_uncached(probe_scache);
1170 if (sc_present)
1171 c->options |= MIPS_CPU_CACHE_CDEX_S;
1172 break;
1173
1174 case CPU_R10000:
1175 case CPU_R12000:
1176 case CPU_R14000:
1177 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1178 c->scache.linesz = 64 << ((config >> 13) & 1);
1179 c->scache.ways = 2;
1180 c->scache.waybit= 0;
1181 sc_present = 1;
1182 break;
1183
1184 case CPU_R5000:
1185 case CPU_NEVADA:
1186#ifdef CONFIG_R5000_CPU_SCACHE
1187 r5k_sc_init();
1188#endif
1189 return;
1190
1191 case CPU_RM7000:
1192 case CPU_RM9000:
1193#ifdef CONFIG_RM7000_CPU_SCACHE
1194 rm7k_sc_init();
1195#endif
1196 return;
1197
1198#if defined(CONFIG_CPU_LOONGSON2)
1199 case CPU_LOONGSON2:
1200 loongson2_sc_init();
1201 return;
1202#endif
1203
1204 default:
1205 if (c->isa_level == MIPS_CPU_ISA_M32R1 ||
1206 c->isa_level == MIPS_CPU_ISA_M32R2 ||
1207 c->isa_level == MIPS_CPU_ISA_M64R1 ||
1208 c->isa_level == MIPS_CPU_ISA_M64R2) {
1209#ifdef CONFIG_MIPS_CPU_SCACHE
1210 if (mips_sc_init ()) {
1211 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1212 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1213 scache_size >> 10,
1214 way_string[c->scache.ways], c->scache.linesz);
1215 }
1216#else
1217 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1218 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1219#endif
1220 return;
1221 }
1222 sc_present = 0;
1223 }
1224
1225 if (!sc_present)
1226 return;
1227
1228
1229 c->scache.waysize = scache_size / c->scache.ways;
1230
1231 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1232
1233 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1234 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1235
1236 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1237}
1238
1239void au1x00_fixup_config_od(void)
1240{
1241
1242
1243
1244
1245
1246 switch (read_c0_prid()) {
1247 case 0x00030100:
1248 case 0x00030201:
1249 case 0x00030202:
1250 case 0x01030200:
1251
1252
1253
1254
1255
1256 case 0x02030200:
1257 case 0x02030201:
1258 case 0x02030202:
1259 set_c0_config(1 << 19);
1260 break;
1261 }
1262}
1263
1264
1265#define NXP_BARRIER() \
1266 __asm__ __volatile__( \
1267 ".set noreorder\n\t" \
1268 "nop; nop; nop; nop; nop; nop;\n\t" \
1269 ".set reorder\n\t")
1270
1271static void nxp_pr4450_fixup_config(void)
1272{
1273 unsigned long config0;
1274
1275 config0 = read_c0_config();
1276
1277
1278 config0 &= ~(0x7 | (7 << 25) | (7 << 28));
1279 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) |
1280 ((_page_cachable_default >> _CACHE_SHIFT) << 25) |
1281 ((_page_cachable_default >> _CACHE_SHIFT) << 28));
1282 write_c0_config(config0);
1283 NXP_BARRIER();
1284}
1285
1286static int __cpuinitdata cca = -1;
1287
1288static int __init cca_setup(char *str)
1289{
1290 get_option(&str, &cca);
1291
1292 return 1;
1293}
1294
1295__setup("cca=", cca_setup);
1296
1297static void __cpuinit coherency_setup(void)
1298{
1299 if (cca < 0 || cca > 7)
1300 cca = read_c0_config() & CONF_CM_CMASK;
1301 _page_cachable_default = cca << _CACHE_SHIFT;
1302
1303 pr_debug("Using cache attribute %d\n", cca);
1304 change_c0_config(CONF_CM_CMASK, cca);
1305
1306
1307
1308
1309
1310
1311
1312
1313 switch (current_cpu_type()) {
1314 case CPU_R4000PC:
1315 case CPU_R4000SC:
1316 case CPU_R4000MC:
1317 case CPU_R4400PC:
1318 case CPU_R4400SC:
1319 case CPU_R4400MC:
1320 clear_c0_config(CONF_CU);
1321 break;
1322
1323
1324
1325
1326
1327 case CPU_ALCHEMY:
1328 au1x00_fixup_config_od();
1329 break;
1330
1331 case PRID_IMP_PR4450:
1332 nxp_pr4450_fixup_config();
1333 break;
1334 }
1335}
1336
1337#if defined(CONFIG_DMA_NONCOHERENT)
1338
1339static int __cpuinitdata coherentio;
1340
1341static int __init setcoherentio(char *str)
1342{
1343 coherentio = 1;
1344
1345 return 1;
1346}
1347
1348__setup("coherentio", setcoherentio);
1349#endif
1350
1351void __cpuinit r4k_cache_init(void)
1352{
1353 extern void build_clear_page(void);
1354 extern void build_copy_page(void);
1355 extern char __weak except_vec2_generic;
1356 extern char __weak except_vec2_sb1;
1357 struct cpuinfo_mips *c = ¤t_cpu_data;
1358
1359 switch (c->cputype) {
1360 case CPU_SB1:
1361 case CPU_SB1A:
1362 set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
1363 break;
1364
1365 default:
1366 set_uncached_handler(0x100, &except_vec2_generic, 0x80);
1367 break;
1368 }
1369
1370 probe_pcache();
1371 setup_scache();
1372
1373 r4k_blast_dcache_page_setup();
1374 r4k_blast_dcache_page_indexed_setup();
1375 r4k_blast_dcache_setup();
1376 r4k_blast_icache_page_setup();
1377 r4k_blast_icache_page_indexed_setup();
1378 r4k_blast_icache_setup();
1379 r4k_blast_scache_page_setup();
1380 r4k_blast_scache_page_indexed_setup();
1381 r4k_blast_scache_setup();
1382
1383
1384
1385
1386
1387
1388 if (c->dcache.linesz)
1389 shm_align_mask = max_t( unsigned long,
1390 c->dcache.sets * c->dcache.linesz - 1,
1391 PAGE_SIZE - 1);
1392 else
1393 shm_align_mask = PAGE_SIZE-1;
1394
1395 __flush_cache_vmap = r4k__flush_cache_vmap;
1396 __flush_cache_vunmap = r4k__flush_cache_vunmap;
1397
1398 flush_cache_all = cache_noop;
1399 __flush_cache_all = r4k___flush_cache_all;
1400 flush_cache_mm = r4k_flush_cache_mm;
1401 flush_cache_page = r4k_flush_cache_page;
1402 flush_cache_range = r4k_flush_cache_range;
1403
1404 flush_cache_sigtramp = r4k_flush_cache_sigtramp;
1405 flush_icache_all = r4k_flush_icache_all;
1406 local_flush_data_cache_page = local_r4k_flush_data_cache_page;
1407 flush_data_cache_page = r4k_flush_data_cache_page;
1408 flush_icache_range = r4k_flush_icache_range;
1409 local_flush_icache_range = local_r4k_flush_icache_range;
1410
1411#if defined(CONFIG_DMA_NONCOHERENT)
1412 if (coherentio) {
1413 _dma_cache_wback_inv = (void *)cache_noop;
1414 _dma_cache_wback = (void *)cache_noop;
1415 _dma_cache_inv = (void *)cache_noop;
1416 } else {
1417 _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
1418 _dma_cache_wback = r4k_dma_cache_wback_inv;
1419 _dma_cache_inv = r4k_dma_cache_inv;
1420 }
1421#endif
1422
1423 build_clear_page();
1424 build_copy_page();
1425#if !defined(CONFIG_MIPS_CMP)
1426 local_r4k___flush_cache_all(NULL);
1427#endif
1428 coherency_setup();
1429}
1430