1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/clockchips.h>
21#include <linux/kernel.h>
22#include <linux/sched.h>
23#include <linux/smp.h>
24#include <linux/cpumask.h>
25#include <linux/interrupt.h>
26#include <linux/kernel_stat.h>
27#include <linux/module.h>
28#include <linux/ftrace.h>
29#include <linux/slab.h>
30
31#include <asm/cpu.h>
32#include <asm/processor.h>
33#include <linux/atomic.h>
34#include <asm/hardirq.h>
35#include <asm/hazards.h>
36#include <asm/irq.h>
37#include <asm/idle.h>
38#include <asm/mmu_context.h>
39#include <asm/mipsregs.h>
40#include <asm/cacheflush.h>
41#include <asm/time.h>
42#include <asm/addrspace.h>
43#include <asm/smtc.h>
44#include <asm/smtc_proc.h>
45#include <asm/setup.h>
46
47
48
49
50
51
52unsigned long irq_hwmask[NR_IRQS];
53
54#define LOCK_MT_PRA() \
55 local_irq_save(flags); \
56 mtflags = dmt()
57
58#define UNLOCK_MT_PRA() \
59 emt(mtflags); \
60 local_irq_restore(flags)
61
62#define LOCK_CORE_PRA() \
63 local_irq_save(flags); \
64 mtflags = dvpe()
65
66#define UNLOCK_CORE_PRA() \
67 evpe(mtflags); \
68 local_irq_restore(flags)
69
70
71
72
73
74
75
76
77
78
79asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS];
80
81
82
83
84
85#define IPIBUF_PER_CPU 4
86
87struct smtc_ipi_q IPIQ[NR_CPUS];
88static struct smtc_ipi_q freeIPIq;
89
90
91
92
93
94
95static int smtc_nconf1[MAX_SMTC_VPES];
96
97
98
99
100void ipi_decode(struct smtc_ipi *);
101static void post_direct_ipi(int cpu, struct smtc_ipi *pipi);
102static void setup_cross_vpe_interrupts(unsigned int nvpe);
103void init_smtc_stats(void);
104
105
106
107unsigned int smtc_status;
108
109
110
111static int vpe0limit;
112static int ipibuffers;
113static int nostlb;
114static int asidmask;
115unsigned long smtc_asid_mask = 0xff;
116
117static int __init vpe0tcs(char *str)
118{
119 get_option(&str, &vpe0limit);
120
121 return 1;
122}
123
124static int __init ipibufs(char *str)
125{
126 get_option(&str, &ipibuffers);
127 return 1;
128}
129
130static int __init stlb_disable(char *s)
131{
132 nostlb = 1;
133 return 1;
134}
135
136static int __init asidmask_set(char *str)
137{
138 get_option(&str, &asidmask);
139 switch (asidmask) {
140 case 0x1:
141 case 0x3:
142 case 0x7:
143 case 0xf:
144 case 0x1f:
145 case 0x3f:
146 case 0x7f:
147 case 0xff:
148 smtc_asid_mask = (unsigned long)asidmask;
149 break;
150 default:
151 printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask);
152 }
153 return 1;
154}
155
156__setup("vpe0tcs=", vpe0tcs);
157__setup("ipibufs=", ipibufs);
158__setup("nostlb", stlb_disable);
159__setup("asidmask=", asidmask_set);
160
161#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
162
163static int hang_trig;
164
165static int __init hangtrig_enable(char *s)
166{
167 hang_trig = 1;
168 return 1;
169}
170
171
172__setup("hangtrig", hangtrig_enable);
173
174#define DEFAULT_BLOCKED_IPI_LIMIT 32
175
176static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT;
177
178static int __init tintq(char *str)
179{
180 get_option(&str, &timerq_limit);
181 return 1;
182}
183
184__setup("tintq=", tintq);
185
186static int imstuckcount[MAX_SMTC_VPES][8];
187
188static int vpemask[MAX_SMTC_VPES][8] = {
189 {0, 0, 1, 0, 0, 0, 0, 1},
190 {0, 0, 0, 0, 0, 0, 0, 1}
191};
192int tcnoprog[NR_CPUS];
193static atomic_t idle_hook_initialized = ATOMIC_INIT(0);
194static int clock_hang_reported[NR_CPUS];
195
196#endif
197
198
199
200
201
202static void smtc_configure_tlb(void)
203{
204 int i, tlbsiz, vpes;
205 unsigned long mvpconf0;
206 unsigned long config1val;
207
208
209 for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) {
210 for(i = 0; i < MAX_SMTC_ASIDS; i++) {
211 smtc_live_asid[vpes][i] = 0;
212 }
213 }
214 mvpconf0 = read_c0_mvpconf0();
215
216 if ((vpes = ((mvpconf0 & MVPCONF0_PVPE)
217 >> MVPCONF0_PVPE_SHIFT) + 1) > 1) {
218
219 if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) {
220
221
222
223
224
225
226 if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE)
227 >> MVPCONF0_PTLBE_SHIFT)) == 0) {
228
229
230
231
232
233
234 settc(1);
235
236 write_tc_c0_tchalt(TCHALT_H);
237 mips_ihb();
238
239 for (i=0; i < vpes; i++) {
240 write_tc_c0_tcbind(i);
241
242
243
244
245
246 write_c0_mvpcontrol(
247 read_c0_mvpcontrol() & ~ MVPCONTROL_VPC );
248 mips_ihb();
249
250
251
252 if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) {
253 config1val = read_vpe_c0_config1();
254 tlbsiz += ((config1val >> 25) & 0x3f) + 1;
255 }
256
257
258 write_c0_mvpcontrol(
259 read_c0_mvpcontrol() | MVPCONTROL_VPC );
260 mips_ihb();
261 }
262 }
263 write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB);
264 ehb();
265
266
267
268
269
270
271
272
273
274 if (tlbsiz > 64)
275 tlbsiz = 64;
276 cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz;
277 smtc_status |= SMTC_TLB_SHARED;
278 local_flush_tlb_all();
279
280 printk("TLB of %d entry pairs shared by %d VPEs\n",
281 tlbsiz, vpes);
282 } else {
283 printk("WARNING: TLB Not Sharable on SMTC Boot!\n");
284 }
285 }
286}
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306int __init smtc_build_cpu_map(int start_cpu_slot)
307{
308 int i, ntcs;
309
310
311
312
313
314
315 ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
316 for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) {
317 set_cpu_possible(i, true);
318 __cpu_number_map[i] = i;
319 __cpu_logical_map[i] = i;
320 }
321#ifdef CONFIG_MIPS_MT_FPAFF
322
323 cpus_clear(mt_fpu_cpumask);
324#endif
325
326
327 printk("%i available secondary CPU TC(s)\n", i - 1);
328
329 return i;
330}
331
332
333
334
335
336
337
338
339
340
341static void smtc_tc_setup(int vpe, int tc, int cpu)
342{
343 static int cp1contexts[MAX_SMTC_VPES];
344
345
346
347
348
349 if (tc == 1)
350 {
351
352
353
354
355 cp1contexts[0] = smtc_nconf1[0] - 1;
356 cp1contexts[1] = smtc_nconf1[1];
357 }
358
359 settc(tc);
360 write_tc_c0_tchalt(TCHALT_H);
361 mips_ihb();
362 write_tc_c0_tcstatus((read_tc_c0_tcstatus()
363 & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT))
364 | TCSTATUS_A);
365
366
367
368
369
370 write_tc_c0_tccontext((sizeof(struct smtc_ipi_q) * cpu) << 16);
371
372
373 write_tc_c0_tcbind(vpe);
374
375
376 memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips));
377
378
379 if (!cp1contexts[vpe])
380 cpu_data[cpu].options &= ~MIPS_CPU_FPU;
381 else
382 cp1contexts[vpe]--;
383
384
385 cpu_data[cpu].vpe_id = vpe;
386 cpu_data[cpu].tc_id = tc;
387
388
389 cpu_data[cpu].core = (read_vpe_c0_ebase() >> 1) & 0xff;
390}
391
392
393
394
395
396
397#define CP0_SKEW 8
398
399void smtc_prepare_cpus(int cpus)
400{
401 int i, vpe, tc, ntc, nvpe, tcpervpe[NR_CPUS], slop, cpu;
402 unsigned long flags;
403 unsigned long val;
404 int nipi;
405 struct smtc_ipi *pipi;
406
407
408 local_irq_save(flags);
409
410 dvpe();
411 dmt();
412
413 spin_lock_init(&freeIPIq.lock);
414
415
416
417
418
419 for (i=0; i<NR_CPUS; i++) {
420 IPIQ[i].head = IPIQ[i].tail = NULL;
421 spin_lock_init(&IPIQ[i].lock);
422 IPIQ[i].depth = 0;
423 IPIQ[i].resched_flag = 0;
424 }
425
426
427 cpu = 0;
428 cpu_data[cpu].vpe_id = 0;
429 cpu_data[cpu].tc_id = 0;
430 cpu_data[cpu].core = (read_c0_ebase() >> 1) & 0xff;
431 cpu++;
432
433
434 mips_mt_set_cpuoptions();
435 if (vpelimit > 0)
436 printk("Limit of %d VPEs set\n", vpelimit);
437 if (tclimit > 0)
438 printk("Limit of %d TCs set\n", tclimit);
439 if (nostlb) {
440 printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n");
441 }
442 if (asidmask)
443 printk("ASID mask value override to 0x%x\n", asidmask);
444
445
446#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
447 if (hang_trig)
448 printk("Logic Analyser Trigger on suspected TC hang\n");
449#endif
450
451
452 write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC );
453
454 val = read_c0_mvpconf0();
455 nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1;
456 if (vpelimit > 0 && nvpe > vpelimit)
457 nvpe = vpelimit;
458 ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
459 if (ntc > NR_CPUS)
460 ntc = NR_CPUS;
461 if (tclimit > 0 && ntc > tclimit)
462 ntc = tclimit;
463 slop = ntc % nvpe;
464 for (i = 0; i < nvpe; i++) {
465 tcpervpe[i] = ntc / nvpe;
466 if (slop) {
467 if((slop - i) > 0) tcpervpe[i]++;
468 }
469 }
470
471 if (vpe0limit > ntc) vpe0limit = ntc;
472 if (vpe0limit > 0) {
473 int slopslop;
474 if (vpe0limit < tcpervpe[0]) {
475
476 slop = tcpervpe[0] - vpe0limit;
477 slopslop = slop % (nvpe - 1);
478 tcpervpe[0] = vpe0limit;
479 for (i = 1; i < nvpe; i++) {
480 tcpervpe[i] += slop / (nvpe - 1);
481 if(slopslop && ((slopslop - (i - 1) > 0)))
482 tcpervpe[i]++;
483 }
484 } else if (vpe0limit > tcpervpe[0]) {
485
486 slop = vpe0limit - tcpervpe[0];
487 slopslop = slop % (nvpe - 1);
488 tcpervpe[0] = vpe0limit;
489 for (i = 1; i < nvpe; i++) {
490 tcpervpe[i] -= slop / (nvpe - 1);
491 if(slopslop && ((slopslop - (i - 1) > 0)))
492 tcpervpe[i]--;
493 }
494 }
495 }
496
497
498 smtc_configure_tlb();
499
500 for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) {
501
502 if (tc == 0)
503 {
504
505
506
507
508
509 smtc_nconf1[0] = ((read_vpe_c0_vpeconf1() &
510 VPECONF1_NCP1) >> VPECONF1_NCP1_SHIFT);
511 if (nvpe == 2)
512 {
513 settc(1);
514 smtc_nconf1[1] = ((read_vpe_c0_vpeconf1() &
515 VPECONF1_NCP1) >> VPECONF1_NCP1_SHIFT);
516 settc(0);
517 }
518 }
519 if (tcpervpe[vpe] == 0)
520 continue;
521 if (vpe != 0)
522 printk(", ");
523 printk("VPE %d: TC", vpe);
524 for (i = 0; i < tcpervpe[vpe]; i++) {
525
526
527
528
529
530 if (tc != 0) {
531 smtc_tc_setup(vpe, tc, cpu);
532 if (vpe != 0) {
533
534
535
536
537
538
539
540 write_vpe_c0_vpeconf0(
541 read_vpe_c0_vpeconf0() |
542 VPECONF0_MVP);
543 }
544 cpu++;
545 }
546 printk(" %d", tc);
547 tc++;
548 }
549 if (vpe != 0) {
550
551
552
553 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() |
554 VPECONF0_MVP);
555
556
557
558
559 write_vpe_c0_cause(0);
560
561
562
563
564
565 write_vpe_c0_status((read_vpe_c0_status()
566 & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM))
567 | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7
568 | ST0_IE));
569
570
571
572
573 write_vpe_c0_config(read_c0_config());
574
575 write_vpe_c0_compare(0);
576
577 write_vpe_c0_config7(read_c0_config7());
578 write_vpe_c0_count(read_c0_count() + CP0_SKEW);
579 ehb();
580 }
581
582 write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE);
583
584 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA);
585 }
586
587
588
589
590 while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) {
591 set_cpu_possible(tc, false);
592 set_cpu_present(tc, false);
593 tc++;
594 }
595
596
597 write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC );
598
599 printk("\n");
600
601
602
603#ifdef CONFIG_MIPS_MT_FPAFF
604 for (tc = 0; tc < ntc; tc++) {
605 if (cpu_data[tc].options & MIPS_CPU_FPU)
606 cpu_set(tc, mt_fpu_cpumask);
607 }
608#endif
609
610
611
612
613
614 setup_cross_vpe_interrupts(nvpe);
615
616
617 nipi = NR_CPUS * IPIBUF_PER_CPU;
618 if (ipibuffers > 0)
619 nipi = ipibuffers;
620
621 pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL);
622 if (pipi == NULL)
623 panic("kmalloc of IPI message buffers failed");
624 else
625 printk("IPI buffer pool of %d buffers\n", nipi);
626 for (i = 0; i < nipi; i++) {
627 smtc_ipi_nq(&freeIPIq, pipi);
628 pipi++;
629 }
630
631
632 emt(EMT_ENABLE);
633 evpe(EVPE_ENABLE);
634 local_irq_restore(flags);
635
636 init_smtc_stats();
637}
638
639
640
641
642
643
644
645
646
647
648void smtc_boot_secondary(int cpu, struct task_struct *idle)
649{
650 extern u32 kernelsp[NR_CPUS];
651 unsigned long flags;
652 int mtflags;
653
654 LOCK_MT_PRA();
655 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
656 dvpe();
657 }
658 settc(cpu_data[cpu].tc_id);
659
660
661 write_tc_c0_tcrestart((unsigned long)&smp_bootstrap);
662
663
664 kernelsp[cpu] = __KSTK_TOS(idle);
665 write_tc_gpr_sp(__KSTK_TOS(idle));
666
667
668 write_tc_gpr_gp((unsigned long)task_thread_info(idle));
669
670 smtc_status |= SMTC_MTC_ACTIVE;
671 write_tc_c0_tchalt(0);
672 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
673 evpe(EVPE_ENABLE);
674 }
675 UNLOCK_MT_PRA();
676}
677
678void smtc_init_secondary(void)
679{
680}
681
682void smtc_smp_finish(void)
683{
684 int cpu = smp_processor_id();
685
686
687
688
689
690
691
692 if (cpu > 0 && (cpu_data[cpu].vpe_id != cpu_data[cpu - 1].vpe_id))
693 write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ);
694
695 local_irq_enable();
696
697 printk("TC %d going on-line as CPU %d\n",
698 cpu_data[smp_processor_id()].tc_id, smp_processor_id());
699}
700
701void smtc_cpus_done(void)
702{
703}
704
705
706
707
708
709
710
711
712
713
714
715int setup_irq_smtc(unsigned int irq, struct irqaction * new,
716 unsigned long hwmask)
717{
718#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
719 unsigned int vpe = current_cpu_data.vpe_id;
720
721 vpemask[vpe][irq - MIPS_CPU_IRQ_BASE] = 1;
722#endif
723 irq_hwmask[irq] = hwmask;
724
725 return setup_irq(irq, new);
726}
727
728#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF
729
730
731
732
733void smtc_set_irq_affinity(unsigned int irq, cpumask_t affinity)
734{
735
736
737
738
739
740}
741
742void smtc_forward_irq(struct irq_data *d)
743{
744 unsigned int irq = d->irq;
745 int target;
746
747
748
749
750
751
752
753
754
755
756
757
758 target = cpumask_first(d->affinity);
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773 if (target >= NR_CPUS)
774 do_IRQ_no_affinity(irq);
775 else
776 smtc_send_ipi(target, IRQ_AFFINITY_IPI, irq);
777}
778
779#endif
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804static void smtc_ipi_qdump(void)
805{
806 int i;
807 struct smtc_ipi *temp;
808
809 for (i = 0; i < NR_CPUS ;i++) {
810 pr_info("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n",
811 i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail,
812 IPIQ[i].depth);
813 temp = IPIQ[i].head;
814
815 while (temp != IPIQ[i].tail) {
816 pr_debug("%d %d %d: ", temp->type, temp->dest,
817 (int)temp->arg);
818#ifdef SMTC_IPI_DEBUG
819 pr_debug("%u %lu\n", temp->sender, temp->stamp);
820#else
821 pr_debug("\n");
822#endif
823 temp = temp->flink;
824 }
825 }
826}
827
828
829
830
831
832
833
834
835
836static inline int atomic_postincrement(atomic_t *v)
837{
838 unsigned long result;
839
840 unsigned long temp;
841
842 __asm__ __volatile__(
843 "1: ll %0, %2 \n"
844 " addu %1, %0, 1 \n"
845 " sc %1, %2 \n"
846 " beqz %1, 1b \n"
847 __WEAK_LLSC_MB
848 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
849 : "m" (v->counter)
850 : "memory");
851
852 return result;
853}
854
855void smtc_send_ipi(int cpu, int type, unsigned int action)
856{
857 int tcstatus;
858 struct smtc_ipi *pipi;
859 unsigned long flags;
860 int mtflags;
861 unsigned long tcrestart;
862 int set_resched_flag = (type == LINUX_SMP_IPI &&
863 action == SMP_RESCHEDULE_YOURSELF);
864
865 if (cpu == smp_processor_id()) {
866 printk("Cannot Send IPI to self!\n");
867 return;
868 }
869 if (set_resched_flag && IPIQ[cpu].resched_flag != 0)
870 return;
871
872
873 pipi = smtc_ipi_dq(&freeIPIq);
874 if (pipi == NULL) {
875 bust_spinlocks(1);
876 mips_mt_regdump(dvpe());
877 panic("IPI Msg. Buffers Depleted");
878 }
879 pipi->type = type;
880 pipi->arg = (void *)action;
881 pipi->dest = cpu;
882 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
883
884 IPIQ[cpu].resched_flag |= set_resched_flag;
885 smtc_ipi_nq(&IPIQ[cpu], pipi);
886 LOCK_CORE_PRA();
887 settc(cpu_data[cpu].tc_id);
888 write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1);
889 UNLOCK_CORE_PRA();
890 } else {
891
892
893
894
895
896 LOCK_CORE_PRA();
897 settc(cpu_data[cpu].tc_id);
898
899 write_tc_c0_tchalt(TCHALT_H);
900 mips_ihb();
901
902
903
904
905
906
907 tcstatus = read_tc_c0_tcstatus();
908
909 if ((tcstatus & TCSTATUS_IXMT) != 0) {
910
911
912
913
914
915 if (cpu_wait == r4k_wait_irqoff) {
916 tcrestart = read_tc_c0_tcrestart();
917 if (address_is_in_r4k_wait_irqoff(tcrestart)) {
918 write_tc_c0_tcrestart(__pastwait);
919 tcstatus &= ~TCSTATUS_IXMT;
920 write_tc_c0_tcstatus(tcstatus);
921 goto postdirect;
922 }
923 }
924
925
926
927
928 write_tc_c0_tchalt(0);
929 UNLOCK_CORE_PRA();
930 IPIQ[cpu].resched_flag |= set_resched_flag;
931 smtc_ipi_nq(&IPIQ[cpu], pipi);
932 } else {
933postdirect:
934 post_direct_ipi(cpu, pipi);
935 write_tc_c0_tchalt(0);
936 UNLOCK_CORE_PRA();
937 }
938 }
939}
940
941
942
943
944static void post_direct_ipi(int cpu, struct smtc_ipi *pipi)
945{
946 struct pt_regs *kstack;
947 unsigned long tcstatus;
948 unsigned long tcrestart;
949 extern u32 kernelsp[NR_CPUS];
950 extern void __smtc_ipi_vector(void);
951
952
953
954 tcstatus = read_tc_c0_tcstatus();
955 tcrestart = read_tc_c0_tcrestart();
956
957 if ((tcrestart & 0x80000000)
958 && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) {
959 tcrestart += 4;
960 }
961
962
963
964
965
966
967 if (tcstatus & ST0_CU0) {
968
969 kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1;
970 } else {
971 kstack = ((struct pt_regs *)kernelsp[cpu]) - 1;
972 }
973
974 kstack->cp0_epc = (long)tcrestart;
975
976 kstack->cp0_tcstatus = tcstatus;
977
978 kstack->pad0[4] = (unsigned long)pipi;
979
980 kstack->pad0[5] = (unsigned long)&ipi_decode;
981
982 tcstatus |= TCSTATUS_IXMT;
983 tcstatus &= ~TCSTATUS_TKSU;
984 write_tc_c0_tcstatus(tcstatus);
985 ehb();
986
987 write_tc_c0_tcrestart(__smtc_ipi_vector);
988}
989
990static void ipi_resched_interrupt(void)
991{
992 scheduler_ipi();
993}
994
995static void ipi_call_interrupt(void)
996{
997
998 smp_call_function_interrupt();
999}
1000
1001DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device);
1002
1003static void __irq_entry smtc_clock_tick_interrupt(void)
1004{
1005 unsigned int cpu = smp_processor_id();
1006 struct clock_event_device *cd;
1007 int irq = MIPS_CPU_IRQ_BASE + 1;
1008
1009 irq_enter();
1010 kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
1011 cd = &per_cpu(mips_clockevent_device, cpu);
1012 cd->event_handler(cd);
1013 irq_exit();
1014}
1015
1016void ipi_decode(struct smtc_ipi *pipi)
1017{
1018 void *arg_copy = pipi->arg;
1019 int type_copy = pipi->type;
1020
1021 smtc_ipi_nq(&freeIPIq, pipi);
1022
1023 switch (type_copy) {
1024 case SMTC_CLOCK_TICK:
1025 smtc_clock_tick_interrupt();
1026 break;
1027
1028 case LINUX_SMP_IPI:
1029 switch ((int)arg_copy) {
1030 case SMP_RESCHEDULE_YOURSELF:
1031 ipi_resched_interrupt();
1032 break;
1033 case SMP_CALL_FUNCTION:
1034 ipi_call_interrupt();
1035 break;
1036 default:
1037 printk("Impossible SMTC IPI Argument %p\n", arg_copy);
1038 break;
1039 }
1040 break;
1041#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF
1042 case IRQ_AFFINITY_IPI:
1043
1044
1045
1046
1047 do_IRQ_no_affinity((int)arg_copy);
1048 break;
1049#endif
1050 default:
1051 printk("Impossible SMTC IPI Type 0x%x\n", type_copy);
1052 break;
1053 }
1054}
1055
1056
1057
1058
1059
1060
1061
1062void deferred_smtc_ipi(void)
1063{
1064 int cpu = smp_processor_id();
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075 while (IPIQ[cpu].head != NULL) {
1076 struct smtc_ipi_q *q = &IPIQ[cpu];
1077 struct smtc_ipi *pipi;
1078 unsigned long flags;
1079
1080
1081
1082
1083
1084 local_irq_save(flags);
1085 spin_lock(&q->lock);
1086 pipi = __smtc_ipi_dq(q);
1087 spin_unlock(&q->lock);
1088 if (pipi != NULL) {
1089 if (pipi->type == LINUX_SMP_IPI &&
1090 (int)pipi->arg == SMP_RESCHEDULE_YOURSELF)
1091 IPIQ[cpu].resched_flag = 0;
1092 ipi_decode(pipi);
1093 }
1094
1095
1096
1097
1098
1099
1100 __arch_local_irq_restore(flags);
1101 }
1102}
1103
1104
1105
1106
1107
1108
1109
1110
1111static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ;
1112
1113static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
1114{
1115 int my_vpe = cpu_data[smp_processor_id()].vpe_id;
1116 int my_tc = cpu_data[smp_processor_id()].tc_id;
1117 int cpu;
1118 struct smtc_ipi *pipi;
1119 unsigned long tcstatus;
1120 int sent;
1121 unsigned long flags;
1122 unsigned int mtflags;
1123 unsigned int vpflags;
1124
1125
1126
1127
1128
1129
1130
1131 local_irq_save(flags);
1132 vpflags = dvpe();
1133 clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ);
1134 set_c0_status(0x100 << MIPS_CPU_IPI_IRQ);
1135 irq_enable_hazard();
1136 evpe(vpflags);
1137 local_irq_restore(flags);
1138
1139
1140
1141
1142
1143
1144
1145
1146 for_each_online_cpu(cpu) {
1147 if (cpu_data[cpu].vpe_id != my_vpe)
1148 continue;
1149
1150 pipi = smtc_ipi_dq(&IPIQ[cpu]);
1151 if (pipi != NULL) {
1152 if (cpu_data[cpu].tc_id != my_tc) {
1153 sent = 0;
1154 LOCK_MT_PRA();
1155 settc(cpu_data[cpu].tc_id);
1156 write_tc_c0_tchalt(TCHALT_H);
1157 mips_ihb();
1158 tcstatus = read_tc_c0_tcstatus();
1159 if ((tcstatus & TCSTATUS_IXMT) == 0) {
1160 post_direct_ipi(cpu, pipi);
1161 sent = 1;
1162 }
1163 write_tc_c0_tchalt(0);
1164 UNLOCK_MT_PRA();
1165 if (!sent) {
1166 smtc_ipi_req(&IPIQ[cpu], pipi);
1167 }
1168 } else {
1169
1170
1171
1172
1173 local_irq_save(flags);
1174 if (pipi->type == LINUX_SMP_IPI &&
1175 (int)pipi->arg == SMP_RESCHEDULE_YOURSELF)
1176 IPIQ[cpu].resched_flag = 0;
1177 ipi_decode(pipi);
1178 local_irq_restore(flags);
1179 }
1180 }
1181 }
1182
1183 return IRQ_HANDLED;
1184}
1185
1186static void ipi_irq_dispatch(void)
1187{
1188 do_IRQ(cpu_ipi_irq);
1189}
1190
1191static struct irqaction irq_ipi = {
1192 .handler = ipi_interrupt,
1193 .flags = IRQF_PERCPU,
1194 .name = "SMTC_IPI"
1195};
1196
1197static void setup_cross_vpe_interrupts(unsigned int nvpe)
1198{
1199 if (nvpe < 1)
1200 return;
1201
1202 if (!cpu_has_vint)
1203 panic("SMTC Kernel requires Vectored Interrupt support");
1204
1205 set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch);
1206
1207 setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ));
1208
1209 irq_set_handler(cpu_ipi_irq, handle_percpu_irq);
1210}
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220void smtc_ipi_replay(void)
1221{
1222 unsigned int cpu = smp_processor_id();
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235 while (IPIQ[cpu].head != NULL) {
1236 struct smtc_ipi_q *q = &IPIQ[cpu];
1237 struct smtc_ipi *pipi;
1238 unsigned long flags;
1239
1240
1241
1242
1243
1244 local_irq_save(flags);
1245
1246 spin_lock(&q->lock);
1247 pipi = __smtc_ipi_dq(q);
1248 spin_unlock(&q->lock);
1249
1250
1251
1252 __arch_local_irq_restore(flags);
1253
1254 if (pipi) {
1255 self_ipi(pipi);
1256 smtc_cpu_stats[cpu].selfipis++;
1257 }
1258 }
1259}
1260
1261EXPORT_SYMBOL(smtc_ipi_replay);
1262
1263void smtc_idle_loop_hook(void)
1264{
1265#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
1266 int im;
1267 int flags;
1268 int mtflags;
1269 int bit;
1270 int vpe;
1271 int tc;
1272 int hook_ntcs;
1273
1274
1275
1276
1277 char *pdb_msg;
1278 char id_ho_db_msg[768];
1279
1280 if (atomic_read(&idle_hook_initialized) == 0) {
1281 if (atomic_add_return(1, &idle_hook_initialized) == 1) {
1282 int mvpconf0;
1283
1284 mvpconf0 = read_c0_mvpconf0();
1285 hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
1286 if (hook_ntcs > NR_CPUS)
1287 hook_ntcs = NR_CPUS;
1288 for (tc = 0; tc < hook_ntcs; tc++) {
1289 tcnoprog[tc] = 0;
1290 clock_hang_reported[tc] = 0;
1291 }
1292 for (vpe = 0; vpe < 2; vpe++)
1293 for (im = 0; im < 8; im++)
1294 imstuckcount[vpe][im] = 0;
1295 printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs);
1296 atomic_set(&idle_hook_initialized, 1000);
1297 } else {
1298
1299 while (atomic_read(&idle_hook_initialized) < 1000)
1300 ;
1301 }
1302 }
1303
1304
1305 if (read_c0_tcstatus() & 0x400) {
1306 write_c0_tcstatus(read_c0_tcstatus() & ~0x400);
1307 ehb();
1308 printk("Dangling IXMT in cpu_idle()\n");
1309 }
1310
1311
1312#define IM_LIMIT 2000
1313 local_irq_save(flags);
1314 mtflags = dmt();
1315 pdb_msg = &id_ho_db_msg[0];
1316 im = read_c0_status();
1317 vpe = current_cpu_data.vpe_id;
1318 for (bit = 0; bit < 8; bit++) {
1319
1320
1321
1322
1323 if (vpemask[vpe][bit]) {
1324 if (!(im & (0x100 << bit)))
1325 imstuckcount[vpe][bit]++;
1326 else
1327 imstuckcount[vpe][bit] = 0;
1328 if (imstuckcount[vpe][bit] > IM_LIMIT) {
1329 set_c0_status(0x100 << bit);
1330 ehb();
1331 imstuckcount[vpe][bit] = 0;
1332 pdb_msg += sprintf(pdb_msg,
1333 "Dangling IM %d fixed for VPE %d\n", bit,
1334 vpe);
1335 }
1336 }
1337 }
1338
1339 emt(mtflags);
1340 local_irq_restore(flags);
1341 if (pdb_msg != &id_ho_db_msg[0])
1342 printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg);
1343#endif
1344
1345 smtc_ipi_replay();
1346}
1347
1348void smtc_soft_dump(void)
1349{
1350 int i;
1351
1352 printk("Counter Interrupts taken per CPU (TC)\n");
1353 for (i=0; i < NR_CPUS; i++) {
1354 printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints);
1355 }
1356 printk("Self-IPI invocations:\n");
1357 for (i=0; i < NR_CPUS; i++) {
1358 printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis);
1359 }
1360 smtc_ipi_qdump();
1361 printk("%d Recoveries of \"stolen\" FPU\n",
1362 atomic_read(&smtc_fpu_recoveries));
1363}
1364
1365
1366
1367
1368
1369
1370void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu)
1371{
1372 unsigned long flags, mtflags, tcstat, prevhalt, asid;
1373 int tlb, i;
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386 local_irq_save(flags);
1387 if (smtc_status & SMTC_TLB_SHARED) {
1388 mtflags = dvpe();
1389 tlb = 0;
1390 } else {
1391 mtflags = dmt();
1392 tlb = cpu_data[cpu].vpe_id;
1393 }
1394 asid = asid_cache(cpu);
1395
1396 do {
1397 if (!((asid += ASID_INC) & ASID_MASK) ) {
1398 if (cpu_has_vtag_icache)
1399 flush_icache_all();
1400
1401 for_each_online_cpu(i) {
1402
1403
1404
1405
1406 if ((i != smp_processor_id()) &&
1407 ((smtc_status & SMTC_TLB_SHARED) ||
1408 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) {
1409 settc(cpu_data[i].tc_id);
1410 prevhalt = read_tc_c0_tchalt() & TCHALT_H;
1411 if (!prevhalt) {
1412 write_tc_c0_tchalt(TCHALT_H);
1413 mips_ihb();
1414 }
1415 tcstat = read_tc_c0_tcstatus();
1416 smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i);
1417 if (!prevhalt)
1418 write_tc_c0_tchalt(0);
1419 }
1420 }
1421 if (!asid)
1422 asid = ASID_FIRST_VERSION;
1423 local_flush_tlb_all();
1424 }
1425 } while (smtc_live_asid[tlb][(asid & ASID_MASK)]);
1426
1427
1428
1429
1430 for_each_online_cpu(i) {
1431 if ((smtc_status & SMTC_TLB_SHARED) ||
1432 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))
1433 cpu_context(i, mm) = asid_cache(i) = asid;
1434 }
1435
1436 if (smtc_status & SMTC_TLB_SHARED)
1437 evpe(mtflags);
1438 else
1439 emt(mtflags);
1440 local_irq_restore(flags);
1441}
1442
1443
1444
1445
1446
1447
1448
1449void smtc_flush_tlb_asid(unsigned long asid)
1450{
1451 int entry;
1452 unsigned long ehi;
1453
1454 entry = read_c0_wired();
1455
1456
1457 while (entry < current_cpu_data.tlbsize) {
1458 write_c0_index(entry);
1459 ehb();
1460 tlb_read();
1461 ehb();
1462 ehi = read_c0_entryhi();
1463 if ((ehi & ASID_MASK) == asid) {
1464
1465
1466
1467
1468 write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
1469 write_c0_entrylo0(0);
1470 write_c0_entrylo1(0);
1471 mtc0_tlbw_hazard();
1472 tlb_write_indexed();
1473 }
1474 entry++;
1475 }
1476 write_c0_index(PARKED_INDEX);
1477 tlbw_use_hazard();
1478}
1479
1480
1481
1482
1483
1484static int halt_state_save[NR_CPUS];
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495void smtc_cflush_lockdown(void)
1496{
1497 int cpu;
1498
1499 for_each_online_cpu(cpu) {
1500 if (cpu != smp_processor_id()) {
1501 settc(cpu_data[cpu].tc_id);
1502 halt_state_save[cpu] = read_tc_c0_tchalt();
1503 write_tc_c0_tchalt(TCHALT_H);
1504 }
1505 }
1506 mips_ihb();
1507}
1508
1509
1510
1511void smtc_cflush_release(void)
1512{
1513 int cpu;
1514
1515
1516
1517
1518
1519 mips_ihb();
1520
1521 for_each_online_cpu(cpu) {
1522 if (cpu != smp_processor_id()) {
1523 settc(cpu_data[cpu].tc_id);
1524 write_tc_c0_tchalt(halt_state_save[cpu]);
1525 }
1526 }
1527 mips_ihb();
1528}
1529