1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/cpu.h>
16#include <linux/cpumask.h>
17#include <linux/delay.h>
18#include <linux/init.h>
19#include <linux/interrupt.h>
20#include <linux/irqdomain.h>
21#include <linux/irq.h>
22#include <linux/kdebug.h>
23#include <linux/module.h>
24#include <linux/sched/mm.h>
25#include <linux/sched/hotplug.h>
26#include <linux/sched/task_stack.h>
27#include <linux/reboot.h>
28#include <linux/seq_file.h>
29#include <linux/smp.h>
30#include <linux/thread_info.h>
31
32#include <asm/cacheflush.h>
33#include <asm/kdebug.h>
34#include <asm/mmu_context.h>
35#include <asm/mxregs.h>
36#include <asm/platform.h>
37#include <asm/tlbflush.h>
38#include <asm/traps.h>
39
40#ifdef CONFIG_SMP
41# if XCHAL_HAVE_S32C1I == 0
42# error "The S32C1I option is required for SMP."
43# endif
44#endif
45
46static void system_invalidate_dcache_range(unsigned long start,
47 unsigned long size);
48static void system_flush_invalidate_dcache_range(unsigned long start,
49 unsigned long size);
50
51
52
53#define IPI_IRQ 0
54
55static irqreturn_t ipi_interrupt(int irq, void *dev_id);
56
57void ipi_init(void)
58{
59 unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
60 if (request_irq(irq, ipi_interrupt, IRQF_PERCPU, "ipi", NULL))
61 pr_err("Failed to request irq %u (ipi)\n", irq);
62}
63
64static inline unsigned int get_core_count(void)
65{
66
67 unsigned int syscfgid = get_er(SYSCFGID);
68 return ((syscfgid >> 18) & 0xf) + 1;
69}
70
71static inline int get_core_id(void)
72{
73
74 unsigned int core_id = get_er(SYSCFGID);
75 return core_id & 0x3fff;
76}
77
78void __init smp_prepare_cpus(unsigned int max_cpus)
79{
80 unsigned i;
81
82 for_each_possible_cpu(i)
83 set_cpu_present(i, true);
84}
85
86void __init smp_init_cpus(void)
87{
88 unsigned i;
89 unsigned int ncpus = get_core_count();
90 unsigned int core_id = get_core_id();
91
92 pr_info("%s: Core Count = %d\n", __func__, ncpus);
93 pr_info("%s: Core Id = %d\n", __func__, core_id);
94
95 if (ncpus > NR_CPUS) {
96 ncpus = NR_CPUS;
97 pr_info("%s: limiting core count by %d\n", __func__, ncpus);
98 }
99
100 for (i = 0; i < ncpus; ++i)
101 set_cpu_possible(i, true);
102}
103
104void __init smp_prepare_boot_cpu(void)
105{
106 unsigned int cpu = smp_processor_id();
107 BUG_ON(cpu != 0);
108 cpu_asid_cache(cpu) = ASID_USER_FIRST;
109}
110
111void __init smp_cpus_done(unsigned int max_cpus)
112{
113}
114
115static int boot_secondary_processors = 1;
116static DECLARE_COMPLETION(cpu_running);
117
118void secondary_start_kernel(void)
119{
120 struct mm_struct *mm = &init_mm;
121 unsigned int cpu = smp_processor_id();
122
123 init_mmu();
124
125#ifdef CONFIG_DEBUG_MISC
126 if (boot_secondary_processors == 0) {
127 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
128 __func__, boot_secondary_processors, cpu);
129 for (;;)
130 __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
131 }
132
133 pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
134 __func__, boot_secondary_processors, cpu);
135#endif
136
137
138 secondary_trap_init();
139
140
141
142 mmget(mm);
143 mmgrab(mm);
144 current->active_mm = mm;
145 cpumask_set_cpu(cpu, mm_cpumask(mm));
146 enter_lazy_tlb(mm, current);
147
148 trace_hardirqs_off();
149
150 calibrate_delay();
151
152 notify_cpu_starting(cpu);
153
154 secondary_init_irq();
155 local_timer_setup(cpu);
156
157 set_cpu_online(cpu, true);
158
159 local_irq_enable();
160
161 complete(&cpu_running);
162
163 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
164}
165
166static void mx_cpu_start(void *p)
167{
168 unsigned cpu = (unsigned)p;
169 unsigned long run_stall_mask = get_er(MPSCORE);
170
171 set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
172 pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
173 __func__, cpu, run_stall_mask, get_er(MPSCORE));
174}
175
176static void mx_cpu_stop(void *p)
177{
178 unsigned cpu = (unsigned)p;
179 unsigned long run_stall_mask = get_er(MPSCORE);
180
181 set_er(run_stall_mask | (1u << cpu), MPSCORE);
182 pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
183 __func__, cpu, run_stall_mask, get_er(MPSCORE));
184}
185
186#ifdef CONFIG_HOTPLUG_CPU
187unsigned long cpu_start_id __cacheline_aligned;
188#endif
189unsigned long cpu_start_ccount;
190
191static int boot_secondary(unsigned int cpu, struct task_struct *ts)
192{
193 unsigned long timeout = jiffies + msecs_to_jiffies(1000);
194 unsigned long ccount;
195 int i;
196
197#ifdef CONFIG_HOTPLUG_CPU
198 WRITE_ONCE(cpu_start_id, cpu);
199
200 mb();
201 system_flush_invalidate_dcache_range((unsigned long)&cpu_start_id,
202 sizeof(cpu_start_id));
203#endif
204 smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
205
206 for (i = 0; i < 2; ++i) {
207 do
208 ccount = get_ccount();
209 while (!ccount);
210
211 WRITE_ONCE(cpu_start_ccount, ccount);
212
213 do {
214
215
216
217
218 mb();
219 ccount = READ_ONCE(cpu_start_ccount);
220 } while (ccount && time_before(jiffies, timeout));
221
222 if (ccount) {
223 smp_call_function_single(0, mx_cpu_stop,
224 (void *)cpu, 1);
225 WRITE_ONCE(cpu_start_ccount, 0);
226 return -EIO;
227 }
228 }
229 return 0;
230}
231
232int __cpu_up(unsigned int cpu, struct task_struct *idle)
233{
234 int ret = 0;
235
236 if (cpu_asid_cache(cpu) == 0)
237 cpu_asid_cache(cpu) = ASID_USER_FIRST;
238
239 start_info.stack = (unsigned long)task_pt_regs(idle);
240 wmb();
241
242 pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
243 __func__, cpu, idle, start_info.stack);
244
245 init_completion(&cpu_running);
246 ret = boot_secondary(cpu, idle);
247 if (ret == 0) {
248 wait_for_completion_timeout(&cpu_running,
249 msecs_to_jiffies(1000));
250 if (!cpu_online(cpu))
251 ret = -EIO;
252 }
253
254 if (ret)
255 pr_err("CPU %u failed to boot\n", cpu);
256
257 return ret;
258}
259
260#ifdef CONFIG_HOTPLUG_CPU
261
262
263
264
265int __cpu_disable(void)
266{
267 unsigned int cpu = smp_processor_id();
268
269
270
271
272
273 set_cpu_online(cpu, false);
274
275
276
277
278 migrate_irqs();
279
280
281
282
283
284 local_flush_cache_all();
285 local_flush_tlb_all();
286 invalidate_page_directory();
287
288 clear_tasks_mm_cpumask(cpu);
289
290 return 0;
291}
292
293static void platform_cpu_kill(unsigned int cpu)
294{
295 smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
296}
297
298
299
300
301
302void __cpu_die(unsigned int cpu)
303{
304 unsigned long timeout = jiffies + msecs_to_jiffies(1000);
305 while (time_before(jiffies, timeout)) {
306 system_invalidate_dcache_range((unsigned long)&cpu_start_id,
307 sizeof(cpu_start_id));
308
309 mb();
310 if (READ_ONCE(cpu_start_id) == -cpu) {
311 platform_cpu_kill(cpu);
312 return;
313 }
314 }
315 pr_err("CPU%u: unable to kill\n", cpu);
316}
317
318void arch_cpu_idle_dead(void)
319{
320 cpu_die();
321}
322
323
324
325
326
327
328
329
330void __ref cpu_die(void)
331{
332 idle_task_exit();
333 local_irq_disable();
334 __asm__ __volatile__(
335 " movi a2, cpu_restart\n"
336 " jx a2\n");
337}
338
339#endif
340
341enum ipi_msg_type {
342 IPI_RESCHEDULE = 0,
343 IPI_CALL_FUNC,
344 IPI_CPU_STOP,
345 IPI_MAX
346};
347
348static const struct {
349 const char *short_text;
350 const char *long_text;
351} ipi_text[] = {
352 { .short_text = "RES", .long_text = "Rescheduling interrupts" },
353 { .short_text = "CAL", .long_text = "Function call interrupts" },
354 { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
355};
356
357struct ipi_data {
358 unsigned long ipi_count[IPI_MAX];
359};
360
361static DEFINE_PER_CPU(struct ipi_data, ipi_data);
362
363static void send_ipi_message(const struct cpumask *callmask,
364 enum ipi_msg_type msg_id)
365{
366 int index;
367 unsigned long mask = 0;
368
369 for_each_cpu(index, callmask)
370 mask |= 1 << index;
371
372 set_er(mask, MIPISET(msg_id));
373}
374
375void arch_send_call_function_ipi_mask(const struct cpumask *mask)
376{
377 send_ipi_message(mask, IPI_CALL_FUNC);
378}
379
380void arch_send_call_function_single_ipi(int cpu)
381{
382 send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
383}
384
385void smp_send_reschedule(int cpu)
386{
387 send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
388}
389
390void smp_send_stop(void)
391{
392 struct cpumask targets;
393
394 cpumask_copy(&targets, cpu_online_mask);
395 cpumask_clear_cpu(smp_processor_id(), &targets);
396 send_ipi_message(&targets, IPI_CPU_STOP);
397}
398
399static void ipi_cpu_stop(unsigned int cpu)
400{
401 set_cpu_online(cpu, false);
402 machine_halt();
403}
404
405irqreturn_t ipi_interrupt(int irq, void *dev_id)
406{
407 unsigned int cpu = smp_processor_id();
408 struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
409
410 for (;;) {
411 unsigned int msg;
412
413 msg = get_er(MIPICAUSE(cpu));
414 set_er(msg, MIPICAUSE(cpu));
415
416 if (!msg)
417 break;
418
419 if (msg & (1 << IPI_CALL_FUNC)) {
420 ++ipi->ipi_count[IPI_CALL_FUNC];
421 generic_smp_call_function_interrupt();
422 }
423
424 if (msg & (1 << IPI_RESCHEDULE)) {
425 ++ipi->ipi_count[IPI_RESCHEDULE];
426 scheduler_ipi();
427 }
428
429 if (msg & (1 << IPI_CPU_STOP)) {
430 ++ipi->ipi_count[IPI_CPU_STOP];
431 ipi_cpu_stop(cpu);
432 }
433 }
434
435 return IRQ_HANDLED;
436}
437
438void show_ipi_list(struct seq_file *p, int prec)
439{
440 unsigned int cpu;
441 unsigned i;
442
443 for (i = 0; i < IPI_MAX; ++i) {
444 seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
445 for_each_online_cpu(cpu)
446 seq_printf(p, " %10lu",
447 per_cpu(ipi_data, cpu).ipi_count[i]);
448 seq_printf(p, " %s\n", ipi_text[i].long_text);
449 }
450}
451
452int setup_profiling_timer(unsigned int multiplier)
453{
454 pr_debug("setup_profiling_timer %d\n", multiplier);
455 return 0;
456}
457
458
459
460struct flush_data {
461 struct vm_area_struct *vma;
462 unsigned long addr1;
463 unsigned long addr2;
464};
465
466static void ipi_flush_tlb_all(void *arg)
467{
468 local_flush_tlb_all();
469}
470
471void flush_tlb_all(void)
472{
473 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
474}
475
476static void ipi_flush_tlb_mm(void *arg)
477{
478 local_flush_tlb_mm(arg);
479}
480
481void flush_tlb_mm(struct mm_struct *mm)
482{
483 on_each_cpu(ipi_flush_tlb_mm, mm, 1);
484}
485
486static void ipi_flush_tlb_page(void *arg)
487{
488 struct flush_data *fd = arg;
489 local_flush_tlb_page(fd->vma, fd->addr1);
490}
491
492void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
493{
494 struct flush_data fd = {
495 .vma = vma,
496 .addr1 = addr,
497 };
498 on_each_cpu(ipi_flush_tlb_page, &fd, 1);
499}
500
501static void ipi_flush_tlb_range(void *arg)
502{
503 struct flush_data *fd = arg;
504 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
505}
506
507void flush_tlb_range(struct vm_area_struct *vma,
508 unsigned long start, unsigned long end)
509{
510 struct flush_data fd = {
511 .vma = vma,
512 .addr1 = start,
513 .addr2 = end,
514 };
515 on_each_cpu(ipi_flush_tlb_range, &fd, 1);
516}
517
518static void ipi_flush_tlb_kernel_range(void *arg)
519{
520 struct flush_data *fd = arg;
521 local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
522}
523
524void flush_tlb_kernel_range(unsigned long start, unsigned long end)
525{
526 struct flush_data fd = {
527 .addr1 = start,
528 .addr2 = end,
529 };
530 on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
531}
532
533
534
535static void ipi_flush_cache_all(void *arg)
536{
537 local_flush_cache_all();
538}
539
540void flush_cache_all(void)
541{
542 on_each_cpu(ipi_flush_cache_all, NULL, 1);
543}
544
545static void ipi_flush_cache_page(void *arg)
546{
547 struct flush_data *fd = arg;
548 local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
549}
550
551void flush_cache_page(struct vm_area_struct *vma,
552 unsigned long address, unsigned long pfn)
553{
554 struct flush_data fd = {
555 .vma = vma,
556 .addr1 = address,
557 .addr2 = pfn,
558 };
559 on_each_cpu(ipi_flush_cache_page, &fd, 1);
560}
561
562static void ipi_flush_cache_range(void *arg)
563{
564 struct flush_data *fd = arg;
565 local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
566}
567
568void flush_cache_range(struct vm_area_struct *vma,
569 unsigned long start, unsigned long end)
570{
571 struct flush_data fd = {
572 .vma = vma,
573 .addr1 = start,
574 .addr2 = end,
575 };
576 on_each_cpu(ipi_flush_cache_range, &fd, 1);
577}
578
579static void ipi_flush_icache_range(void *arg)
580{
581 struct flush_data *fd = arg;
582 local_flush_icache_range(fd->addr1, fd->addr2);
583}
584
585void flush_icache_range(unsigned long start, unsigned long end)
586{
587 struct flush_data fd = {
588 .addr1 = start,
589 .addr2 = end,
590 };
591 on_each_cpu(ipi_flush_icache_range, &fd, 1);
592}
593EXPORT_SYMBOL(flush_icache_range);
594
595
596
597static void ipi_invalidate_dcache_range(void *arg)
598{
599 struct flush_data *fd = arg;
600 __invalidate_dcache_range(fd->addr1, fd->addr2);
601}
602
603static void system_invalidate_dcache_range(unsigned long start,
604 unsigned long size)
605{
606 struct flush_data fd = {
607 .addr1 = start,
608 .addr2 = size,
609 };
610 on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
611}
612
613static void ipi_flush_invalidate_dcache_range(void *arg)
614{
615 struct flush_data *fd = arg;
616 __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
617}
618
619static void system_flush_invalidate_dcache_range(unsigned long start,
620 unsigned long size)
621{
622 struct flush_data fd = {
623 .addr1 = start,
624 .addr2 = size,
625 };
626 on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
627}
628