1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/stddef.h>
14#include <linux/kernel.h>
15#include <linux/init.h>
16#include <linux/delay.h>
17#include <linux/of.h>
18#include <linux/kexec.h>
19#include <linux/highmem.h>
20#include <linux/cpu.h>
21#include <linux/fsl/guts.h>
22
23#include <asm/machdep.h>
24#include <asm/pgtable.h>
25#include <asm/page.h>
26#include <asm/mpic.h>
27#include <asm/cacheflush.h>
28#include <asm/dbell.h>
29#include <asm/code-patching.h>
30#include <asm/cputhreads.h>
31#include <asm/fsl_pm.h>
32
33#include <sysdev/fsl_soc.h>
34#include <sysdev/mpic.h>
35#include "smp.h"
36
37struct epapr_spin_table {
38 u32 addr_h;
39 u32 addr_l;
40 u32 r3_h;
41 u32 r3_l;
42 u32 reserved;
43 u32 pir;
44};
45
46#ifdef CONFIG_HOTPLUG_CPU
47static u64 timebase;
48static int tb_req;
49static int tb_valid;
50
51static void mpc85xx_give_timebase(void)
52{
53 unsigned long flags;
54
55 local_irq_save(flags);
56 hard_irq_disable();
57
58 while (!tb_req)
59 barrier();
60 tb_req = 0;
61
62 qoriq_pm_ops->freeze_time_base(true);
63#ifdef CONFIG_PPC64
64
65
66
67
68
69
70
71
72
73
74 {
75 u64 prev;
76
77 asm volatile("mfspr %0, %1" : "=r" (timebase) :
78 "i" (SPRN_TBRL));
79
80 do {
81 prev = timebase;
82 asm volatile("mfspr %0, %1" : "=r" (timebase) :
83 "i" (SPRN_TBRL));
84 } while (prev != timebase);
85 }
86#else
87 timebase = get_tb();
88#endif
89 mb();
90 tb_valid = 1;
91
92 while (tb_valid)
93 barrier();
94
95 qoriq_pm_ops->freeze_time_base(false);
96
97 local_irq_restore(flags);
98}
99
100static void mpc85xx_take_timebase(void)
101{
102 unsigned long flags;
103
104 local_irq_save(flags);
105 hard_irq_disable();
106
107 tb_req = 1;
108 while (!tb_valid)
109 barrier();
110
111 set_tb(timebase >> 32, timebase & 0xffffffff);
112 isync();
113 tb_valid = 0;
114
115 local_irq_restore(flags);
116}
117
118static void smp_85xx_mach_cpu_die(void)
119{
120 unsigned int cpu = smp_processor_id();
121
122 local_irq_disable();
123 hard_irq_disable();
124
125 qoriq_pm_ops->irq_mask(cpu);
126
127 idle_task_exit();
128
129 mtspr(SPRN_TCR, 0);
130 mtspr(SPRN_TSR, mfspr(SPRN_TSR));
131
132 generic_set_cpu_dead(cpu);
133
134 cur_cpu_spec->cpu_down_flush();
135
136 qoriq_pm_ops->cpu_die(cpu);
137
138 while (1)
139 ;
140}
141
142static void qoriq_cpu_kill(unsigned int cpu)
143{
144 int i;
145
146 for (i = 0; i < 500; i++) {
147 if (is_cpu_dead(cpu)) {
148#ifdef CONFIG_PPC64
149 paca[cpu].cpu_start = 0;
150#endif
151 return;
152 }
153 msleep(20);
154 }
155 pr_err("CPU%d didn't die...\n", cpu);
156}
157#endif
158
159
160
161
162
163
164
165
166static inline void flush_spin_table(void *spin_table)
167{
168 flush_dcache_range((ulong)spin_table,
169 (ulong)spin_table + sizeof(struct epapr_spin_table));
170}
171
172static inline u32 read_spin_table_addr_l(void *spin_table)
173{
174 flush_dcache_range((ulong)spin_table,
175 (ulong)spin_table + sizeof(struct epapr_spin_table));
176 return in_be32(&((struct epapr_spin_table *)spin_table)->addr_l);
177}
178
179#ifdef CONFIG_PPC64
180static void wake_hw_thread(void *info)
181{
182 void fsl_secondary_thread_init(void);
183 unsigned long inia;
184 int cpu = *(const int *)info;
185
186 inia = *(unsigned long *)fsl_secondary_thread_init;
187 book3e_start_thread(cpu_thread_in_core(cpu), inia);
188}
189#endif
190
191static int smp_85xx_start_cpu(int cpu)
192{
193 int ret = 0;
194 struct device_node *np;
195 const u64 *cpu_rel_addr;
196 unsigned long flags;
197 int ioremappable;
198 int hw_cpu = get_hard_smp_processor_id(cpu);
199 struct epapr_spin_table __iomem *spin_table;
200
201 np = of_get_cpu_node(cpu, NULL);
202 cpu_rel_addr = of_get_property(np, "cpu-release-addr", NULL);
203 if (!cpu_rel_addr) {
204 pr_err("No cpu-release-addr for cpu %d\n", cpu);
205 return -ENOENT;
206 }
207
208
209
210
211
212
213
214 ioremappable = *cpu_rel_addr > virt_to_phys(high_memory);
215
216
217 if (ioremappable)
218 spin_table = ioremap_prot(*cpu_rel_addr,
219 sizeof(struct epapr_spin_table), _PAGE_COHERENT);
220 else
221 spin_table = phys_to_virt(*cpu_rel_addr);
222
223 local_irq_save(flags);
224 hard_irq_disable();
225
226 if (qoriq_pm_ops)
227 qoriq_pm_ops->cpu_up_prepare(cpu);
228
229
230 if (read_spin_table_addr_l(spin_table) != 1) {
231
232
233
234
235 mpic_reset_core(cpu);
236
237
238
239
240
241
242 if (!spin_event_timeout(
243 read_spin_table_addr_l(spin_table) == 1,
244 10000, 100)) {
245 pr_err("timeout waiting for cpu %d to reset\n",
246 hw_cpu);
247 ret = -EAGAIN;
248 goto err;
249 }
250 }
251
252 flush_spin_table(spin_table);
253 out_be32(&spin_table->pir, hw_cpu);
254#ifdef CONFIG_PPC64
255 out_be64((u64 *)(&spin_table->addr_h),
256 __pa(ppc_function_entry(generic_secondary_smp_init)));
257#else
258 out_be32(&spin_table->addr_l, __pa(__early_start));
259#endif
260 flush_spin_table(spin_table);
261err:
262 local_irq_restore(flags);
263
264 if (ioremappable)
265 iounmap(spin_table);
266
267 return ret;
268}
269
270static int smp_85xx_kick_cpu(int nr)
271{
272 int ret = 0;
273#ifdef CONFIG_PPC64
274 int primary = nr;
275#endif
276
277 WARN_ON(nr < 0 || nr >= num_possible_cpus());
278
279 pr_debug("kick CPU #%d\n", nr);
280
281#ifdef CONFIG_PPC64
282 if (threads_per_core == 2) {
283 if (WARN_ON_ONCE(!cpu_has_feature(CPU_FTR_SMT)))
284 return -ENOENT;
285
286 booting_thread_hwid = cpu_thread_in_core(nr);
287 primary = cpu_first_thread_sibling(nr);
288
289 if (qoriq_pm_ops)
290 qoriq_pm_ops->cpu_up_prepare(nr);
291
292
293
294
295
296 if (cpu_online(primary)) {
297 smp_call_function_single(primary,
298 wake_hw_thread, &nr, 1);
299 goto done;
300 } else if (cpu_online(primary + 1)) {
301 smp_call_function_single(primary + 1,
302 wake_hw_thread, &nr, 1);
303 goto done;
304 }
305
306
307
308
309
310
311
312
313 } else if (threads_per_core == 1) {
314
315
316
317
318 booting_thread_hwid = INVALID_THREAD_HWID;
319
320 } else if (threads_per_core > 2) {
321 pr_err("Do not support more than 2 threads per CPU.");
322 return -EINVAL;
323 }
324
325 ret = smp_85xx_start_cpu(primary);
326 if (ret)
327 return ret;
328
329done:
330 paca[nr].cpu_start = 1;
331 generic_set_cpu_up(nr);
332
333 return ret;
334#else
335 ret = smp_85xx_start_cpu(nr);
336 if (ret)
337 return ret;
338
339 generic_set_cpu_up(nr);
340
341 return ret;
342#endif
343}
344
345struct smp_ops_t smp_85xx_ops = {
346 .kick_cpu = smp_85xx_kick_cpu,
347 .cpu_bootable = smp_generic_cpu_bootable,
348#ifdef CONFIG_HOTPLUG_CPU
349 .cpu_disable = generic_cpu_disable,
350 .cpu_die = generic_cpu_die,
351#endif
352#if defined(CONFIG_KEXEC) && !defined(CONFIG_PPC64)
353 .give_timebase = smp_generic_give_timebase,
354 .take_timebase = smp_generic_take_timebase,
355#endif
356};
357
358#ifdef CONFIG_KEXEC
359#ifdef CONFIG_PPC32
360atomic_t kexec_down_cpus = ATOMIC_INIT(0);
361
362void mpc85xx_smp_kexec_cpu_down(int crash_shutdown, int secondary)
363{
364 local_irq_disable();
365
366 if (secondary) {
367 cur_cpu_spec->cpu_down_flush();
368 atomic_inc(&kexec_down_cpus);
369
370 while (1);
371 }
372}
373
374static void mpc85xx_smp_kexec_down(void *arg)
375{
376 if (ppc_md.kexec_cpu_down)
377 ppc_md.kexec_cpu_down(0,1);
378}
379#else
380void mpc85xx_smp_kexec_cpu_down(int crash_shutdown, int secondary)
381{
382 int cpu = smp_processor_id();
383 int sibling = cpu_last_thread_sibling(cpu);
384 bool notified = false;
385 int disable_cpu;
386 int disable_threadbit = 0;
387 long start = mftb();
388 long now;
389
390 local_irq_disable();
391 hard_irq_disable();
392 mpic_teardown_this_cpu(secondary);
393
394 if (cpu == crashing_cpu && cpu_thread_in_core(cpu) != 0) {
395
396
397
398
399
400 disable_threadbit = 1;
401 disable_cpu = cpu_first_thread_sibling(cpu);
402 } else if (sibling != crashing_cpu &&
403 cpu_thread_in_core(cpu) == 0 &&
404 cpu_thread_in_core(sibling) != 0) {
405 disable_threadbit = 2;
406 disable_cpu = sibling;
407 }
408
409 if (disable_threadbit) {
410 while (paca[disable_cpu].kexec_state < KEXEC_STATE_REAL_MODE) {
411 barrier();
412 now = mftb();
413 if (!notified && now - start > 1000000) {
414 pr_info("%s/%d: waiting for cpu %d to enter KEXEC_STATE_REAL_MODE (%d)\n",
415 __func__, smp_processor_id(),
416 disable_cpu,
417 paca[disable_cpu].kexec_state);
418 notified = true;
419 }
420 }
421
422 if (notified) {
423 pr_info("%s: cpu %d done waiting\n",
424 __func__, disable_cpu);
425 }
426
427 mtspr(SPRN_TENC, disable_threadbit);
428 while (mfspr(SPRN_TENSR) & disable_threadbit)
429 cpu_relax();
430 }
431}
432#endif
433
434static void mpc85xx_smp_machine_kexec(struct kimage *image)
435{
436#ifdef CONFIG_PPC32
437 int timeout = INT_MAX;
438 int i, num_cpus = num_present_cpus();
439
440 if (image->type == KEXEC_TYPE_DEFAULT)
441 smp_call_function(mpc85xx_smp_kexec_down, NULL, 0);
442
443 while ( (atomic_read(&kexec_down_cpus) != (num_cpus - 1)) &&
444 ( timeout > 0 ) )
445 {
446 timeout--;
447 }
448
449 if ( !timeout )
450 printk(KERN_ERR "Unable to bring down secondary cpu(s)");
451
452 for_each_online_cpu(i)
453 {
454 if ( i == smp_processor_id() ) continue;
455 mpic_reset_core(i);
456 }
457#endif
458
459 default_machine_kexec(image);
460}
461#endif
462
463static void smp_85xx_basic_setup(int cpu_nr)
464{
465 if (cpu_has_feature(CPU_FTR_DBELL))
466 doorbell_setup_this_cpu();
467}
468
469static void smp_85xx_setup_cpu(int cpu_nr)
470{
471 mpic_setup_this_cpu();
472 smp_85xx_basic_setup(cpu_nr);
473}
474
475void __init mpc85xx_smp_init(void)
476{
477 struct device_node *np;
478
479
480 np = of_find_node_by_type(NULL, "open-pic");
481 if (np) {
482 smp_85xx_ops.probe = smp_mpic_probe;
483 smp_85xx_ops.setup_cpu = smp_85xx_setup_cpu;
484 smp_85xx_ops.message_pass = smp_mpic_message_pass;
485 } else
486 smp_85xx_ops.setup_cpu = smp_85xx_basic_setup;
487
488 if (cpu_has_feature(CPU_FTR_DBELL)) {
489
490
491
492
493 smp_85xx_ops.message_pass = NULL;
494 smp_85xx_ops.cause_ipi = doorbell_cause_ipi;
495 smp_85xx_ops.probe = NULL;
496 }
497
498#ifdef CONFIG_HOTPLUG_CPU
499#ifdef CONFIG_FSL_CORENET_RCPM
500 fsl_rcpm_init();
501#endif
502
503#ifdef CONFIG_FSL_PMC
504 mpc85xx_setup_pmc();
505#endif
506 if (qoriq_pm_ops) {
507 smp_85xx_ops.give_timebase = mpc85xx_give_timebase;
508 smp_85xx_ops.take_timebase = mpc85xx_take_timebase;
509 ppc_md.cpu_die = smp_85xx_mach_cpu_die;
510 smp_85xx_ops.cpu_die = qoriq_cpu_kill;
511 }
512#endif
513 smp_ops = &smp_85xx_ops;
514
515#ifdef CONFIG_KEXEC
516 ppc_md.kexec_cpu_down = mpc85xx_smp_kexec_cpu_down;
517 ppc_md.machine_kexec = mpc85xx_smp_machine_kexec;
518#endif
519}
520