1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/sched.h>
16#include <linux/err.h>
17#include <linux/smp.h>
18
19#include <asm/paravirt.h>
20#include <asm/desc.h>
21#include <asm/pgtable.h>
22#include <asm/cpu.h>
23
24#include <xen/interface/xen.h>
25#include <xen/interface/vcpu.h>
26
27#include <asm/xen/interface.h>
28#include <asm/xen/hypercall.h>
29
30#include <xen/page.h>
31#include <xen/events.h>
32
33#include "xen-ops.h"
34#include "mmu.h"
35
36cpumask_var_t xen_cpu_initialized_map;
37
38static DEFINE_PER_CPU(int, resched_irq);
39static DEFINE_PER_CPU(int, callfunc_irq);
40static DEFINE_PER_CPU(int, callfuncsingle_irq);
41static DEFINE_PER_CPU(int, debug_irq) = -1;
42
43static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
44static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
45
46
47
48
49
50
51static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
52{
53 inc_irq_stat(irq_resched_count);
54
55 return IRQ_HANDLED;
56}
57
58static __cpuinit void cpu_bringup(void)
59{
60 int cpu = smp_processor_id();
61
62 cpu_init();
63 touch_softlockup_watchdog();
64 preempt_disable();
65
66 xen_enable_sysenter();
67 xen_enable_syscall();
68
69 cpu = smp_processor_id();
70 smp_store_cpu_info(cpu);
71 cpu_data(cpu).x86_max_cores = 1;
72 set_cpu_sibling_map(cpu);
73
74 xen_setup_cpu_clockevents();
75
76 cpu_set(cpu, cpu_online_map);
77 percpu_write(cpu_state, CPU_ONLINE);
78 wmb();
79
80
81 local_irq_enable();
82
83 wmb();
84}
85
86static __cpuinit void cpu_bringup_and_idle(void)
87{
88 cpu_bringup();
89 cpu_idle();
90}
91
92static int xen_smp_intr_init(unsigned int cpu)
93{
94 int rc;
95 const char *resched_name, *callfunc_name, *debug_name;
96
97 resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
98 rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
99 cpu,
100 xen_reschedule_interrupt,
101 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
102 resched_name,
103 NULL);
104 if (rc < 0)
105 goto fail;
106 per_cpu(resched_irq, cpu) = rc;
107
108 callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
109 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
110 cpu,
111 xen_call_function_interrupt,
112 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
113 callfunc_name,
114 NULL);
115 if (rc < 0)
116 goto fail;
117 per_cpu(callfunc_irq, cpu) = rc;
118
119 debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
120 rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
121 IRQF_DISABLED | IRQF_PERCPU | IRQF_NOBALANCING,
122 debug_name, NULL);
123 if (rc < 0)
124 goto fail;
125 per_cpu(debug_irq, cpu) = rc;
126
127 callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
128 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
129 cpu,
130 xen_call_function_single_interrupt,
131 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
132 callfunc_name,
133 NULL);
134 if (rc < 0)
135 goto fail;
136 per_cpu(callfuncsingle_irq, cpu) = rc;
137
138 return 0;
139
140 fail:
141 if (per_cpu(resched_irq, cpu) >= 0)
142 unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
143 if (per_cpu(callfunc_irq, cpu) >= 0)
144 unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
145 if (per_cpu(debug_irq, cpu) >= 0)
146 unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
147 if (per_cpu(callfuncsingle_irq, cpu) >= 0)
148 unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
149
150 return rc;
151}
152
153static void __init xen_fill_possible_map(void)
154{
155 int i, rc;
156
157 for (i = 0; i < nr_cpu_ids; i++) {
158 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
159 if (rc >= 0) {
160 num_processors++;
161 set_cpu_possible(i, true);
162 }
163 }
164}
165
166static void __init xen_smp_prepare_boot_cpu(void)
167{
168 BUG_ON(smp_processor_id() != 0);
169 native_smp_prepare_boot_cpu();
170
171
172
173 make_lowmem_page_readwrite(xen_initial_gdt);
174
175 xen_setup_vcpu_info_placement();
176}
177
178static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
179{
180 unsigned cpu;
181
182 xen_init_lock_cpu(0);
183
184 smp_store_cpu_info(0);
185 cpu_data(0).x86_max_cores = 1;
186 set_cpu_sibling_map(0);
187
188 if (xen_smp_intr_init(0))
189 BUG();
190
191 if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
192 panic("could not allocate xen_cpu_initialized_map\n");
193
194 cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
195
196
197 while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
198 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
199 continue;
200 set_cpu_possible(cpu, false);
201 }
202
203 for_each_possible_cpu (cpu) {
204 struct task_struct *idle;
205
206 if (cpu == 0)
207 continue;
208
209 idle = fork_idle(cpu);
210 if (IS_ERR(idle))
211 panic("failed fork for CPU %d", cpu);
212
213 set_cpu_present(cpu, true);
214 }
215}
216
217static __cpuinit int
218cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
219{
220 struct vcpu_guest_context *ctxt;
221 struct desc_struct *gdt;
222 unsigned long gdt_mfn;
223
224 if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
225 return 0;
226
227 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
228 if (ctxt == NULL)
229 return -ENOMEM;
230
231 gdt = get_cpu_gdt_table(cpu);
232
233 ctxt->flags = VGCF_IN_KERNEL;
234 ctxt->user_regs.ds = __USER_DS;
235 ctxt->user_regs.es = __USER_DS;
236 ctxt->user_regs.ss = __KERNEL_DS;
237#ifdef CONFIG_X86_32
238 ctxt->user_regs.fs = __KERNEL_PERCPU;
239 ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
240#else
241 ctxt->gs_base_kernel = per_cpu_offset(cpu);
242#endif
243 ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
244 ctxt->user_regs.eflags = 0x1000;
245
246 memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
247
248 xen_copy_trap_info(ctxt->trap_ctxt);
249
250 ctxt->ldt_ents = 0;
251
252 BUG_ON((unsigned long)gdt & ~PAGE_MASK);
253
254 gdt_mfn = arbitrary_virt_to_mfn(gdt);
255 make_lowmem_page_readonly(gdt);
256 make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
257
258 ctxt->gdt_frames[0] = gdt_mfn;
259 ctxt->gdt_ents = GDT_ENTRIES;
260
261 ctxt->user_regs.cs = __KERNEL_CS;
262 ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
263
264 ctxt->kernel_ss = __KERNEL_DS;
265 ctxt->kernel_sp = idle->thread.sp0;
266
267#ifdef CONFIG_X86_32
268 ctxt->event_callback_cs = __KERNEL_CS;
269 ctxt->failsafe_callback_cs = __KERNEL_CS;
270#endif
271 ctxt->event_callback_eip = (unsigned long)xen_hypervisor_callback;
272 ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback;
273
274 per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
275 ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
276
277 if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
278 BUG();
279
280 kfree(ctxt);
281 return 0;
282}
283
284static int __cpuinit xen_cpu_up(unsigned int cpu)
285{
286 struct task_struct *idle = idle_task(cpu);
287 int rc;
288
289 per_cpu(current_task, cpu) = idle;
290#ifdef CONFIG_X86_32
291 irq_ctx_init(cpu);
292#else
293 clear_tsk_thread_flag(idle, TIF_FORK);
294 per_cpu(kernel_stack, cpu) =
295 (unsigned long)task_stack_page(idle) -
296 KERNEL_STACK_OFFSET + THREAD_SIZE;
297#endif
298 xen_setup_timer(cpu);
299 xen_init_lock_cpu(cpu);
300
301 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
302
303
304 per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
305
306 rc = cpu_initialize_context(cpu, idle);
307 if (rc)
308 return rc;
309
310 if (num_online_cpus() == 1)
311 alternatives_smp_switch(1);
312
313 rc = xen_smp_intr_init(cpu);
314 if (rc)
315 return rc;
316
317 rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
318 BUG_ON(rc);
319
320 while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
321 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
322 barrier();
323 }
324
325 return 0;
326}
327
328static void xen_smp_cpus_done(unsigned int max_cpus)
329{
330}
331
332#ifdef CONFIG_HOTPLUG_CPU
333static int xen_cpu_disable(void)
334{
335 unsigned int cpu = smp_processor_id();
336 if (cpu == 0)
337 return -EBUSY;
338
339 cpu_disable_common();
340
341 load_cr3(swapper_pg_dir);
342 return 0;
343}
344
345static void xen_cpu_die(unsigned int cpu)
346{
347 while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
348 current->state = TASK_UNINTERRUPTIBLE;
349 schedule_timeout(HZ/10);
350 }
351 unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
352 unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
353 unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
354 unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
355 xen_uninit_lock_cpu(cpu);
356 xen_teardown_timer(cpu);
357
358 if (num_online_cpus() == 1)
359 alternatives_smp_switch(0);
360}
361
362static void __cpuinit xen_play_dead(void)
363{
364 play_dead_common();
365 HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
366 cpu_bringup();
367}
368
369#else
370static int xen_cpu_disable(void)
371{
372 return -ENOSYS;
373}
374
375static void xen_cpu_die(unsigned int cpu)
376{
377 BUG();
378}
379
380static void xen_play_dead(void)
381{
382 BUG();
383}
384
385#endif
386static void stop_self(void *v)
387{
388 int cpu = smp_processor_id();
389
390
391 load_cr3(swapper_pg_dir);
392
393
394 HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
395 BUG();
396}
397
398static void xen_smp_send_stop(void)
399{
400 smp_call_function(stop_self, NULL, 0);
401}
402
403static void xen_smp_send_reschedule(int cpu)
404{
405 xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
406}
407
408static void xen_send_IPI_mask(const struct cpumask *mask,
409 enum ipi_vector vector)
410{
411 unsigned cpu;
412
413 for_each_cpu_and(cpu, mask, cpu_online_mask)
414 xen_send_IPI_one(cpu, vector);
415}
416
417static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
418{
419 int cpu;
420
421 xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
422
423
424 for_each_cpu(cpu, mask) {
425 if (xen_vcpu_stolen(cpu)) {
426 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
427 break;
428 }
429 }
430}
431
432static void xen_smp_send_call_function_single_ipi(int cpu)
433{
434 xen_send_IPI_mask(cpumask_of(cpu),
435 XEN_CALL_FUNCTION_SINGLE_VECTOR);
436}
437
438static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
439{
440 irq_enter();
441 generic_smp_call_function_interrupt();
442 inc_irq_stat(irq_call_count);
443 irq_exit();
444
445 return IRQ_HANDLED;
446}
447
448static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
449{
450 irq_enter();
451 generic_smp_call_function_single_interrupt();
452 inc_irq_stat(irq_call_count);
453 irq_exit();
454
455 return IRQ_HANDLED;
456}
457
458static const struct smp_ops xen_smp_ops __initdata = {
459 .smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
460 .smp_prepare_cpus = xen_smp_prepare_cpus,
461 .smp_cpus_done = xen_smp_cpus_done,
462
463 .cpu_up = xen_cpu_up,
464 .cpu_die = xen_cpu_die,
465 .cpu_disable = xen_cpu_disable,
466 .play_dead = xen_play_dead,
467
468 .smp_send_stop = xen_smp_send_stop,
469 .smp_send_reschedule = xen_smp_send_reschedule,
470
471 .send_call_func_ipi = xen_smp_send_call_function_ipi,
472 .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
473};
474
475void __init xen_smp_init(void)
476{
477 smp_ops = xen_smp_ops;
478 xen_fill_possible_map();
479 xen_init_spinlocks();
480}
481
