1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/sched.h>
17#include <linux/sched/task_stack.h>
18#include <linux/err.h>
19#include <linux/slab.h>
20#include <linux/smp.h>
21#include <linux/irq_work.h>
22#include <linux/tick.h>
23#include <linux/nmi.h>
24#include <linux/cpuhotplug.h>
25#include <linux/stackprotector.h>
26#include <linux/pgtable.h>
27
28#include <asm/paravirt.h>
29#include <asm/idtentry.h>
30#include <asm/desc.h>
31#include <asm/cpu.h>
32
33#include <xen/interface/xen.h>
34#include <xen/interface/vcpu.h>
35#include <xen/interface/xenpmu.h>
36
37#include <asm/spec-ctrl.h>
38#include <asm/xen/interface.h>
39#include <asm/xen/hypercall.h>
40
41#include <xen/xen.h>
42#include <xen/page.h>
43#include <xen/events.h>
44
45#include <xen/hvc-console.h>
46#include "xen-ops.h"
47#include "mmu.h"
48#include "smp.h"
49#include "pmu.h"
50
51cpumask_var_t xen_cpu_initialized_map;
52
53static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
54static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 };
55
56static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
57void asm_cpu_bringup_and_idle(void);
58
59static void cpu_bringup(void)
60{
61 int cpu;
62
63 cr4_init();
64 cpu_init();
65 touch_softlockup_watchdog();
66 preempt_disable();
67
68
69 if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
70 xen_enable_sysenter();
71 xen_enable_syscall();
72 }
73 cpu = smp_processor_id();
74 smp_store_cpu_info(cpu);
75 cpu_data(cpu).x86_max_cores = 1;
76 set_cpu_sibling_map(cpu);
77
78 speculative_store_bypass_ht_init();
79
80 xen_setup_cpu_clockevents();
81
82 notify_cpu_starting(cpu);
83
84 set_cpu_online(cpu, true);
85
86 cpu_set_state_online(cpu);
87
88
89 local_irq_enable();
90}
91
92asmlinkage __visible void cpu_bringup_and_idle(void)
93{
94 cpu_bringup();
95 boot_init_stack_canary();
96 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
97 prevent_tail_call_optimization();
98}
99
100void xen_smp_intr_free_pv(unsigned int cpu)
101{
102 if (per_cpu(xen_irq_work, cpu).irq >= 0) {
103 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
104 per_cpu(xen_irq_work, cpu).irq = -1;
105 kfree(per_cpu(xen_irq_work, cpu).name);
106 per_cpu(xen_irq_work, cpu).name = NULL;
107 }
108
109 if (per_cpu(xen_pmu_irq, cpu).irq >= 0) {
110 unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL);
111 per_cpu(xen_pmu_irq, cpu).irq = -1;
112 kfree(per_cpu(xen_pmu_irq, cpu).name);
113 per_cpu(xen_pmu_irq, cpu).name = NULL;
114 }
115}
116
117int xen_smp_intr_init_pv(unsigned int cpu)
118{
119 int rc;
120 char *callfunc_name, *pmu_name;
121
122 callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
123 rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
124 cpu,
125 xen_irq_work_interrupt,
126 IRQF_PERCPU|IRQF_NOBALANCING,
127 callfunc_name,
128 NULL);
129 if (rc < 0)
130 goto fail;
131 per_cpu(xen_irq_work, cpu).irq = rc;
132 per_cpu(xen_irq_work, cpu).name = callfunc_name;
133
134 if (is_xen_pmu(cpu)) {
135 pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
136 rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
137 xen_pmu_irq_handler,
138 IRQF_PERCPU|IRQF_NOBALANCING,
139 pmu_name, NULL);
140 if (rc < 0)
141 goto fail;
142 per_cpu(xen_pmu_irq, cpu).irq = rc;
143 per_cpu(xen_pmu_irq, cpu).name = pmu_name;
144 }
145
146 return 0;
147
148 fail:
149 xen_smp_intr_free_pv(cpu);
150 return rc;
151}
152
153static void __init xen_fill_possible_map(void)
154{
155 int i, rc;
156
157 if (xen_initial_domain())
158 return;
159
160 for (i = 0; i < nr_cpu_ids; i++) {
161 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
162 if (rc >= 0) {
163 num_processors++;
164 set_cpu_possible(i, true);
165 }
166 }
167}
168
169static void __init xen_filter_cpu_maps(void)
170{
171 int i, rc;
172 unsigned int subtract = 0;
173
174 if (!xen_initial_domain())
175 return;
176
177 num_processors = 0;
178 disabled_cpus = 0;
179 for (i = 0; i < nr_cpu_ids; i++) {
180 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
181 if (rc >= 0) {
182 num_processors++;
183 set_cpu_possible(i, true);
184 } else {
185 set_cpu_possible(i, false);
186 set_cpu_present(i, false);
187 subtract++;
188 }
189 }
190#ifdef CONFIG_HOTPLUG_CPU
191
192
193
194
195
196
197
198
199 if (subtract)
200 nr_cpu_ids = nr_cpu_ids - subtract;
201#endif
202
203}
204
205static void __init xen_pv_smp_prepare_boot_cpu(void)
206{
207 BUG_ON(smp_processor_id() != 0);
208 native_smp_prepare_boot_cpu();
209
210 if (!xen_feature(XENFEAT_writable_page_tables))
211
212
213 make_lowmem_page_readwrite(xen_initial_gdt);
214
215#ifdef CONFIG_X86_32
216
217
218
219
220 loadsegment(ds, __USER_DS);
221 loadsegment(es, __USER_DS);
222#endif
223
224 xen_filter_cpu_maps();
225 xen_setup_vcpu_info_placement();
226
227
228
229
230
231
232
233 xen_init_spinlocks();
234}
235
236static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus)
237{
238 unsigned cpu;
239 unsigned int i;
240
241 if (skip_ioapic_setup) {
242 char *m = (max_cpus == 0) ?
243 "The nosmp parameter is incompatible with Xen; " \
244 "use Xen dom0_max_vcpus=1 parameter" :
245 "The noapic parameter is incompatible with Xen";
246
247 xen_raw_printk(m);
248 panic(m);
249 }
250 xen_init_lock_cpu(0);
251
252 smp_store_boot_cpu_info();
253 cpu_data(0).x86_max_cores = 1;
254
255 for_each_possible_cpu(i) {
256 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
257 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
258 zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL);
259 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
260 }
261 set_cpu_sibling_map(0);
262
263 speculative_store_bypass_ht_init();
264
265 xen_pmu_init(0);
266
267 if (xen_smp_intr_init(0) || xen_smp_intr_init_pv(0))
268 BUG();
269
270 if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
271 panic("could not allocate xen_cpu_initialized_map\n");
272
273 cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
274
275
276 while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
277 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
278 continue;
279 set_cpu_possible(cpu, false);
280 }
281
282 for_each_possible_cpu(cpu)
283 set_cpu_present(cpu, true);
284}
285
286static int
287cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
288{
289 struct vcpu_guest_context *ctxt;
290 struct desc_struct *gdt;
291 unsigned long gdt_mfn;
292
293
294 cpumask_set_cpu(cpu, cpu_callout_mask);
295 if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
296 return 0;
297
298 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
299 if (ctxt == NULL)
300 return -ENOMEM;
301
302 gdt = get_cpu_gdt_rw(cpu);
303
304#ifdef CONFIG_X86_32
305 ctxt->user_regs.fs = __KERNEL_PERCPU;
306 ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
307#endif
308 memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
309
310
311
312
313
314
315 ctxt->user_regs.eip = (unsigned long)asm_cpu_bringup_and_idle;
316 ctxt->flags = VGCF_IN_KERNEL;
317 ctxt->user_regs.eflags = 0x1000;
318 ctxt->user_regs.ds = __USER_DS;
319 ctxt->user_regs.es = __USER_DS;
320 ctxt->user_regs.ss = __KERNEL_DS;
321 ctxt->user_regs.cs = __KERNEL_CS;
322 ctxt->user_regs.esp = (unsigned long)task_pt_regs(idle);
323
324 xen_copy_trap_info(ctxt->trap_ctxt);
325
326 ctxt->ldt_ents = 0;
327
328 BUG_ON((unsigned long)gdt & ~PAGE_MASK);
329
330 gdt_mfn = arbitrary_virt_to_mfn(gdt);
331 make_lowmem_page_readonly(gdt);
332 make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
333
334 ctxt->gdt_frames[0] = gdt_mfn;
335 ctxt->gdt_ents = GDT_ENTRIES;
336
337
338
339
340
341
342 ctxt->kernel_ss = __KERNEL_DS;
343 ctxt->kernel_sp = task_top_of_stack(idle);
344
345#ifdef CONFIG_X86_32
346 ctxt->event_callback_cs = __KERNEL_CS;
347 ctxt->failsafe_callback_cs = __KERNEL_CS;
348#else
349 ctxt->gs_base_kernel = per_cpu_offset(cpu);
350#endif
351 ctxt->event_callback_eip =
352 (unsigned long)xen_asm_exc_xen_hypervisor_callback;
353 ctxt->failsafe_callback_eip =
354 (unsigned long)xen_failsafe_callback;
355 per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
356
357 ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
358 if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt))
359 BUG();
360
361 kfree(ctxt);
362 return 0;
363}
364
365static int xen_pv_cpu_up(unsigned int cpu, struct task_struct *idle)
366{
367 int rc;
368
369 rc = common_cpu_up(cpu, idle);
370 if (rc)
371 return rc;
372
373 xen_setup_runstate_info(cpu);
374
375
376
377
378
379 rc = cpu_check_up_prepare(cpu);
380 if (rc)
381 return rc;
382
383
384 per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
385
386 rc = cpu_initialize_context(cpu, idle);
387 if (rc)
388 return rc;
389
390 xen_pmu_init(cpu);
391
392 rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL);
393 BUG_ON(rc);
394
395 while (cpu_report_state(cpu) != CPU_ONLINE)
396 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
397
398 return 0;
399}
400
401#ifdef CONFIG_HOTPLUG_CPU
402static int xen_pv_cpu_disable(void)
403{
404 unsigned int cpu = smp_processor_id();
405 if (cpu == 0)
406 return -EBUSY;
407
408 cpu_disable_common();
409
410 load_cr3(swapper_pg_dir);
411 return 0;
412}
413
414static void xen_pv_cpu_die(unsigned int cpu)
415{
416 while (HYPERVISOR_vcpu_op(VCPUOP_is_up,
417 xen_vcpu_nr(cpu), NULL)) {
418 __set_current_state(TASK_UNINTERRUPTIBLE);
419 schedule_timeout(HZ/10);
420 }
421
422 if (common_cpu_die(cpu) == 0) {
423 xen_smp_intr_free(cpu);
424 xen_uninit_lock_cpu(cpu);
425 xen_teardown_timer(cpu);
426 xen_pmu_finish(cpu);
427 }
428}
429
430static void xen_pv_play_dead(void)
431{
432 play_dead_common();
433 HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
434 cpu_bringup();
435
436
437
438
439
440
441 tick_nohz_idle_enter();
442 tick_nohz_idle_stop_tick_protected();
443
444 cpuhp_online_idle(CPUHP_AP_ONLINE_IDLE);
445}
446
447#else
448static int xen_pv_cpu_disable(void)
449{
450 return -ENOSYS;
451}
452
453static void xen_pv_cpu_die(unsigned int cpu)
454{
455 BUG();
456}
457
458static void xen_pv_play_dead(void)
459{
460 BUG();
461}
462
463#endif
464static void stop_self(void *v)
465{
466 int cpu = smp_processor_id();
467
468
469 load_cr3(swapper_pg_dir);
470
471
472 set_cpu_online(cpu, false);
473
474 HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL);
475 BUG();
476}
477
478static void xen_pv_stop_other_cpus(int wait)
479{
480 smp_call_function(stop_self, NULL, wait);
481}
482
483static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
484{
485 irq_enter();
486 irq_work_run();
487 inc_irq_stat(apic_irq_work_irqs);
488 irq_exit();
489
490 return IRQ_HANDLED;
491}
492
493static const struct smp_ops xen_smp_ops __initconst = {
494 .smp_prepare_boot_cpu = xen_pv_smp_prepare_boot_cpu,
495 .smp_prepare_cpus = xen_pv_smp_prepare_cpus,
496 .smp_cpus_done = xen_smp_cpus_done,
497
498 .cpu_up = xen_pv_cpu_up,
499 .cpu_die = xen_pv_cpu_die,
500 .cpu_disable = xen_pv_cpu_disable,
501 .play_dead = xen_pv_play_dead,
502
503 .stop_other_cpus = xen_pv_stop_other_cpus,
504 .smp_send_reschedule = xen_smp_send_reschedule,
505
506 .send_call_func_ipi = xen_smp_send_call_function_ipi,
507 .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
508};
509
510void __init xen_smp_init(void)
511{
512 smp_ops = xen_smp_ops;
513 xen_fill_possible_map();
514}
515