1/*P:010 2 * A hypervisor allows multiple Operating Systems to run on a single machine. 3 * To quote David Wheeler: "Any problem in computer science can be solved with 4 * another layer of indirection." 5 * 6 * We keep things simple in two ways. First, we start with a normal Linux 7 * kernel and insert a module (lg.ko) which allows us to run other Linux 8 * kernels the same way we'd run processes. We call the first kernel the Host, 9 * and the others the Guests. The program which sets up and configures Guests 10 * (such as the example in Documentation/lguest/lguest.c) is called the 11 * Launcher. 12 * 13 * Secondly, we only run specially modified Guests, not normal kernels: setting 14 * CONFIG_LGUEST_GUEST to "y" compiles this file into the kernel so it knows 15 * how to be a Guest at boot time. This means that you can use the same kernel 16 * you boot normally (ie. as a Host) as a Guest. 17 * 18 * These Guests know that they cannot do privileged operations, such as disable 19 * interrupts, and that they have to ask the Host to do such things explicitly. 20 * This file consists of all the replacements for such low-level native 21 * hardware operations: these special Guest versions call the Host. 22 * 23 * So how does the kernel know it's a Guest? We'll see that later, but let's 24 * just say that we end up here where we replace the native functions various 25 * "paravirt" structures with our Guest versions, then boot like normal. 26:*/ 27 28/* 29 * Copyright (C) 2006, Rusty Russell <rusty@rustcorp.com.au> IBM Corporation. 30 * 31 * This program is free software; you can redistribute it and/or modify 32 * it under the terms of the GNU General Public License as published by 33 * the Free Software Foundation; either version 2 of the License, or 34 * (at your option) any later version. 35 * 36 * This program is distributed in the hope that it will be useful, but 37 * WITHOUT ANY WARRANTY; without even the implied warranty of 38 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 39 * NON INFRINGEMENT. See the GNU General Public License for more 40 * details. 41 * 42 * You should have received a copy of the GNU General Public License 43 * along with this program; if not, write to the Free Software 44 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 45 */ 46#include <linux/kernel.h> 47#include <linux/start_kernel.h> 48#include <linux/string.h> 49#include <linux/console.h> 50#include <linux/screen_info.h> 51#include <linux/irq.h> 52#include <linux/interrupt.h> 53#include <linux/clocksource.h> 54#include <linux/clockchips.h> 55#include <linux/lguest.h> 56#include <linux/lguest_launcher.h> 57#include <linux/virtio_console.h> 58#include <linux/pm.h> 59#include <asm/apic.h> 60#include <asm/lguest.h> 61#include <asm/paravirt.h> 62#include <asm/param.h> 63#include <asm/page.h> 64#include <asm/pgtable.h> 65#include <asm/desc.h> 66#include <asm/setup.h> 67#include <asm/e820.h> 68#include <asm/mce.h> 69#include <asm/io.h> 70#include <asm/i387.h> 71#include <asm/stackprotector.h> 72#include <asm/reboot.h> /* for struct machine_ops */ 73 74/*G:010 Welcome to the Guest! 75 * 76 * The Guest in our tale is a simple creature: identical to the Host but 77 * behaving in simplified but equivalent ways. In particular, the Guest is the 78 * same kernel as the Host (or at least, built from the same source code). 79:*/ 80 81struct lguest_data lguest_data = { 82 .hcall_status = { [0 ... LHCALL_RING_SIZE-1] = 0xFF }, 83 .noirq_start = (u32)lguest_noirq_start, 84 .noirq_end = (u32)lguest_noirq_end, 85 .kernel_address = PAGE_OFFSET, 86 .blocked_interrupts = { 1 }, /* Block timer interrupts */ 87 .syscall_vec = SYSCALL_VECTOR, 88}; 89 90/*G:037 91 * async_hcall() is pretty simple: I'm quite proud of it really. We have a 92 * ring buffer of stored hypercalls which the Host will run though next time we 93 * do a normal hypercall. Each entry in the ring has 5 slots for the hypercall 94 * arguments, and a "hcall_status" word which is 0 if the call is ready to go, 95 * and 255 once the Host has finished with it. 96 * 97 * If we come around to a slot which hasn't been finished, then the table is 98 * full and we just make the hypercall directly. This has the nice side 99 * effect of causing the Host to run all the stored calls in the ring buffer 100 * which empties it for next time! 101 */ 102static void async_hcall(unsigned long call, unsigned long arg1, 103 unsigned long arg2, unsigned long arg3, 104 unsigned long arg4) 105{ 106 /* Note: This code assumes we're uniprocessor. */ 107 static unsigned int next_call; 108 unsigned long flags; 109 110 /* 111 * Disable interrupts if not already disabled: we don't want an 112 * interrupt handler making a hypercall while we're already doing 113 * one! 114 */ 115 local_irq_save(flags); 116 if (lguest_data.hcall_status[next_call] != 0xFF) { 117 /* Table full, so do normal hcall which will flush table. */ 118 kvm_hypercall4(call, arg1, arg2, arg3, arg4); 119 } else { 120 lguest_data.hcalls[next_call].arg0 = call; 121 lguest_data.hcalls[next_call].arg1 = arg1; 122 lguest_data.hcalls[next_call].arg2 = arg2; 123 lguest_data.hcalls[next_call].arg3 = arg3; 124 lguest_data.hcalls[next_call].arg4 = arg4; 125 /* Arguments must all be written before we mark it to go */ 126 wmb(); 127 lguest_data.hcall_status[next_call] = 0; 128 if (++next_call == LHCALL_RING_SIZE) 129 next_call = 0; 130 } 131 local_irq_restore(flags); 132} 133 134/*G:035 135 * Notice the lazy_hcall() above, rather than hcall(). This is our first real 136 * optimization trick! 137 * 138 * When lazy_mode is set, it means we're allowed to defer all hypercalls and do 139 * them as a batch when lazy_mode is eventually turned off. Because hypercalls 140 * are reasonably expensive, batching them up makes sense. For example, a 141 * large munmap might update dozens of page table entries: that code calls 142 * paravirt_enter_lazy_mmu(), does the dozen updates, then calls 143 * lguest_leave_lazy_mode(). 144 * 145 * So, when we're in lazy mode, we call async_hcall() to store the call for 146 * future processing: 147 */ 148static void lazy_hcall1(unsigned long call, 149 unsigned long arg1) 150{ 151 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) 152 kvm_hypercall1(call, arg1); 153 else 154 async_hcall(call, arg1, 0, 0, 0); 155} 156 157/* You can imagine what lazy_hcall2, 3 and 4 look like. :*/ 158static void lazy_hcall2(unsigned long call, 159 unsigned long arg1, 160 unsigned long arg2) 161{ 162 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) 163 kvm_hypercall2(call, arg1, arg2); 164 else 165 async_hcall(call, arg1, arg2, 0, 0); 166} 167 168static void lazy_hcall3(unsigned long call, 169 unsigned long arg1, 170 unsigned long arg2, 171 unsigned long arg3) 172{ 173 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) 174 kvm_hypercall3(call, arg1, arg2, arg3); 175 else 176 async_hcall(call, arg1, arg2, arg3, 0); 177} 178 179#ifdef CONFIG_X86_PAE 180static void lazy_hcall4(unsigned long call, 181 unsigned long arg1, 182 unsigned long arg2, 183 unsigned long arg3, 184 unsigned long arg4) 185{ 186 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) 187 kvm_hypercall4(call, arg1, arg2, arg3, arg4); 188 else 189 async_hcall(call, arg1, arg2, arg3, arg4); 190} 191#endif 192 193/*G:036 194 * When lazy mode is turned off reset the per-cpu lazy mode variable and then 195 * issue the do-nothing hypercall to flush any stored calls. 196:*/ 197static void lguest_leave_lazy_mmu_mode(void) 198{ 199 kvm_hypercall0(LHCALL_FLUSH_ASYNC); 200 paravirt_leave_lazy_mmu(); 201} 202 203static void lguest_end_context_switch(struct task_struct *next) 204{ 205 kvm_hypercall0(LHCALL_FLUSH_ASYNC); 206 paravirt_end_context_switch(next); 207} 208 209/*G:032 210 * After that diversion we return to our first native-instruction 211 * replacements: four functions for interrupt control. 212 * 213 * The simplest way of implementing these would be to have "turn interrupts 214 * off" and "turn interrupts on" hypercalls. Unfortunately, this is too slow: 215 * these are by far the most commonly called functions of those we override. 216 * 217 * So instead we keep an "irq_enabled" field inside our "struct lguest_data", 218 * which the Guest can update with a single instruction. The Host knows to 219 * check there before it tries to deliver an interrupt. 220 */ 221 222/* 223 * save_flags() is expected to return the processor state (ie. "flags"). The 224 * flags word contains all kind of stuff, but in practice Linux only cares 225 * about the interrupt flag. Our "save_flags()" just returns that. 226 */ 227static unsigned long save_fl(void) 228{ 229 return lguest_data.irq_enabled; 230} 231 232/* Interrupts go off... */ 233static void irq_disable(void) 234{ 235 lguest_data.irq_enabled = 0; 236} 237 238/* 239 * Let's pause a moment. Remember how I said these are called so often? 240 * Jeremy Fitzhardinge optimized them so hard early in 2009 that he had to 241 * break some rules. In particular, these functions are assumed to save their 242 * own registers if they need to: normal C functions assume they can trash the 243 * eax register. To use normal C functions, we use 244 * PV_CALLEE_SAVE_REGS_THUNK(), which pushes %eax onto the stack, calls the 245 * C function, then restores it. 246 */ 247PV_CALLEE_SAVE_REGS_THUNK(save_fl); 248PV_CALLEE_SAVE_REGS_THUNK(irq_disable); 249/*:*/ 250 251/* These are in i386_head.S */ 252extern void lg_irq_enable(void); 253extern void lg_restore_fl(unsigned long flags); 254 255/*M:003 256 * We could be more efficient in our checking of outstanding interrupts, rather 257 * than using a branch. One way would be to put the "irq_enabled" field in a 258 * page by itself, and have the Host write-protect it when an interrupt comes 259 * in when irqs are disabled. There will then be a page fault as soon as 260 * interrupts are re-enabled. 261 * 262 * A better method is to implement soft interrupt disable generally for x86: 263 * instead of disabling interrupts, we set a flag. If an interrupt does come 264 * in, we then disable them for real. This is uncommon, so we could simply use 265 * a hypercall for interrupt control and not worry about efficiency. 266:*/ 267 268/*G:034 269 * The Interrupt Descriptor Table (IDT). 270 * 271 * The IDT tells the processor what to do when an interrupt comes in. Each 272 * entry in the table is a 64-bit descriptor: this holds the privilege level, 273 * address of the handler, and... well, who cares? The Guest just asks the 274 * Host to make the change anyway, because the Host controls the real IDT. 275 */ 276static void lguest_write_idt_entry(gate_desc *dt, 277 int entrynum, const gate_desc *g) 278{ 279 /* 280 * The gate_desc structure is 8 bytes long: we hand it to the Host in 281 * two 32-bit chunks. The whole 32-bit kernel used to hand descriptors 282 * around like this; typesafety wasn't a big concern in Linux's early 283 * years. 284 */ 285 u32 *desc = (u32 *)g; 286 /* Keep the local copy up to date. */ 287 native_write_idt_entry(dt, entrynum, g); 288 /* Tell Host about this new entry. */ 289 kvm_hypercall3(LHCALL_LOAD_IDT_ENTRY, entrynum, desc[0], desc[1]); 290} 291 292/* 293 * Changing to a different IDT is very rare: we keep the IDT up-to-date every 294 * time it is written, so we can simply loop through all entries and tell the 295 * Host about them. 296 */ 297static void lguest_load_idt(const struct desc_ptr *desc) 298{ 299 unsigned int i; 300 struct desc_struct *idt = (void *)desc->address; 301 302 for (i = 0; i < (desc->size+1)/8; i++) 303 kvm_hypercall3(LHCALL_LOAD_IDT_ENTRY, i, idt[i].a, idt[i].b); 304} 305 306/* 307 * The Global Descriptor Table. 308 * 309 * The Intel architecture defines another table, called the Global Descriptor 310 * Table (GDT). You tell the CPU where it is (and its size) using the "lgdt" 311 * instruction, and then several other instructions refer to entries in the 312 * table. There are three entries which the Switcher needs, so the Host simply 313 * controls the entire thing and the Guest asks it to make changes using the 314 * LOAD_GDT hypercall. 315 * 316 * This is the exactly like the IDT code. 317 */ 318static void lguest_load_gdt(const struct desc_ptr *desc) 319{ 320 unsigned int i; 321 struct desc_struct *gdt = (void *)desc->address; 322 323 for (i = 0; i < (desc->size+1)/8; i++) 324 kvm_hypercall3(LHCALL_LOAD_GDT_ENTRY, i, gdt[i].a, gdt[i].b); 325} 326 327/* 328 * For a single GDT entry which changes, we do the lazy thing: alter our GDT, 329 * then tell the Host to reload the entire thing. This operation is so rare 330 * that this naive implementation is reasonable. 331 */ 332static void lguest_write_gdt_entry(struct desc_struct *dt, int entrynum, 333 const void *desc, int type) 334{ 335 native_write_gdt_entry(dt, entrynum, desc, type); 336 /* Tell Host about this new entry. */ 337 kvm_hypercall3(LHCALL_LOAD_GDT_ENTRY, entrynum, 338 dt[entrynum].a, dt[entrynum].b); 339} 340 341/* 342 * OK, I lied. There are three "thread local storage" GDT entries which change 343 * on every context switch (these three entries are how glibc implements 344 * __thread variables). So we have a hypercall specifically for this case. 345 */ 346static void lguest_load_tls(struct thread_struct *t, unsigned int cpu) 347{ 348 /* 349 * There's one problem which normal hardware doesn't have: the Host 350 * can't handle us removing entries we're currently using. So we clear 351 * the GS register here: if it's needed it'll be reloaded anyway. 352 */ 353 lazy_load_gs(0); 354 lazy_hcall2(LHCALL_LOAD_TLS, __pa(&t->tls_array), cpu); 355} 356 357/*G:038 358 * That's enough excitement for now, back to ploughing through each of the 359 * different pv_ops structures (we're about 1/3 of the way through). 360 * 361 * This is the Local Descriptor Table, another weird Intel thingy. Linux only 362 * uses this for some strange applications like Wine. We don't do anything 363 * here, so they'll get an informative and friendly Segmentation Fault. 364 */ 365static void lguest_set_ldt(const void *addr, unsigned entries) 366{ 367} 368 369/* 370 * This loads a GDT entry into the "Task Register": that entry points to a 371 * structure called the Task State Segment. Some comments scattered though the 372 * kernel code indicate that this used for task switching in ages past, along 373 * with blood sacrifice and astrology. 374 * 375 * Now there's nothing interesting in here that we don't get told elsewhere. 376 * But the native version uses the "ltr" instruction, which makes the Host 377 * complain to the Guest about a Segmentation Fault and it'll oops. So we 378 * override the native version with a do-nothing version. 379 */ 380static void lguest_load_tr_desc(void) 381{ 382} 383 384/* 385 * The "cpuid" instruction is a way of querying both the CPU identity 386 * (manufacturer, model, etc) and its features. It was introduced before the 387 * Pentium in 1993 and keeps getting extended by both Intel, AMD and others. 388 * As you might imagine, after a decade and a half this treatment, it is now a 389 * giant ball of hair. Its entry in the current Intel manual runs to 28 pages. 390 * 391 * This instruction even it has its own Wikipedia entry. The Wikipedia entry 392 * has been translated into 5 languages. I am not making this up! 393 * 394 * We could get funky here and identify ourselves as "GenuineLguest", but 395 * instead we just use the real "cpuid" instruction. Then I pretty much turned 396 * off feature bits until the Guest booted. (Don't say that: you'll damage 397 * lguest sales!) Shut up, inner voice! (Hey, just pointing out that this is 398 * hardly future proof.) Noone's listening! They don't like you anyway, 399 * parenthetic weirdo! 400 * 401 * Replacing the cpuid so we can turn features off is great for the kernel, but 402 * anyone (including userspace) can just use the raw "cpuid" instruction and 403 * the Host won't even notice since it isn't privileged. So we try not to get 404 * too worked up about it. 405 */ 406static void lguest_cpuid(unsigned int *ax, unsigned int *bx, 407 unsigned int *cx, unsigned int *dx) 408{ 409 int function = *ax; 410 411 native_cpuid(ax, bx, cx, dx); 412 switch (function) { 413 /* 414 * CPUID 0 gives the highest legal CPUID number (and the ID string). 415 * We futureproof our code a little by sticking to known CPUID values. 416 */ 417 case 0: 418 if (*ax > 5) 419 *ax = 5; 420 break; 421 422 /* 423 * CPUID 1 is a basic feature request. 424 * 425 * CX: we only allow kernel to see SSE3, CMPXCHG16B and SSSE3 426 * DX: SSE, SSE2, FXSR, MMX, CMOV, CMPXCHG8B, TSC, FPU and PAE. 427 */ 428 case 1: 429 *cx &= 0x00002201; 430 *dx &= 0x07808151; 431 /* 432 * The Host can do a nice optimization if it knows that the 433 * kernel mappings (addresses above 0xC0000000 or whatever 434 * PAGE_OFFSET is set to) haven't changed. But Linux calls 435 * flush_tlb_user() for both user and kernel mappings unless 436 * the Page Global Enable (PGE) feature bit is set. 437 */ 438 *dx |= 0x00002000; 439 /* 440 * We also lie, and say we're family id 5. 6 or greater 441 * leads to a rdmsr in early_init_intel which we can't handle. 442 * Family ID is returned as bits 8-12 in ax. 443 */ 444 *ax &= 0xFFFFF0FF; 445 *ax |= 0x00000500; 446 break; 447 /* 448 * 0x80000000 returns the highest Extended Function, so we futureproof 449 * like we do above by limiting it to known fields. 450 */ 451 case 0x80000000: 452 if (*ax > 0x80000008) 453 *ax = 0x80000008; 454 break; 455 456 /* 457 * PAE systems can mark pages as non-executable. Linux calls this the 458 * NX bit. Intel calls it XD (eXecute Disable), AMD EVP (Enhanced 459 * Virus Protection). We just switch turn if off here, since we don't 460 * support it. 461 */ 462 case 0x80000001: 463 *dx &= ~(1 << 20); 464 break; 465 } 466} 467 468/* 469 * Intel has four control registers, imaginatively named cr0, cr2, cr3 and cr4. 470 * I assume there's a cr1, but it hasn't bothered us yet, so we'll not bother 471 * it. The Host needs to know when the Guest wants to change them, so we have 472 * a whole series of functions like read_cr0() and write_cr0(). 473 * 474 * We start with cr0. cr0 allows you to turn on and off all kinds of basic 475 * features, but Linux only really cares about one: the horrifically-named Task 476 * Switched (TS) bit at bit 3 (ie. 8) 477 * 478 * What does the TS bit do? Well, it causes the CPU to trap (interrupt 7) if 479 * the floating point unit is used. Which allows us to restore FPU state 480 * lazily after a task switch, and Linux uses that gratefully, but wouldn't a 481 * name like "FPUTRAP bit" be a little less cryptic? 482 * 483 * We store cr0 locally because the Host never changes it. The Guest sometimes 484 * wants to read it and we'd prefer not to bother the Host unnecessarily. 485 */ 486static unsigned long current_cr0; 487static void lguest_write_cr0(unsigned long val) 488{ 489 lazy_hcall1(LHCALL_TS, val & X86_CR0_TS); 490 current_cr0 = val; 491} 492 493static unsigned long lguest_read_cr0(void) 494{ 495 return current_cr0; 496} 497 498/* 499 * Intel provided a special instruction to clear the TS bit for people too cool 500 * to use write_cr0() to do it. This "clts" instruction is faster, because all 501 * the vowels have been optimized out. 502 */ 503static void lguest_clts(void) 504{ 505 lazy_hcall1(LHCALL_TS, 0); 506 current_cr0 &= ~X86_CR0_TS; 507} 508 509/* 510 * cr2 is the virtual address of the last page fault, which the Guest only ever 511 * reads. The Host kindly writes this into our "struct lguest_data", so we 512 * just read it out of there. 513 */ 514static unsigned long lguest_read_cr2(void) 515{ 516 return lguest_data.cr2; 517} 518 519/* See lguest_set_pte() below. */ 520static bool cr3_changed = false; 521 522/* 523 * cr3 is the current toplevel pagetable page: the principle is the same as 524 * cr0. Keep a local copy, and tell the Host when it changes. The only 525 * difference is that our local copy is in lguest_data because the Host needs 526 * to set it upon our initial hypercall. 527 */ 528static void lguest_write_cr3(unsigned long cr3) 529{ 530 lguest_data.pgdir = cr3; 531 lazy_hcall1(LHCALL_NEW_PGTABLE, cr3); 532 cr3_changed = true; 533} 534 535static unsigned long lguest_read_cr3(void) 536{ 537 return lguest_data.pgdir; 538} 539 540/* cr4 is used to enable and disable PGE, but we don't care. */ 541static unsigned long lguest_read_cr4(void) 542{ 543 return 0; 544} 545 546static void lguest_write_cr4(unsigned long val) 547{ 548} 549 550/* 551 * Page Table Handling. 552 * 553 * Now would be a good time to take a rest and grab a coffee or similarly 554 * relaxing stimulant. The easy parts are behind us, and the trek gradually 555 * winds uphill from here. 556 * 557 * Quick refresher: memory is divided into "pages" of 4096 bytes each. The CPU 558 * maps virtual addresses to physical addresses using "page tables". We could 559 * use one huge index of 1 million entries: each address is 4 bytes, so that's 560 * 1024 pages just to hold the page tables. But since most virtual addresses 561 * are unused, we use a two level index which saves space. The cr3 register 562 * contains the physical address of the top level "page directory" page, which 563 * contains physical addresses of up to 1024 second-level pages. Each of these 564 * second level pages contains up to 1024 physical addresses of actual pages, 565 * or Page Table Entries (PTEs). 566 * 567 * Here's a diagram, where arrows indicate physical addresses: 568 * 569 * cr3 ---> +---------+ 570 * | --------->+---------+ 571 * | | | PADDR1 | 572 * Mid-level | | PADDR2 | 573 * (PMD) page | | | 574 * | | Lower-level | 575 * | | (PTE) page | 576 * | | | | 577 * .... .... 578 * 579 * So to convert a virtual address to a physical address, we look up the top 580 * level, which points us to the second level, which gives us the physical 581 * address of that page. If the top level entry was not present, or the second 582 * level entry was not present, then the virtual address is invalid (we 583 * say "the page was not mapped"). 584 * 585 * Put another way, a 32-bit virtual address is divided up like so: 586 * 587 * 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 588 * |<---- 10 bits ---->|<---- 10 bits ---->|<------ 12 bits ------>| 589 * Index into top Index into second Offset within page 590 * page directory page pagetable page 591 * 592 * Now, unfortunately, this isn't the whole story: Intel added Physical Address 593 * Extension (PAE) to allow 32 bit systems to use 64GB of memory (ie. 36 bits). 594 * These are held in 64-bit page table entries, so we can now only fit 512 595 * entries in a page, and the neat three-level tree breaks down. 596 * 597 * The result is a four level page table: 598 * 599 * cr3 --> [ 4 Upper ] 600 * [ Level ] 601 * [ Entries ] 602 * [(PUD Page)]---> +---------+ 603 * | --------->+---------+ 604 * | | | PADDR1 | 605 * Mid-level | | PADDR2 | 606 * (PMD) page | | | 607 * | | Lower-level | 608 * | | (PTE) page | 609 * | | | | 610 * .... .... 611 * 612 * 613 * And the virtual address is decoded as: 614 * 615 * 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 616 * |<-2->|<--- 9 bits ---->|<---- 9 bits --->|<------ 12 bits ------>| 617 * Index into Index into mid Index into lower Offset within page 618 * top entries directory page pagetable page 619 * 620 * It's too hard to switch between these two formats at runtime, so Linux only 621 * supports one or the other depending on whether CONFIG_X86_PAE is set. Many 622 * distributions turn it on, and not just for people with silly amounts of 623 * memory: the larger PTE entries allow room for the NX bit, which lets the 624 * kernel disable execution of pages and increase security. 625 * 626 * This was a problem for lguest, which couldn't run on these distributions; 627 * then Matias Zabaljauregui figured it all out and implemented it, and only a 628 * handful of puppies were crushed in the process! 629 * 630 * Back to our point: the kernel spends a lot of time changing both the 631 * top-level page directory and lower-level pagetable pages. The Guest doesn't 632 * know physical addresses, so while it maintains these page tables exactly 633 * like normal, it also needs to keep the Host informed whenever it makes a 634 * change: the Host will create the real page tables based on the Guests'. 635 */ 636 637/* 638 * The Guest calls this after it has set a second-level entry (pte), ie. to map 639 * a page into a process' address space. Wetell the Host the toplevel and 640 * address this corresponds to. The Guest uses one pagetable per process, so 641 * we need to tell the Host which one we're changing (mm->pgd). 642 */ 643static void lguest_pte_update(struct mm_struct *mm, unsigned long addr, 644 pte_t *ptep) 645{ 646#ifdef CONFIG_X86_PAE 647 /* PAE needs to hand a 64 bit page table entry, so it uses two args. */ 648 lazy_hcall4(LHCALL_SET_PTE, __pa(mm->pgd), addr, 649 ptep->pte_low, ptep->pte_high); 650#else 651 lazy_hcall3(LHCALL_SET_PTE, __pa(mm->pgd), addr, ptep->pte_low); 652#endif 653} 654 655/* This is the "set and update" combo-meal-deal version. */ 656static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr, 657 pte_t *ptep, pte_t pteval) 658{ 659 native_set_pte(ptep, pteval); 660 lguest_pte_update(mm, addr, ptep); 661} 662 663/* 664 * The Guest calls lguest_set_pud to set a top-level entry and lguest_set_pmd 665 * to set a middle-level entry when PAE is activated. 666 * 667 * Again, we set the entry then tell the Host which page we changed, 668 * and the index of the entry we changed. 669 */ 670#ifdef CONFIG_X86_PAE 671static void lguest_set_pud(pud_t *pudp, pud_t pudval) 672{ 673 native_set_pud(pudp, pudval); 674 675 /* 32 bytes aligned pdpt address and the index. */ 676 lazy_hcall2(LHCALL_SET_PGD, __pa(pudp) & 0xFFFFFFE0, 677 (__pa(pudp) & 0x1F) / sizeof(pud_t)); 678} 679 680static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) 681{ 682 native_set_pmd(pmdp, pmdval); 683 lazy_hcall2(LHCALL_SET_PMD, __pa(pmdp) & PAGE_MASK, 684 (__pa(pmdp) & (PAGE_SIZE - 1)) / sizeof(pmd_t)); 685} 686#else 687 688/* The Guest calls lguest_set_pmd to set a top-level entry when !PAE. */ 689static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) 690{ 691 native_set_pmd(pmdp, pmdval); 692 lazy_hcall2(LHCALL_SET_PGD, __pa(pmdp) & PAGE_MASK, 693 (__pa(pmdp) & (PAGE_SIZE - 1)) / sizeof(pmd_t)); 694} 695#endif 696 697/* 698 * There are a couple of legacy places where the kernel sets a PTE, but we 699 * don't know the top level any more. This is useless for us, since we don't 700 * know which pagetable is changing or what address, so we just tell the Host 701 * to forget all of them. Fortunately, this is very rare. 702 * 703 * ... except in early boot when the kernel sets up the initial pagetables, 704 * which makes booting astonishingly slow: 1.83 seconds! So we don't even tell 705 * the Host anything changed until we've done the first page table switch, 706 * which brings boot back to 0.25 seconds. 707 */ 708static void lguest_set_pte(pte_t *ptep, pte_t pteval) 709{ 710 native_set_pte(ptep, pteval); 711 if (cr3_changed) 712 lazy_hcall1(LHCALL_FLUSH_TLB, 1); 713} 714 715#ifdef CONFIG_X86_PAE 716/* 717 * With 64-bit PTE values, we need to be careful setting them: if we set 32 718 * bits at a time, the hardware could see a weird half-set entry. These 719 * versions ensure we update all 64 bits at once. 720 */ 721static void lguest_set_pte_atomic(pte_t *ptep, pte_t pte) 722{ 723 native_set_pte_atomic(ptep, pte); 724 if (cr3_changed) 725 lazy_hcall1(LHCALL_FLUSH_TLB, 1); 726} 727 728static void lguest_pte_clear(struct mm_struct *mm, unsigned long addr, 729 pte_t *ptep) 730{ 731 native_pte_clear(mm, addr, ptep); 732 lguest_pte_update(mm, addr, ptep); 733} 734 735static void lguest_pmd_clear(pmd_t *pmdp) 736{ 737 lguest_set_pmd(pmdp, __pmd(0)); 738} 739#endif 740 741/* 742 * Unfortunately for Lguest, the pv_mmu_ops for page tables were based on 743 * native page table operations. On native hardware you can set a new page 744 * table entry whenever you want, but if you want to remove one you have to do 745 * a TLB flush (a TLB is a little cache of page table entries kept by the CPU). 746 * 747 * So the lguest_set_pte_at() and lguest_set_pmd() functions above are only 748 * called when a valid entry is written, not when it's removed (ie. marked not 749 * present). Instead, this is where we come when the Guest wants to remove a 750 * page table entry: we tell the Host to set that entry to 0 (ie. the present 751 * bit is zero). 752 */ 753static void lguest_flush_tlb_single(unsigned long addr) 754{ 755 /* Simply set it to zero: if it was not, it will fault back in. */ 756 lazy_hcall3(LHCALL_SET_PTE, lguest_data.pgdir, addr, 0); 757} 758 759/* 760 * This is what happens after the Guest has removed a large number of entries. 761 * This tells the Host that any of the page table entries for userspace might 762 * have changed, ie. virtual addresses below PAGE_OFFSET. 763 */ 764static void lguest_flush_tlb_user(void) 765{ 766 lazy_hcall1(LHCALL_FLUSH_TLB, 0); 767} 768 769/* 770 * This is called when the kernel page tables have changed. That's not very 771 * common (unless the Guest is using highmem, which makes the Guest extremely 772 * slow), so it's worth separating this from the user flushing above. 773 */ 774static void lguest_flush_tlb_kernel(void) 775{ 776 lazy_hcall1(LHCALL_FLUSH_TLB, 1); 777} 778 779/* 780 * The Unadvanced Programmable Interrupt Controller. 781 * 782 * This is an attempt to implement the simplest possible interrupt controller. 783 * I spent some time looking though routines like set_irq_chip_and_handler, 784 * set_irq_chip_and_handler_name, set_irq_chip_data and set_phasers_to_stun and 785 * I *think* this is as simple as it gets. 786 * 787 * We can tell the Host what interrupts we want blocked ready for using the 788 * lguest_data.interrupts bitmap, so disabling (aka "masking") them is as 789 * simple as setting a bit. We don't actually "ack" interrupts as such, we 790 * just mask and unmask them. I wonder if we should be cleverer? 791 */ 792static void disable_lguest_irq(unsigned int irq) 793{ 794 set_bit(irq, lguest_data.blocked_interrupts); 795} 796 797static void enable_lguest_irq(unsigned int irq) 798{ 799 clear_bit(irq, lguest_data.blocked_interrupts); 800} 801 802/* This structure describes the lguest IRQ controller. */ 803static struct irq_chip lguest_irq_controller = { 804 .name = "lguest", 805 .mask = disable_lguest_irq, 806 .mask_ack = disable_lguest_irq, 807 .unmask = enable_lguest_irq, 808}; 809 810/* 811 * This sets up the Interrupt Descriptor Table (IDT) entry for each hardware 812 * interrupt (except 128, which is used for system calls), and then tells the 813 * Linux infrastructure that each interrupt is controlled by our level-based 814 * lguest interrupt controller. 815 */ 816static void __init lguest_init_IRQ(void) 817{ 818 unsigned int i; 819 820 for (i = FIRST_EXTERNAL_VECTOR; i < NR_VECTORS; i++) { 821 /* Some systems map "vectors" to interrupts weirdly. Not us! */ 822 __get_cpu_var(vector_irq)[i] = i - FIRST_EXTERNAL_VECTOR; 823 if (i != SYSCALL_VECTOR) 824 set_intr_gate(i, interrupt[i - FIRST_EXTERNAL_VECTOR]); 825 } 826 827 /* 828 * This call is required to set up for 4k stacks, where we have 829 * separate stacks for hard and soft interrupts. 830 */ 831 irq_ctx_init(smp_processor_id()); 832} 833 834/* 835 * With CONFIG_SPARSE_IRQ, interrupt descriptors are allocated as-needed, so 836 * rather than set them in lguest_init_IRQ we are called here every time an 837 * lguest device needs an interrupt. 838 * 839 * FIXME: irq_to_desc_alloc_node() can fail due to lack of memory, we should 840 * pass that up! 841 */ 842void lguest_setup_irq(unsigned int irq) 843{ 844 irq_to_desc_alloc_node(irq, 0); 845 set_irq_chip_and_handler_name(irq, &lguest_irq_controller, 846 handle_level_irq, "level"); 847} 848 849/* 850 * Time. 851 * 852 * It would be far better for everyone if the Guest had its own clock, but 853 * until then the Host gives us the time on every interrupt. 854 */ 855static unsigned long lguest_get_wallclock(void) 856{ 857 return lguest_data.time.tv_sec; 858} 859 860/* 861 * The TSC is an Intel thing called the Time Stamp Counter. The Host tells us 862 * what speed it runs at, or 0 if it's unusable as a reliable clock source. 863 * This matches what we want here: if we return 0 from this function, the x86 864 * TSC clock will give up and not register itself. 865 */ 866static unsigned long lguest_tsc_khz(void) 867{ 868 return lguest_data.tsc_khz; 869} 870 871/* 872 * If we can't use the TSC, the kernel falls back to our lower-priority 873 * "lguest_clock", where we read the time value given to us by the Host. 874 */ 875static cycle_t lguest_clock_read(struct clocksource *cs) 876{ 877 unsigned long sec, nsec; 878 879 /* 880 * Since the time is in two parts (seconds and nanoseconds), we risk 881 * reading it just as it's changing from 99 & 0.999999999 to 100 and 0, 882 * and getting 99 and 0. As Linux tends to come apart under the stress 883 * of time travel, we must be careful: 884 */ 885 do { 886 /* First we read the seconds part. */ 887 sec = lguest_data.time.tv_sec; 888 /* 889 * This read memory barrier tells the compiler and the CPU that 890 * this can't be reordered: we have to complete the above 891 * before going on. 892 */ 893 rmb(); 894 /* Now we read the nanoseconds part. */ 895 nsec = lguest_data.time.tv_nsec; 896 /* Make sure we've done that. */ 897 rmb(); 898 /* Now if the seconds part has changed, try again. */ 899 } while (unlikely(lguest_data.time.tv_sec != sec)); 900 901 /* Our lguest clock is in real nanoseconds. */ 902 return sec*1000000000ULL + nsec; 903} 904 905/* This is the fallback clocksource: lower priority than the TSC clocksource. */ 906static struct clocksource lguest_clock = { 907 .name = "lguest", 908 .rating = 200, 909 .read = lguest_clock_read, 910 .mask = CLOCKSOURCE_MASK(64), 911 .mult = 1 << 22, 912 .shift = 22, 913 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 914}; 915 916/* 917 * We also need a "struct clock_event_device": Linux asks us to set it to go 918 * off some time in the future. Actually, James Morris figured all this out, I 919 * just applied the patch. 920 */ 921static int lguest_clockevent_set_next_event(unsigned long delta, 922 struct clock_event_device *evt) 923{ 924 /* FIXME: I don't think this can ever happen, but James tells me he had 925 * to put this code in. Maybe we should remove it now. Anyone? */ 926 if (delta < LG_CLOCK_MIN_DELTA) { 927 if (printk_ratelimit()) 928 printk(KERN_DEBUG "%s: small delta %lu ns\n", 929 __func__, delta); 930 return -ETIME; 931 } 932 933 /* Please wake us this far in the future. */ 934 kvm_hypercall1(LHCALL_SET_CLOCKEVENT, delta); 935 return 0; 936} 937 938static void lguest_clockevent_set_mode(enum clock_event_mode mode, 939 struct clock_event_device *evt) 940{ 941 switch (mode) { 942 case CLOCK_EVT_MODE_UNUSED: 943 case CLOCK_EVT_MODE_SHUTDOWN: 944 /* A 0 argument shuts the clock down. */ 945 kvm_hypercall0(LHCALL_SET_CLOCKEVENT); 946 break; 947 case CLOCK_EVT_MODE_ONESHOT: 948 /* This is what we expect. */ 949 break; 950 case CLOCK_EVT_MODE_PERIODIC: 951 BUG(); 952 case CLOCK_EVT_MODE_RESUME: 953 break; 954 } 955} 956 957/* This describes our primitive timer chip. */ 958static struct clock_event_device lguest_clockevent = { 959 .name = "lguest", 960 .features = CLOCK_EVT_FEAT_ONESHOT, 961 .set_next_event = lguest_clockevent_set_next_event, 962 .set_mode = lguest_clockevent_set_mode, 963 .rating = INT_MAX, 964 .mult = 1, 965 .shift = 0, 966 .min_delta_ns = LG_CLOCK_MIN_DELTA, 967 .max_delta_ns = LG_CLOCK_MAX_DELTA, 968}; 969 970/* 971 * This is the Guest timer interrupt handler (hardware interrupt 0). We just 972 * call the clockevent infrastructure and it does whatever needs doing. 973 */ 974static void lguest_time_irq(unsigned int irq, struct irq_desc *desc) 975{ 976 unsigned long flags; 977 978 /* Don't interrupt us while this is running. */ 979 local_irq_save(flags); 980 lguest_clockevent.event_handler(&lguest_clockevent); 981 local_irq_restore(flags); 982} 983 984/* 985 * At some point in the boot process, we get asked to set up our timing 986 * infrastructure. The kernel doesn't expect timer interrupts before this, but 987 * we cleverly initialized the "blocked_interrupts" field of "struct 988 * lguest_data" so that timer interrupts were blocked until now. 989 */ 990static void lguest_time_init(void) 991{ 992 /* Set up the timer interrupt (0) to go to our simple timer routine */ 993 set_irq_handler(0, lguest_time_irq); 994 995 clocksource_register(&lguest_clock); 996 997 /* We can't set cpumask in the initializer: damn C limitations! Set it 998 * here and register our timer device. */ 999 lguest_clockevent.cpumask = cpumask_of(0); 1000 clockevents_register_device(&lguest_clockevent);
1001 1002 /* Finally, we unblock the timer interrupt. */ 1003 enable_lguest_irq(0); 1004} 1005 1006/* 1007 * Miscellaneous bits and pieces. 1008 * 1009 * Here is an oddball collection of functions which the Guest needs for things 1010 * to work. They're pretty simple. 1011 */ 1012 1013/* 1014 * The Guest needs to tell the Host what stack it expects traps to use. For 1015 * native hardware, this is part of the Task State Segment mentioned above in 1016 * lguest_load_tr_desc(), but to help hypervisors there's this special call. 1017 * 1018 * We tell the Host the segment we want to use (__KERNEL_DS is the kernel data 1019 * segment), the privilege level (we're privilege level 1, the Host is 0 and 1020 * will not tolerate us trying to use that), the stack pointer, and the number 1021 * of pages in the stack. 1022 */ 1023static void lguest_load_sp0(struct tss_struct *tss, 1024 struct thread_struct *thread) 1025{ 1026 lazy_hcall3(LHCALL_SET_STACK, __KERNEL_DS | 0x1, thread->sp0, 1027 THREAD_SIZE / PAGE_SIZE); 1028} 1029 1030/* Let's just say, I wouldn't do debugging under a Guest. */ 1031static void lguest_set_debugreg(int regno, unsigned long value) 1032{ 1033 /* FIXME: Implement */ 1034} 1035 1036/* 1037 * There are times when the kernel wants to make sure that no memory writes are 1038 * caught in the cache (that they've all reached real hardware devices). This 1039 * doesn't matter for the Guest which has virtual hardware. 1040 * 1041 * On the Pentium 4 and above, cpuid() indicates that the Cache Line Flush 1042 * (clflush) instruction is available and the kernel uses that. Otherwise, it 1043 * uses the older "Write Back and Invalidate Cache" (wbinvd) instruction. 1044 * Unlike clflush, wbinvd can only be run at privilege level 0. So we can 1045 * ignore clflush, but replace wbinvd. 1046 */ 1047static void lguest_wbinvd(void) 1048{ 1049} 1050 1051/* 1052 * If the Guest expects to have an Advanced Programmable Interrupt Controller, 1053 * we play dumb by ignoring writes and returning 0 for reads. So it's no 1054 * longer Programmable nor Controlling anything, and I don't think 8 lines of 1055 * code qualifies for Advanced. It will also never interrupt anything. It 1056 * does, however, allow us to get through the Linux boot code. 1057 */ 1058#ifdef CONFIG_X86_LOCAL_APIC 1059static void lguest_apic_write(u32 reg, u32 v) 1060{ 1061} 1062 1063static u32 lguest_apic_read(u32 reg) 1064{ 1065 return 0; 1066} 1067 1068static u64 lguest_apic_icr_read(void) 1069{ 1070 return 0; 1071} 1072 1073static void lguest_apic_icr_write(u32 low, u32 id) 1074{ 1075 /* Warn to see if there's any stray references */ 1076 WARN_ON(1); 1077} 1078 1079static void lguest_apic_wait_icr_idle(void) 1080{ 1081 return; 1082} 1083 1084static u32 lguest_apic_safe_wait_icr_idle(void) 1085{ 1086 return 0; 1087} 1088 1089static void set_lguest_basic_apic_ops(void) 1090{ 1091 apic->read = lguest_apic_read; 1092 apic->write = lguest_apic_write; 1093 apic->icr_read = lguest_apic_icr_read; 1094 apic->icr_write = lguest_apic_icr_write; 1095 apic->wait_icr_idle = lguest_apic_wait_icr_idle; 1096 apic->safe_wait_icr_idle = lguest_apic_safe_wait_icr_idle; 1097}; 1098#endif 1099 1100/* STOP! Until an interrupt comes in. */ 1101static void lguest_safe_halt(void) 1102{ 1103 kvm_hypercall0(LHCALL_HALT); 1104} 1105 1106/* 1107 * The SHUTDOWN hypercall takes a string to describe what's happening, and 1108 * an argument which says whether this to restart (reboot) the Guest or not. 1109 * 1110 * Note that the Host always prefers that the Guest speak in physical addresses 1111 * rather than virtual addresses, so we use __pa() here. 1112 */ 1113static void lguest_power_off(void) 1114{ 1115 kvm_hypercall2(LHCALL_SHUTDOWN, __pa("Power down"), 1116 LGUEST_SHUTDOWN_POWEROFF); 1117} 1118 1119/* 1120 * Panicing. 1121 * 1122 * Don't. But if you did, this is what happens. 1123 */ 1124static int lguest_panic(struct notifier_block *nb, unsigned long l, void *p) 1125{ 1126 kvm_hypercall2(LHCALL_SHUTDOWN, __pa(p), LGUEST_SHUTDOWN_POWEROFF); 1127 /* The hcall won't return, but to keep gcc happy, we're "done". */ 1128 return NOTIFY_DONE; 1129} 1130 1131static struct notifier_block paniced = { 1132 .notifier_call = lguest_panic 1133}; 1134 1135/* Setting up memory is fairly easy. */ 1136static __init char *lguest_memory_setup(void) 1137{ 1138 /* 1139 *The Linux bootloader header contains an "e820" memory map: the 1140 * Launcher populated the first entry with our memory limit. 1141 */ 1142 e820_add_region(boot_params.e820_map[0].addr, 1143 boot_params.e820_map[0].size, 1144 boot_params.e820_map[0].type); 1145 1146 /* This string is for the boot messages. */ 1147 return "LGUEST"; 1148} 1149 1150/* 1151 * We will eventually use the virtio console device to produce console output, 1152 * but before that is set up we use LHCALL_NOTIFY on normal memory to produce 1153 * console output. 1154 */ 1155static __init int early_put_chars(u32 vtermno, const char *buf, int count) 1156{ 1157 char scratch[17]; 1158 unsigned int len = count; 1159 1160 /* We use a nul-terminated string, so we make a copy. Icky, huh? */ 1161 if (len > sizeof(scratch) - 1) 1162 len = sizeof(scratch) - 1; 1163 scratch[len] = '\0'; 1164 memcpy(scratch, buf, len); 1165 kvm_hypercall1(LHCALL_NOTIFY, __pa(scratch)); 1166 1167 /* This routine returns the number of bytes actually written. */ 1168 return len; 1169} 1170 1171/* 1172 * Rebooting also tells the Host we're finished, but the RESTART flag tells the 1173 * Launcher to reboot us. 1174 */ 1175static void lguest_restart(char *reason) 1176{ 1177 kvm_hypercall2(LHCALL_SHUTDOWN, __pa(reason), LGUEST_SHUTDOWN_RESTART); 1178} 1179 1180/*G:050 1181 * Patching (Powerfully Placating Performance Pedants) 1182 * 1183 * We have already seen that pv_ops structures let us replace simple native 1184 * instructions with calls to the appropriate back end all throughout the 1185 * kernel. This allows the same kernel to run as a Guest and as a native 1186 * kernel, but it's slow because of all the indirect branches. 1187 * 1188 * Remember that David Wheeler quote about "Any problem in computer science can 1189 * be solved with another layer of indirection"? The rest of that quote is 1190 * "... But that usually will create another problem." This is the first of 1191 * those problems. 1192 * 1193 * Our current solution is to allow the paravirt back end to optionally patch 1194 * over the indirect calls to replace them with something more efficient. We 1195 * patch two of the simplest of the most commonly called functions: disable 1196 * interrupts and save interrupts. We usually have 6 or 10 bytes to patch 1197 * into: the Guest versions of these operations are small enough that we can 1198 * fit comfortably. 1199 * 1200 * First we need assembly templates of each of the patchable Guest operations, 1201 * and these are in i386_head.S. 1202 */ 1203 1204/*G:060 We construct a table from the assembler templates: */ 1205static const struct lguest_insns 1206{ 1207 const char *start, *end; 1208} lguest_insns[] = { 1209 [PARAVIRT_PATCH(pv_irq_ops.irq_disable)] = { lgstart_cli, lgend_cli }, 1210 [PARAVIRT_PATCH(pv_irq_ops.save_fl)] = { lgstart_pushf, lgend_pushf }, 1211}; 1212 1213/* 1214 * Now our patch routine is fairly simple (based on the native one in 1215 * paravirt.c). If we have a replacement, we copy it in and return how much of 1216 * the available space we used. 1217 */ 1218static unsigned lguest_patch(u8 type, u16 clobber, void *ibuf, 1219 unsigned long addr, unsigned len) 1220{ 1221 unsigned int insn_len; 1222 1223 /* Don't do anything special if we don't have a replacement */ 1224 if (type >= ARRAY_SIZE(lguest_insns) || !lguest_insns[type].start) 1225 return paravirt_patch_default(type, clobber, ibuf, addr, len); 1226 1227 insn_len = lguest_insns[type].end - lguest_insns[type].start; 1228 1229 /* Similarly if it can't fit (doesn't happen, but let's be thorough). */ 1230 if (len < insn_len) 1231 return paravirt_patch_default(type, clobber, ibuf, addr, len); 1232 1233 /* Copy in our instructions. */ 1234 memcpy(ibuf, lguest_insns[type].start, insn_len); 1235 return insn_len; 1236} 1237 1238/*G:029 1239 * Once we get to lguest_init(), we know we're a Guest. The various 1240 * pv_ops structures in the kernel provide points for (almost) every routine we 1241 * have to override to avoid privileged instructions. 1242 */ 1243__init void lguest_init(void) 1244{ 1245 /* We're under lguest. */ 1246 pv_info.name = "lguest"; 1247 /* Paravirt is enabled. */ 1248 pv_info.paravirt_enabled = 1; 1249 /* We're running at privilege level 1, not 0 as normal. */ 1250 pv_info.kernel_rpl = 1; 1251 /* Everyone except Xen runs with this set. */ 1252 pv_info.shared_kernel_pmd = 1; 1253 1254 /* 1255 * We set up all the lguest overrides for sensitive operations. These 1256 * are detailed with the operations themselves. 1257 */ 1258 1259 /* Interrupt-related operations */ 1260 pv_irq_ops.save_fl = PV_CALLEE_SAVE(save_fl); 1261 pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(lg_restore_fl); 1262 pv_irq_ops.irq_disable = PV_CALLEE_SAVE(irq_disable); 1263 pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(lg_irq_enable); 1264 pv_irq_ops.safe_halt = lguest_safe_halt; 1265 1266 /* Setup operations */ 1267 pv_init_ops.patch = lguest_patch; 1268 1269 /* Intercepts of various CPU instructions */ 1270 pv_cpu_ops.load_gdt = lguest_load_gdt; 1271 pv_cpu_ops.cpuid = lguest_cpuid; 1272 pv_cpu_ops.load_idt = lguest_load_idt; 1273 pv_cpu_ops.iret = lguest_iret; 1274 pv_cpu_ops.load_sp0 = lguest_load_sp0; 1275 pv_cpu_ops.load_tr_desc = lguest_load_tr_desc; 1276 pv_cpu_ops.set_ldt = lguest_set_ldt; 1277 pv_cpu_ops.load_tls = lguest_load_tls; 1278 pv_cpu_ops.set_debugreg = lguest_set_debugreg; 1279 pv_cpu_ops.clts = lguest_clts; 1280 pv_cpu_ops.read_cr0 = lguest_read_cr0; 1281 pv_cpu_ops.write_cr0 = lguest_write_cr0; 1282 pv_cpu_ops.read_cr4 = lguest_read_cr4; 1283 pv_cpu_ops.write_cr4 = lguest_write_cr4; 1284 pv_cpu_ops.write_gdt_entry = lguest_write_gdt_entry; 1285 pv_cpu_ops.write_idt_entry = lguest_write_idt_entry; 1286 pv_cpu_ops.wbinvd = lguest_wbinvd; 1287 pv_cpu_ops.start_context_switch = paravirt_start_context_switch; 1288 pv_cpu_ops.end_context_switch = lguest_end_context_switch; 1289 1290 /* Pagetable management */ 1291 pv_mmu_ops.write_cr3 = lguest_write_cr3; 1292 pv_mmu_ops.flush_tlb_user = lguest_flush_tlb_user; 1293 pv_mmu_ops.flush_tlb_single = lguest_flush_tlb_single; 1294 pv_mmu_ops.flush_tlb_kernel = lguest_flush_tlb_kernel; 1295 pv_mmu_ops.set_pte = lguest_set_pte; 1296 pv_mmu_ops.set_pte_at = lguest_set_pte_at; 1297 pv_mmu_ops.set_pmd = lguest_set_pmd; 1298#ifdef CONFIG_X86_PAE 1299 pv_mmu_ops.set_pte_atomic = lguest_set_pte_atomic; 1300 pv_mmu_ops.pte_clear = lguest_pte_clear; 1301 pv_mmu_ops.pmd_clear = lguest_pmd_clear; 1302 pv_mmu_ops.set_pud = lguest_set_pud; 1303#endif 1304 pv_mmu_ops.read_cr2 = lguest_read_cr2; 1305 pv_mmu_ops.read_cr3 = lguest_read_cr3; 1306 pv_mmu_ops.lazy_mode.enter = paravirt_enter_lazy_mmu; 1307 pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mmu_mode; 1308 pv_mmu_ops.pte_update = lguest_pte_update; 1309 pv_mmu_ops.pte_update_defer = lguest_pte_update; 1310 1311#ifdef CONFIG_X86_LOCAL_APIC 1312 /* APIC read/write intercepts */ 1313 set_lguest_basic_apic_ops(); 1314#endif 1315 1316 x86_init.resources.memory_setup = lguest_memory_setup; 1317 x86_init.irqs.intr_init = lguest_init_IRQ; 1318 x86_init.timers.timer_init = lguest_time_init; 1319 x86_platform.calibrate_tsc = lguest_tsc_khz; 1320 x86_platform.get_wallclock = lguest_get_wallclock; 1321 1322 /* 1323 * Now is a good time to look at the implementations of these functions 1324 * before returning to the rest of lguest_init(). 1325 */ 1326 1327 /*G:070 1328 * Now we've seen all the paravirt_ops, we return to 1329 * lguest_init() where the rest of the fairly chaotic boot setup 1330 * occurs. 1331 */ 1332 1333 /* 1334 * The stack protector is a weird thing where gcc places a canary 1335 * value on the stack and then checks it on return. This file is 1336 * compiled with -fno-stack-protector it, so we got this far without 1337 * problems. The value of the canary is kept at offset 20 from the 1338 * %gs register, so we need to set that up before calling C functions 1339 * in other files. 1340 */ 1341 setup_stack_canary_segment(0); 1342 1343 /* 1344 * We could just call load_stack_canary_segment(), but we might as well 1345 * call switch_to_new_gdt() which loads the whole table and sets up the 1346 * per-cpu segment descriptor register %fs as well. 1347 */ 1348 switch_to_new_gdt(0); 1349 1350 /* We actually boot with all memory mapped, but let's say 128MB. */ 1351 max_pfn_mapped = (128*1024*1024) >> PAGE_SHIFT; 1352 1353 /* 1354 * The Host<->Guest Switcher lives at the top of our address space, and 1355 * the Host told us how big it is when we made LGUEST_INIT hypercall: 1356 * it put the answer in lguest_data.reserve_mem 1357 */ 1358 reserve_top_address(lguest_data.reserve_mem); 1359 1360 /* 1361 * If we don't initialize the lock dependency checker now, it crashes 1362 * atomic_notifier_chain_register, then paravirt_disable_iospace. 1363 */ 1364 lockdep_init(); 1365 1366 /* Hook in our special panic hypercall code. */ 1367 atomic_notifier_chain_register(&panic_notifier_list, &paniced); 1368 1369 /* 1370 * The IDE code spends about 3 seconds probing for disks: if we reserve 1371 * all the I/O ports up front it can't get them and so doesn't probe. 1372 * Other device drivers are similar (but less severe). This cuts the 1373 * kernel boot time on my machine from 4.1 seconds to 0.45 seconds. 1374 */ 1375 paravirt_disable_iospace(); 1376 1377 /* 1378 * This is messy CPU setup stuff which the native boot code does before 1379 * start_kernel, so we have to do, too: 1380 */ 1381 cpu_detect(&new_cpu_data); 1382 /* head.S usually sets up the first capability word, so do it here. */ 1383 new_cpu_data.x86_capability[0] = cpuid_edx(1); 1384 1385 /* Math is always hard! */ 1386 new_cpu_data.hard_math = 1; 1387 1388 /* We don't have features. We have puppies! Puppies! */ 1389#ifdef CONFIG_X86_MCE 1390 mce_disabled = 1; 1391#endif 1392#ifdef CONFIG_ACPI 1393 acpi_disabled = 1; 1394 acpi_ht = 0; 1395#endif 1396 1397 /* 1398 * We set the preferred console to "hvc". This is the "hypervisor 1399 * virtual console" driver written by the PowerPC people, which we also 1400 * adapted for lguest's use. 1401 */ 1402 add_preferred_console("hvc", 0, NULL); 1403 1404 /* Register our very early console. */ 1405 virtio_cons_early_init(early_put_chars); 1406 1407 /* 1408 * Last of all, we set the power management poweroff hook to point to 1409 * the Guest routine to power off, and the reboot hook to our restart 1410 * routine. 1411 */ 1412 pm_power_off = lguest_power_off; 1413 machine_ops.restart = lguest_restart; 1414 1415 /* 1416 * Now we're set up, call i386_start_kernel() in head32.c and we proceed 1417 * to boot as normal. It never returns. 1418 */ 1419 i386_start_kernel(); 1420} 1421/* 1422 * This marks the end of stage II of our journey, The Guest. 1423 * 1424 * It is now time for us to explore the layer of virtual drivers and complete 1425 * our understanding of the Guest in "make Drivers". 1426 */ 1427