linux/arch/x86/kernel/cpu/mtrr/cleanup.c
<<
>>
Prefs
   1/*
   2 * MTRR (Memory Type Range Register) cleanup
   3 *
   4 *  Copyright (C) 2009 Yinghai Lu
   5 *
   6 * This library is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU Library General Public
   8 * License as published by the Free Software Foundation; either
   9 * version 2 of the License, or (at your option) any later version.
  10 *
  11 * This library is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 * Library General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU Library General Public
  17 * License along with this library; if not, write to the Free
  18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20#include <linux/module.h>
  21#include <linux/init.h>
  22#include <linux/pci.h>
  23#include <linux/smp.h>
  24#include <linux/cpu.h>
  25#include <linux/mutex.h>
  26#include <linux/uaccess.h>
  27#include <linux/kvm_para.h>
  28#include <linux/range.h>
  29
  30#include <asm/processor.h>
  31#include <asm/e820.h>
  32#include <asm/mtrr.h>
  33#include <asm/msr.h>
  34
  35#include "mtrr.h"
  36
  37struct var_mtrr_range_state {
  38        unsigned long   base_pfn;
  39        unsigned long   size_pfn;
  40        mtrr_type       type;
  41};
  42
  43struct var_mtrr_state {
  44        unsigned long   range_startk;
  45        unsigned long   range_sizek;
  46        unsigned long   chunk_sizek;
  47        unsigned long   gran_sizek;
  48        unsigned int    reg;
  49};
  50
  51/* Should be related to MTRR_VAR_RANGES nums */
  52#define RANGE_NUM                               256
  53
  54static struct range __initdata          range[RANGE_NUM];
  55static int __initdata                           nr_range;
  56
  57static struct var_mtrr_range_state __initdata   range_state[RANGE_NUM];
  58
  59static int __initdata debug_print;
  60#define Dprintk(x...) do { if (debug_print) printk(KERN_DEBUG x); } while (0)
  61
  62#define BIOS_BUG_MSG KERN_WARNING \
  63        "WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n"
  64
  65static int __init
  66x86_get_mtrr_mem_range(struct range *range, int nr_range,
  67                       unsigned long extra_remove_base,
  68                       unsigned long extra_remove_size)
  69{
  70        unsigned long base, size;
  71        mtrr_type type;
  72        int i;
  73
  74        for (i = 0; i < num_var_ranges; i++) {
  75                type = range_state[i].type;
  76                if (type != MTRR_TYPE_WRBACK)
  77                        continue;
  78                base = range_state[i].base_pfn;
  79                size = range_state[i].size_pfn;
  80                nr_range = add_range_with_merge(range, RANGE_NUM, nr_range,
  81                                                base, base + size);
  82        }
  83        if (debug_print) {
  84                printk(KERN_DEBUG "After WB checking\n");
  85                for (i = 0; i < nr_range; i++)
  86                        printk(KERN_DEBUG "MTRR MAP PFN: %016llx - %016llx\n",
  87                                 range[i].start, range[i].end);
  88        }
  89
  90        /* Take out UC ranges: */
  91        for (i = 0; i < num_var_ranges; i++) {
  92                type = range_state[i].type;
  93                if (type != MTRR_TYPE_UNCACHABLE &&
  94                    type != MTRR_TYPE_WRPROT)
  95                        continue;
  96                size = range_state[i].size_pfn;
  97                if (!size)
  98                        continue;
  99                base = range_state[i].base_pfn;
 100                if (base < (1<<(20-PAGE_SHIFT)) && mtrr_state.have_fixed &&
 101                    (mtrr_state.enabled & 1)) {
 102                        /* Var MTRR contains UC entry below 1M? Skip it: */
 103                        printk(BIOS_BUG_MSG, i);
 104                        if (base + size <= (1<<(20-PAGE_SHIFT)))
 105                                continue;
 106                        size -= (1<<(20-PAGE_SHIFT)) - base;
 107                        base = 1<<(20-PAGE_SHIFT);
 108                }
 109                subtract_range(range, RANGE_NUM, base, base + size);
 110        }
 111        if (extra_remove_size)
 112                subtract_range(range, RANGE_NUM, extra_remove_base,
 113                                 extra_remove_base + extra_remove_size);
 114
 115        if  (debug_print) {
 116                printk(KERN_DEBUG "After UC checking\n");
 117                for (i = 0; i < RANGE_NUM; i++) {
 118                        if (!range[i].end)
 119                                continue;
 120                        printk(KERN_DEBUG "MTRR MAP PFN: %016llx - %016llx\n",
 121                                 range[i].start, range[i].end);
 122                }
 123        }
 124
 125        /* sort the ranges */
 126        nr_range = clean_sort_range(range, RANGE_NUM);
 127        if  (debug_print) {
 128                printk(KERN_DEBUG "After sorting\n");
 129                for (i = 0; i < nr_range; i++)
 130                        printk(KERN_DEBUG "MTRR MAP PFN: %016llx - %016llx\n",
 131                                 range[i].start, range[i].end);
 132        }
 133
 134        return nr_range;
 135}
 136
 137#ifdef CONFIG_MTRR_SANITIZER
 138
 139static unsigned long __init sum_ranges(struct range *range, int nr_range)
 140{
 141        unsigned long sum = 0;
 142        int i;
 143
 144        for (i = 0; i < nr_range; i++)
 145                sum += range[i].end - range[i].start;
 146
 147        return sum;
 148}
 149
 150static int enable_mtrr_cleanup __initdata =
 151        CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT;
 152
 153static int __init disable_mtrr_cleanup_setup(char *str)
 154{
 155        enable_mtrr_cleanup = 0;
 156        return 0;
 157}
 158early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup);
 159
 160static int __init enable_mtrr_cleanup_setup(char *str)
 161{
 162        enable_mtrr_cleanup = 1;
 163        return 0;
 164}
 165early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup);
 166
 167static int __init mtrr_cleanup_debug_setup(char *str)
 168{
 169        debug_print = 1;
 170        return 0;
 171}
 172early_param("mtrr_cleanup_debug", mtrr_cleanup_debug_setup);
 173
 174static void __init
 175set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
 176             unsigned char type, unsigned int address_bits)
 177{
 178        u32 base_lo, base_hi, mask_lo, mask_hi;
 179        u64 base, mask;
 180
 181        if (!sizek) {
 182                fill_mtrr_var_range(reg, 0, 0, 0, 0);
 183                return;
 184        }
 185
 186        mask = (1ULL << address_bits) - 1;
 187        mask &= ~((((u64)sizek) << 10) - 1);
 188
 189        base = ((u64)basek) << 10;
 190
 191        base |= type;
 192        mask |= 0x800;
 193
 194        base_lo = base & ((1ULL<<32) - 1);
 195        base_hi = base >> 32;
 196
 197        mask_lo = mask & ((1ULL<<32) - 1);
 198        mask_hi = mask >> 32;
 199
 200        fill_mtrr_var_range(reg, base_lo, base_hi, mask_lo, mask_hi);
 201}
 202
 203static void __init
 204save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek,
 205              unsigned char type)
 206{
 207        range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10);
 208        range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10);
 209        range_state[reg].type = type;
 210}
 211
 212static void __init set_var_mtrr_all(unsigned int address_bits)
 213{
 214        unsigned long basek, sizek;
 215        unsigned char type;
 216        unsigned int reg;
 217
 218        for (reg = 0; reg < num_var_ranges; reg++) {
 219                basek = range_state[reg].base_pfn << (PAGE_SHIFT - 10);
 220                sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10);
 221                type = range_state[reg].type;
 222
 223                set_var_mtrr(reg, basek, sizek, type, address_bits);
 224        }
 225}
 226
 227static unsigned long to_size_factor(unsigned long sizek, char *factorp)
 228{
 229        unsigned long base = sizek;
 230        char factor;
 231
 232        if (base & ((1<<10) - 1)) {
 233                /* Not MB-aligned: */
 234                factor = 'K';
 235        } else if (base & ((1<<20) - 1)) {
 236                factor = 'M';
 237                base >>= 10;
 238        } else {
 239                factor = 'G';
 240                base >>= 20;
 241        }
 242
 243        *factorp = factor;
 244
 245        return base;
 246}
 247
 248static unsigned int __init
 249range_to_mtrr(unsigned int reg, unsigned long range_startk,
 250              unsigned long range_sizek, unsigned char type)
 251{
 252        if (!range_sizek || (reg >= num_var_ranges))
 253                return reg;
 254
 255        while (range_sizek) {
 256                unsigned long max_align, align;
 257                unsigned long sizek;
 258
 259                /* Compute the maximum size with which we can make a range: */
 260                if (range_startk)
 261                        max_align = ffs(range_startk) - 1;
 262                else
 263                        max_align = 32;
 264
 265                align = fls(range_sizek) - 1;
 266                if (align > max_align)
 267                        align = max_align;
 268
 269                sizek = 1 << align;
 270                if (debug_print) {
 271                        char start_factor = 'K', size_factor = 'K';
 272                        unsigned long start_base, size_base;
 273
 274                        start_base = to_size_factor(range_startk, &start_factor);
 275                        size_base = to_size_factor(sizek, &size_factor);
 276
 277                        Dprintk("Setting variable MTRR %d, "
 278                                "base: %ld%cB, range: %ld%cB, type %s\n",
 279                                reg, start_base, start_factor,
 280                                size_base, size_factor,
 281                                (type == MTRR_TYPE_UNCACHABLE) ? "UC" :
 282                                   ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other")
 283                                );
 284                }
 285                save_var_mtrr(reg++, range_startk, sizek, type);
 286                range_startk += sizek;
 287                range_sizek -= sizek;
 288                if (reg >= num_var_ranges)
 289                        break;
 290        }
 291        return reg;
 292}
 293
 294static unsigned __init
 295range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek,
 296                        unsigned long sizek)
 297{
 298        unsigned long hole_basek, hole_sizek;
 299        unsigned long second_basek, second_sizek;
 300        unsigned long range0_basek, range0_sizek;
 301        unsigned long range_basek, range_sizek;
 302        unsigned long chunk_sizek;
 303        unsigned long gran_sizek;
 304
 305        hole_basek = 0;
 306        hole_sizek = 0;
 307        second_basek = 0;
 308        second_sizek = 0;
 309        chunk_sizek = state->chunk_sizek;
 310        gran_sizek = state->gran_sizek;
 311
 312        /* Align with gran size, prevent small block used up MTRRs: */
 313        range_basek = ALIGN(state->range_startk, gran_sizek);
 314        if ((range_basek > basek) && basek)
 315                return second_sizek;
 316
 317        state->range_sizek -= (range_basek - state->range_startk);
 318        range_sizek = ALIGN(state->range_sizek, gran_sizek);
 319
 320        while (range_sizek > state->range_sizek) {
 321                range_sizek -= gran_sizek;
 322                if (!range_sizek)
 323                        return 0;
 324        }
 325        state->range_sizek = range_sizek;
 326
 327        /* Try to append some small hole: */
 328        range0_basek = state->range_startk;
 329        range0_sizek = ALIGN(state->range_sizek, chunk_sizek);
 330
 331        /* No increase: */
 332        if (range0_sizek == state->range_sizek) {
 333                Dprintk("rangeX: %016lx - %016lx\n",
 334                        range0_basek<<10,
 335                        (range0_basek + state->range_sizek)<<10);
 336                state->reg = range_to_mtrr(state->reg, range0_basek,
 337                                state->range_sizek, MTRR_TYPE_WRBACK);
 338                return 0;
 339        }
 340
 341        /* Only cut back when it is not the last: */
 342        if (sizek) {
 343                while (range0_basek + range0_sizek > (basek + sizek)) {
 344                        if (range0_sizek >= chunk_sizek)
 345                                range0_sizek -= chunk_sizek;
 346                        else
 347                                range0_sizek = 0;
 348
 349                        if (!range0_sizek)
 350                                break;
 351                }
 352        }
 353
 354second_try:
 355        range_basek = range0_basek + range0_sizek;
 356
 357        /* One hole in the middle: */
 358        if (range_basek > basek && range_basek <= (basek + sizek))
 359                second_sizek = range_basek - basek;
 360
 361        if (range0_sizek > state->range_sizek) {
 362
 363                /* One hole in middle or at the end: */
 364                hole_sizek = range0_sizek - state->range_sizek - second_sizek;
 365
 366                /* Hole size should be less than half of range0 size: */
 367                if (hole_sizek >= (range0_sizek >> 1) &&
 368                    range0_sizek >= chunk_sizek) {
 369                        range0_sizek -= chunk_sizek;
 370                        second_sizek = 0;
 371                        hole_sizek = 0;
 372
 373                        goto second_try;
 374                }
 375        }
 376
 377        if (range0_sizek) {
 378                Dprintk("range0: %016lx - %016lx\n",
 379                        range0_basek<<10,
 380                        (range0_basek + range0_sizek)<<10);
 381                state->reg = range_to_mtrr(state->reg, range0_basek,
 382                                range0_sizek, MTRR_TYPE_WRBACK);
 383        }
 384
 385        if (range0_sizek < state->range_sizek) {
 386                /* Need to handle left over range: */
 387                range_sizek = state->range_sizek - range0_sizek;
 388
 389                Dprintk("range: %016lx - %016lx\n",
 390                         range_basek<<10,
 391                         (range_basek + range_sizek)<<10);
 392
 393                state->reg = range_to_mtrr(state->reg, range_basek,
 394                                 range_sizek, MTRR_TYPE_WRBACK);
 395        }
 396
 397        if (hole_sizek) {
 398                hole_basek = range_basek - hole_sizek - second_sizek;
 399                Dprintk("hole: %016lx - %016lx\n",
 400                         hole_basek<<10,
 401                         (hole_basek + hole_sizek)<<10);
 402                state->reg = range_to_mtrr(state->reg, hole_basek,
 403                                 hole_sizek, MTRR_TYPE_UNCACHABLE);
 404        }
 405
 406        return second_sizek;
 407}
 408
 409static void __init
 410set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn,
 411                   unsigned long size_pfn)
 412{
 413        unsigned long basek, sizek;
 414        unsigned long second_sizek = 0;
 415
 416        if (state->reg >= num_var_ranges)
 417                return;
 418
 419        basek = base_pfn << (PAGE_SHIFT - 10);
 420        sizek = size_pfn << (PAGE_SHIFT - 10);
 421
 422        /* See if I can merge with the last range: */
 423        if ((basek <= 1024) ||
 424            (state->range_startk + state->range_sizek == basek)) {
 425                unsigned long endk = basek + sizek;
 426                state->range_sizek = endk - state->range_startk;
 427                return;
 428        }
 429        /* Write the range mtrrs: */
 430        if (state->range_sizek != 0)
 431                second_sizek = range_to_mtrr_with_hole(state, basek, sizek);
 432
 433        /* Allocate an msr: */
 434        state->range_startk = basek + second_sizek;
 435        state->range_sizek  = sizek - second_sizek;
 436}
 437
 438/* Mininum size of mtrr block that can take hole: */
 439static u64 mtrr_chunk_size __initdata = (256ULL<<20);
 440
 441static int __init parse_mtrr_chunk_size_opt(char *p)
 442{
 443        if (!p)
 444                return -EINVAL;
 445        mtrr_chunk_size = memparse(p, &p);
 446        return 0;
 447}
 448early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt);
 449
 450/* Granularity of mtrr of block: */
 451static u64 mtrr_gran_size __initdata;
 452
 453static int __init parse_mtrr_gran_size_opt(char *p)
 454{
 455        if (!p)
 456                return -EINVAL;
 457        mtrr_gran_size = memparse(p, &p);
 458        return 0;
 459}
 460early_param("mtrr_gran_size", parse_mtrr_gran_size_opt);
 461
 462static unsigned long nr_mtrr_spare_reg __initdata =
 463                                 CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT;
 464
 465static int __init parse_mtrr_spare_reg(char *arg)
 466{
 467        if (arg)
 468                nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0);
 469        return 0;
 470}
 471early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg);
 472
 473static int __init
 474x86_setup_var_mtrrs(struct range *range, int nr_range,
 475                    u64 chunk_size, u64 gran_size)
 476{
 477        struct var_mtrr_state var_state;
 478        int num_reg;
 479        int i;
 480
 481        var_state.range_startk  = 0;
 482        var_state.range_sizek   = 0;
 483        var_state.reg           = 0;
 484        var_state.chunk_sizek   = chunk_size >> 10;
 485        var_state.gran_sizek    = gran_size >> 10;
 486
 487        memset(range_state, 0, sizeof(range_state));
 488
 489        /* Write the range: */
 490        for (i = 0; i < nr_range; i++) {
 491                set_var_mtrr_range(&var_state, range[i].start,
 492                                   range[i].end - range[i].start);
 493        }
 494
 495        /* Write the last range: */
 496        if (var_state.range_sizek != 0)
 497                range_to_mtrr_with_hole(&var_state, 0, 0);
 498
 499        num_reg = var_state.reg;
 500        /* Clear out the extra MTRR's: */
 501        while (var_state.reg < num_var_ranges) {
 502                save_var_mtrr(var_state.reg, 0, 0, 0);
 503                var_state.reg++;
 504        }
 505
 506        return num_reg;
 507}
 508
 509struct mtrr_cleanup_result {
 510        unsigned long   gran_sizek;
 511        unsigned long   chunk_sizek;
 512        unsigned long   lose_cover_sizek;
 513        unsigned int    num_reg;
 514        int             bad;
 515};
 516
 517/*
 518 * gran_size: 64K, 128K, 256K, 512K, 1M, 2M, ..., 2G
 519 * chunk size: gran_size, ..., 2G
 520 * so we need (1+16)*8
 521 */
 522#define NUM_RESULT      136
 523#define PSHIFT          (PAGE_SHIFT - 10)
 524
 525static struct mtrr_cleanup_result __initdata result[NUM_RESULT];
 526static unsigned long __initdata min_loss_pfn[RANGE_NUM];
 527
 528static void __init print_out_mtrr_range_state(void)
 529{
 530        char start_factor = 'K', size_factor = 'K';
 531        unsigned long start_base, size_base;
 532        mtrr_type type;
 533        int i;
 534
 535        for (i = 0; i < num_var_ranges; i++) {
 536
 537                size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10);
 538                if (!size_base)
 539                        continue;
 540
 541                size_base = to_size_factor(size_base, &size_factor),
 542                start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10);
 543                start_base = to_size_factor(start_base, &start_factor),
 544                type = range_state[i].type;
 545
 546                printk(KERN_DEBUG "reg %d, base: %ld%cB, range: %ld%cB, type %s\n",
 547                        i, start_base, start_factor,
 548                        size_base, size_factor,
 549                        (type == MTRR_TYPE_UNCACHABLE) ? "UC" :
 550                            ((type == MTRR_TYPE_WRPROT) ? "WP" :
 551                             ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other"))
 552                        );
 553        }
 554}
 555
 556static int __init mtrr_need_cleanup(void)
 557{
 558        int i;
 559        mtrr_type type;
 560        unsigned long size;
 561        /* Extra one for all 0: */
 562        int num[MTRR_NUM_TYPES + 1];
 563
 564        /* Check entries number: */
 565        memset(num, 0, sizeof(num));
 566        for (i = 0; i < num_var_ranges; i++) {
 567                type = range_state[i].type;
 568                size = range_state[i].size_pfn;
 569                if (type >= MTRR_NUM_TYPES)
 570                        continue;
 571                if (!size)
 572                        type = MTRR_NUM_TYPES;
 573                num[type]++;
 574        }
 575
 576        /* Check if we got UC entries: */
 577        if (!num[MTRR_TYPE_UNCACHABLE])
 578                return 0;
 579
 580        /* Check if we only had WB and UC */
 581        if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
 582            num_var_ranges - num[MTRR_NUM_TYPES])
 583                return 0;
 584
 585        return 1;
 586}
 587
 588static unsigned long __initdata range_sums;
 589
 590static void __init
 591mtrr_calc_range_state(u64 chunk_size, u64 gran_size,
 592                      unsigned long x_remove_base,
 593                      unsigned long x_remove_size, int i)
 594{
 595        static struct range range_new[RANGE_NUM];
 596        unsigned long range_sums_new;
 597        static int nr_range_new;
 598        int num_reg;
 599
 600        /* Convert ranges to var ranges state: */
 601        num_reg = x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
 602
 603        /* We got new setting in range_state, check it: */
 604        memset(range_new, 0, sizeof(range_new));
 605        nr_range_new = x86_get_mtrr_mem_range(range_new, 0,
 606                                x_remove_base, x_remove_size);
 607        range_sums_new = sum_ranges(range_new, nr_range_new);
 608
 609        result[i].chunk_sizek = chunk_size >> 10;
 610        result[i].gran_sizek = gran_size >> 10;
 611        result[i].num_reg = num_reg;
 612
 613        if (range_sums < range_sums_new) {
 614                result[i].lose_cover_sizek = (range_sums_new - range_sums) << PSHIFT;
 615                result[i].bad = 1;
 616        } else {
 617                result[i].lose_cover_sizek = (range_sums - range_sums_new) << PSHIFT;
 618        }
 619
 620        /* Double check it: */
 621        if (!result[i].bad && !result[i].lose_cover_sizek) {
 622                if (nr_range_new != nr_range || memcmp(range, range_new, sizeof(range)))
 623                        result[i].bad = 1;
 624        }
 625
 626        if (!result[i].bad && (range_sums - range_sums_new < min_loss_pfn[num_reg]))
 627                min_loss_pfn[num_reg] = range_sums - range_sums_new;
 628}
 629
 630static void __init mtrr_print_out_one_result(int i)
 631{
 632        unsigned long gran_base, chunk_base, lose_base;
 633        char gran_factor, chunk_factor, lose_factor;
 634
 635        gran_base = to_size_factor(result[i].gran_sizek, &gran_factor);
 636        chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor);
 637        lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor);
 638
 639        pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t",
 640                result[i].bad ? "*BAD*" : " ",
 641                gran_base, gran_factor, chunk_base, chunk_factor);
 642        pr_cont("num_reg: %d  \tlose cover RAM: %s%ld%c\n",
 643                result[i].num_reg, result[i].bad ? "-" : "",
 644                lose_base, lose_factor);
 645}
 646
 647static int __init mtrr_search_optimal_index(void)
 648{
 649        int num_reg_good;
 650        int index_good;
 651        int i;
 652
 653        if (nr_mtrr_spare_reg >= num_var_ranges)
 654                nr_mtrr_spare_reg = num_var_ranges - 1;
 655
 656        num_reg_good = -1;
 657        for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) {
 658                if (!min_loss_pfn[i])
 659                        num_reg_good = i;
 660        }
 661
 662        index_good = -1;
 663        if (num_reg_good != -1) {
 664                for (i = 0; i < NUM_RESULT; i++) {
 665                        if (!result[i].bad &&
 666                            result[i].num_reg == num_reg_good &&
 667                            !result[i].lose_cover_sizek) {
 668                                index_good = i;
 669                                break;
 670                        }
 671                }
 672        }
 673
 674        return index_good;
 675}
 676
 677int __init mtrr_cleanup(unsigned address_bits)
 678{
 679        unsigned long x_remove_base, x_remove_size;
 680        unsigned long base, size, def, dummy;
 681        u64 chunk_size, gran_size;
 682        mtrr_type type;
 683        int index_good;
 684        int i;
 685
 686        if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1)
 687                return 0;
 688
 689        rdmsr(MSR_MTRRdefType, def, dummy);
 690        def &= 0xff;
 691        if (def != MTRR_TYPE_UNCACHABLE)
 692                return 0;
 693
 694        /* Get it and store it aside: */
 695        memset(range_state, 0, sizeof(range_state));
 696        for (i = 0; i < num_var_ranges; i++) {
 697                mtrr_if->get(i, &base, &size, &type);
 698                range_state[i].base_pfn = base;
 699                range_state[i].size_pfn = size;
 700                range_state[i].type = type;
 701        }
 702
 703        /* Check if we need handle it and can handle it: */
 704        if (!mtrr_need_cleanup())
 705                return 0;
 706
 707        /* Print original var MTRRs at first, for debugging: */
 708        printk(KERN_DEBUG "original variable MTRRs\n");
 709        print_out_mtrr_range_state();
 710
 711        memset(range, 0, sizeof(range));
 712        x_remove_size = 0;
 713        x_remove_base = 1 << (32 - PAGE_SHIFT);
 714        if (mtrr_tom2)
 715                x_remove_size = (mtrr_tom2 >> PAGE_SHIFT) - x_remove_base;
 716
 717        nr_range = x86_get_mtrr_mem_range(range, 0, x_remove_base, x_remove_size);
 718        /*
 719         * [0, 1M) should always be covered by var mtrr with WB
 720         * and fixed mtrrs should take effect before var mtrr for it:
 721         */
 722        nr_range = add_range_with_merge(range, RANGE_NUM, nr_range, 0,
 723                                        1ULL<<(20 - PAGE_SHIFT));
 724        /* Sort the ranges: */
 725        sort_range(range, nr_range);
 726
 727        range_sums = sum_ranges(range, nr_range);
 728        printk(KERN_INFO "total RAM covered: %ldM\n",
 729               range_sums >> (20 - PAGE_SHIFT));
 730
 731        if (mtrr_chunk_size && mtrr_gran_size) {
 732                i = 0;
 733                mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size,
 734                                      x_remove_base, x_remove_size, i);
 735
 736                mtrr_print_out_one_result(i);
 737
 738                if (!result[i].bad) {
 739                        set_var_mtrr_all(address_bits);
 740                        printk(KERN_DEBUG "New variable MTRRs\n");
 741                        print_out_mtrr_range_state();
 742                        return 1;
 743                }
 744                printk(KERN_INFO "invalid mtrr_gran_size or mtrr_chunk_size, "
 745                       "will find optimal one\n");
 746        }
 747
 748        i = 0;
 749        memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn));
 750        memset(result, 0, sizeof(result));
 751        for (gran_size = (1ULL<<16); gran_size < (1ULL<<32); gran_size <<= 1) {
 752
 753                for (chunk_size = gran_size; chunk_size < (1ULL<<32);
 754                     chunk_size <<= 1) {
 755
 756                        if (i >= NUM_RESULT)
 757                                continue;
 758
 759                        mtrr_calc_range_state(chunk_size, gran_size,
 760                                      x_remove_base, x_remove_size, i);
 761                        if (debug_print) {
 762                                mtrr_print_out_one_result(i);
 763                                printk(KERN_INFO "\n");
 764                        }
 765
 766                        i++;
 767                }
 768        }
 769
 770        /* Try to find the optimal index: */
 771        index_good = mtrr_search_optimal_index();
 772
 773        if (index_good != -1) {
 774                printk(KERN_INFO "Found optimal setting for mtrr clean up\n");
 775                i = index_good;
 776                mtrr_print_out_one_result(i);
 777
 778                /* Convert ranges to var ranges state: */
 779                chunk_size = result[i].chunk_sizek;
 780                chunk_size <<= 10;
 781                gran_size = result[i].gran_sizek;
 782                gran_size <<= 10;
 783                x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size);
 784                set_var_mtrr_all(address_bits);
 785                printk(KERN_DEBUG "New variable MTRRs\n");
 786                print_out_mtrr_range_state();
 787                return 1;
 788        } else {
 789                /* print out all */
 790                for (i = 0; i < NUM_RESULT; i++)
 791                        mtrr_print_out_one_result(i);
 792        }
 793
 794        printk(KERN_INFO "mtrr_cleanup: can not find optimal value\n");
 795        printk(KERN_INFO "please specify mtrr_gran_size/mtrr_chunk_size\n");
 796
 797        return 0;
 798}
 799#else
 800int __init mtrr_cleanup(unsigned address_bits)
 801{
 802        return 0;
 803}
 804#endif
 805
 806static int disable_mtrr_trim;
 807
 808static int __init disable_mtrr_trim_setup(char *str)
 809{
 810        disable_mtrr_trim = 1;
 811        return 0;
 812}
 813early_param("disable_mtrr_trim", disable_mtrr_trim_setup);
 814
 815/*
 816 * Newer AMD K8s and later CPUs have a special magic MSR way to force WB
 817 * for memory >4GB. Check for that here.
 818 * Note this won't check if the MTRRs < 4GB where the magic bit doesn't
 819 * apply to are wrong, but so far we don't know of any such case in the wild.
 820 */
 821#define Tom2Enabled             (1U << 21)
 822#define Tom2ForceMemTypeWB      (1U << 22)
 823
 824int __init amd_special_default_mtrr(void)
 825{
 826        u32 l, h;
 827
 828        if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
 829                return 0;
 830        if (boot_cpu_data.x86 < 0xf)
 831                return 0;
 832        /* In case some hypervisor doesn't pass SYSCFG through: */
 833        if (rdmsr_safe(MSR_K8_SYSCFG, &l, &h) < 0)
 834                return 0;
 835        /*
 836         * Memory between 4GB and top of mem is forced WB by this magic bit.
 837         * Reserved before K8RevF, but should be zero there.
 838         */
 839        if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) ==
 840                 (Tom2Enabled | Tom2ForceMemTypeWB))
 841                return 1;
 842        return 0;
 843}
 844
 845static u64 __init
 846real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn)
 847{
 848        u64 trim_start, trim_size;
 849
 850        trim_start = start_pfn;
 851        trim_start <<= PAGE_SHIFT;
 852
 853        trim_size = limit_pfn;
 854        trim_size <<= PAGE_SHIFT;
 855        trim_size -= trim_start;
 856
 857        return e820_update_range(trim_start, trim_size, E820_RAM, E820_RESERVED);
 858}
 859
 860/**
 861 * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs
 862 * @end_pfn: ending page frame number
 863 *
 864 * Some buggy BIOSes don't setup the MTRRs properly for systems with certain
 865 * memory configurations.  This routine checks that the highest MTRR matches
 866 * the end of memory, to make sure the MTRRs having a write back type cover
 867 * all of the memory the kernel is intending to use.  If not, it'll trim any
 868 * memory off the end by adjusting end_pfn, removing it from the kernel's
 869 * allocation pools, warning the user with an obnoxious message.
 870 */
 871int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
 872{
 873        unsigned long i, base, size, highest_pfn = 0, def, dummy;
 874        mtrr_type type;
 875        u64 total_trim_size;
 876        /* extra one for all 0 */
 877        int num[MTRR_NUM_TYPES + 1];
 878
 879        /*
 880         * Make sure we only trim uncachable memory on machines that
 881         * support the Intel MTRR architecture:
 882         */
 883        if (!is_cpu(INTEL) || disable_mtrr_trim)
 884                return 0;
 885
 886        rdmsr(MSR_MTRRdefType, def, dummy);
 887        def &= 0xff;
 888        if (def != MTRR_TYPE_UNCACHABLE)
 889                return 0;
 890
 891        /* Get it and store it aside: */
 892        memset(range_state, 0, sizeof(range_state));
 893        for (i = 0; i < num_var_ranges; i++) {
 894                mtrr_if->get(i, &base, &size, &type);
 895                range_state[i].base_pfn = base;
 896                range_state[i].size_pfn = size;
 897                range_state[i].type = type;
 898        }
 899
 900        /* Find highest cached pfn: */
 901        for (i = 0; i < num_var_ranges; i++) {
 902                type = range_state[i].type;
 903                if (type != MTRR_TYPE_WRBACK)
 904                        continue;
 905                base = range_state[i].base_pfn;
 906                size = range_state[i].size_pfn;
 907                if (highest_pfn < base + size)
 908                        highest_pfn = base + size;
 909        }
 910
 911        /* kvm/qemu doesn't have mtrr set right, don't trim them all: */
 912        if (!highest_pfn) {
 913                printk(KERN_INFO "CPU MTRRs all blank - virtualized system.\n");
 914                return 0;
 915        }
 916
 917        /* Check entries number: */
 918        memset(num, 0, sizeof(num));
 919        for (i = 0; i < num_var_ranges; i++) {
 920                type = range_state[i].type;
 921                if (type >= MTRR_NUM_TYPES)
 922                        continue;
 923                size = range_state[i].size_pfn;
 924                if (!size)
 925                        type = MTRR_NUM_TYPES;
 926                num[type]++;
 927        }
 928
 929        /* No entry for WB? */
 930        if (!num[MTRR_TYPE_WRBACK])
 931                return 0;
 932
 933        /* Check if we only had WB and UC: */
 934        if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] !=
 935                num_var_ranges - num[MTRR_NUM_TYPES])
 936                return 0;
 937
 938        memset(range, 0, sizeof(range));
 939        nr_range = 0;
 940        if (mtrr_tom2) {
 941                range[nr_range].start = (1ULL<<(32 - PAGE_SHIFT));
 942                range[nr_range].end = mtrr_tom2 >> PAGE_SHIFT;
 943                if (highest_pfn < range[nr_range].end)
 944                        highest_pfn = range[nr_range].end;
 945                nr_range++;
 946        }
 947        nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0);
 948
 949        /* Check the head: */
 950        total_trim_size = 0;
 951        if (range[0].start)
 952                total_trim_size += real_trim_memory(0, range[0].start);
 953
 954        /* Check the holes: */
 955        for (i = 0; i < nr_range - 1; i++) {
 956                if (range[i].end < range[i+1].start)
 957                        total_trim_size += real_trim_memory(range[i].end,
 958                                                            range[i+1].start);
 959        }
 960
 961        /* Check the top: */
 962        i = nr_range - 1;
 963        if (range[i].end < end_pfn)
 964                total_trim_size += real_trim_memory(range[i].end,
 965                                                         end_pfn);
 966
 967        if (total_trim_size) {
 968                pr_warning("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n", total_trim_size >> 20);
 969
 970                if (!changed_by_mtrr_cleanup)
 971                        WARN_ON(1);
 972
 973                pr_info("update e820 for mtrr\n");
 974                update_e820();
 975
 976                return 1;
 977        }
 978
 979        return 0;
 980}
 981