LXR linux/arch/s390/mm/vmem.c

   1/*
   2 *    Copyright IBM Corp. 2006
   3 *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   4 */
   5
   6#include <linux/bootmem.h>
   7#include <linux/pfn.h>
   8#include <linux/mm.h>
   9#include <linux/init.h>
  10#include <linux/list.h>
  11#include <linux/hugetlb.h>
  12#include <linux/slab.h>
  13#include <linux/memblock.h>
  14#include <asm/cacheflush.h>
  15#include <asm/pgalloc.h>
  16#include <asm/pgtable.h>
  17#include <asm/setup.h>
  18#include <asm/tlbflush.h>
  19#include <asm/sections.h>
  20#include <asm/set_memory.h>
  21
  22static DEFINE_MUTEX(vmem_mutex);
  23
  24struct memory_segment {
  25        struct list_head list;
  26        unsigned long start;
  27        unsigned long size;
  28};
  29
  30static LIST_HEAD(mem_segs);
  31
  32static void __ref *vmem_alloc_pages(unsigned int order)
  33{
  34        unsigned long size = PAGE_SIZE << order;
  35
  36        if (slab_is_available())
  37                return (void *)__get_free_pages(GFP_KERNEL, order);
  38        return (void *) memblock_alloc(size, size);
  39}
  40
  41static inline p4d_t *vmem_p4d_alloc(void)
  42{
  43        p4d_t *p4d = NULL;
  44
  45        p4d = vmem_alloc_pages(2);
  46        if (!p4d)
  47                return NULL;
  48        clear_table((unsigned long *) p4d, _REGION2_ENTRY_EMPTY, PAGE_SIZE * 4);
  49        return p4d;
  50}
  51
  52static inline pud_t *vmem_pud_alloc(void)
  53{
  54        pud_t *pud = NULL;
  55
  56        pud = vmem_alloc_pages(2);
  57        if (!pud)
  58                return NULL;
  59        clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  60        return pud;
  61}
  62
  63pmd_t *vmem_pmd_alloc(void)
  64{
  65        pmd_t *pmd = NULL;
  66
  67        pmd = vmem_alloc_pages(2);
  68        if (!pmd)
  69                return NULL;
  70        clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  71        return pmd;
  72}
  73
  74pte_t __ref *vmem_pte_alloc(void)
  75{
  76        unsigned long size = PTRS_PER_PTE * sizeof(pte_t);
  77        pte_t *pte;
  78
  79        if (slab_is_available())
  80                pte = (pte_t *) page_table_alloc(&init_mm);
  81        else
  82                pte = (pte_t *) memblock_alloc(size, size);
  83        if (!pte)
  84                return NULL;
  85        clear_table((unsigned long *) pte, _PAGE_INVALID, size);
  86        return pte;
  87}
  88
  89/*
  90 * Add a physical memory range to the 1:1 mapping.
  91 */
  92static int vmem_add_mem(unsigned long start, unsigned long size)
  93{
  94        unsigned long pgt_prot, sgt_prot, r3_prot;
  95        unsigned long pages4k, pages1m, pages2g;
  96        unsigned long end = start + size;
  97        unsigned long address = start;
  98        pgd_t *pg_dir;
  99        p4d_t *p4_dir;
 100        pud_t *pu_dir;
 101        pmd_t *pm_dir;
 102        pte_t *pt_dir;
 103        int ret = -ENOMEM;
 104
 105        pgt_prot = pgprot_val(PAGE_KERNEL);
 106        sgt_prot = pgprot_val(SEGMENT_KERNEL);
 107        r3_prot = pgprot_val(REGION3_KERNEL);
 108        if (!MACHINE_HAS_NX) {
 109                pgt_prot &= ~_PAGE_NOEXEC;
 110                sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
 111                r3_prot &= ~_REGION_ENTRY_NOEXEC;
 112        }
 113        pages4k = pages1m = pages2g = 0;
 114        while (address < end) {
 115                pg_dir = pgd_offset_k(address);
 116                if (pgd_none(*pg_dir)) {
 117                        p4_dir = vmem_p4d_alloc();
 118                        if (!p4_dir)
 119                                goto out;
 120                        pgd_populate(&init_mm, pg_dir, p4_dir);
 121                }
 122                p4_dir = p4d_offset(pg_dir, address);
 123                if (p4d_none(*p4_dir)) {
 124                        pu_dir = vmem_pud_alloc();
 125                        if (!pu_dir)
 126                                goto out;
 127                        p4d_populate(&init_mm, p4_dir, pu_dir);
 128                }
 129                pu_dir = pud_offset(p4_dir, address);
 130                if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
 131                    !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) &&
 132                     !debug_pagealloc_enabled()) {
 133                        pud_val(*pu_dir) = address | r3_prot;
 134                        address += PUD_SIZE;
 135                        pages2g++;
 136                        continue;
 137                }
 138                if (pud_none(*pu_dir)) {
 139                        pm_dir = vmem_pmd_alloc();
 140                        if (!pm_dir)
 141                                goto out;
 142                        pud_populate(&init_mm, pu_dir, pm_dir);
 143                }
 144                pm_dir = pmd_offset(pu_dir, address);
 145                if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
 146                    !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) &&
 147                    !debug_pagealloc_enabled()) {
 148                        pmd_val(*pm_dir) = address | sgt_prot;
 149                        address += PMD_SIZE;
 150                        pages1m++;
 151                        continue;
 152                }
 153                if (pmd_none(*pm_dir)) {
 154                        pt_dir = vmem_pte_alloc();
 155                        if (!pt_dir)
 156                                goto out;
 157                        pmd_populate(&init_mm, pm_dir, pt_dir);
 158                }
 159
 160                pt_dir = pte_offset_kernel(pm_dir, address);
 161                pte_val(*pt_dir) = address | pgt_prot;
 162                address += PAGE_SIZE;
 163                pages4k++;
 164        }
 165        ret = 0;
 166out:
 167        update_page_count(PG_DIRECT_MAP_4K, pages4k);
 168        update_page_count(PG_DIRECT_MAP_1M, pages1m);
 169        update_page_count(PG_DIRECT_MAP_2G, pages2g);
 170        return ret;
 171}
 172
 173/*
 174 * Remove a physical memory range from the 1:1 mapping.
 175 * Currently only invalidates page table entries.
 176 */
 177static void vmem_remove_range(unsigned long start, unsigned long size)
 178{
 179        unsigned long pages4k, pages1m, pages2g;
 180        unsigned long end = start + size;
 181        unsigned long address = start;
 182        pgd_t *pg_dir;
 183        p4d_t *p4_dir;
 184        pud_t *pu_dir;
 185        pmd_t *pm_dir;
 186        pte_t *pt_dir;
 187
 188        pages4k = pages1m = pages2g = 0;
 189        while (address < end) {
 190                pg_dir = pgd_offset_k(address);
 191                if (pgd_none(*pg_dir)) {
 192                        address += PGDIR_SIZE;
 193                        continue;
 194                }
 195                p4_dir = p4d_offset(pg_dir, address);
 196                if (p4d_none(*p4_dir)) {
 197                        address += P4D_SIZE;
 198                        continue;
 199                }
 200                pu_dir = pud_offset(p4_dir, address);
 201                if (pud_none(*pu_dir)) {
 202                        address += PUD_SIZE;
 203                        continue;
 204                }
 205                if (pud_large(*pu_dir)) {
 206                        pud_clear(pu_dir);
 207                        address += PUD_SIZE;
 208                        pages2g++;
 209                        continue;
 210                }
 211                pm_dir = pmd_offset(pu_dir, address);
 212                if (pmd_none(*pm_dir)) {
 213                        address += PMD_SIZE;
 214                        continue;
 215                }
 216                if (pmd_large(*pm_dir)) {
 217                        pmd_clear(pm_dir);
 218                        address += PMD_SIZE;
 219                        pages1m++;
 220                        continue;
 221                }
 222                pt_dir = pte_offset_kernel(pm_dir, address);
 223                pte_clear(&init_mm, address, pt_dir);
 224                address += PAGE_SIZE;
 225                pages4k++;
 226        }
 227        flush_tlb_kernel_range(start, end);
 228        update_page_count(PG_DIRECT_MAP_4K, -pages4k);
 229        update_page_count(PG_DIRECT_MAP_1M, -pages1m);
 230        update_page_count(PG_DIRECT_MAP_2G, -pages2g);
 231}
 232
 233/*
 234 * Add a backed mem_map array to the virtual mem_map array.
 235 */
 236int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 237{
 238        unsigned long pgt_prot, sgt_prot;
 239        unsigned long address = start;
 240        pgd_t *pg_dir;
 241        p4d_t *p4_dir;
 242        pud_t *pu_dir;
 243        pmd_t *pm_dir;
 244        pte_t *pt_dir;
 245        int ret = -ENOMEM;
 246
 247        pgt_prot = pgprot_val(PAGE_KERNEL);
 248        sgt_prot = pgprot_val(SEGMENT_KERNEL);
 249        if (!MACHINE_HAS_NX) {
 250                pgt_prot &= ~_PAGE_NOEXEC;
 251                sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
 252        }
 253        for (address = start; address < end;) {
 254                pg_dir = pgd_offset_k(address);
 255                if (pgd_none(*pg_dir)) {
 256                        p4_dir = vmem_p4d_alloc();
 257                        if (!p4_dir)
 258                                goto out;
 259                        pgd_populate(&init_mm, pg_dir, p4_dir);
 260                }
 261
 262                p4_dir = p4d_offset(pg_dir, address);
 263                if (p4d_none(*p4_dir)) {
 264                        pu_dir = vmem_pud_alloc();
 265                        if (!pu_dir)
 266                                goto out;
 267                        p4d_populate(&init_mm, p4_dir, pu_dir);
 268                }
 269
 270                pu_dir = pud_offset(p4_dir, address);
 271                if (pud_none(*pu_dir)) {
 272                        pm_dir = vmem_pmd_alloc();
 273                        if (!pm_dir)
 274                                goto out;
 275                        pud_populate(&init_mm, pu_dir, pm_dir);
 276                }
 277
 278                pm_dir = pmd_offset(pu_dir, address);
 279                if (pmd_none(*pm_dir)) {
 280                        /* Use 1MB frames for vmemmap if available. We always
 281                         * use large frames even if they are only partially
 282                         * used.
 283                         * Otherwise we would have also page tables since
 284                         * vmemmap_populate gets called for each section
 285                         * separately. */
 286                        if (MACHINE_HAS_EDAT1) {
 287                                void *new_page;
 288
 289                                new_page = vmemmap_alloc_block(PMD_SIZE, node);
 290                                if (!new_page)
 291                                        goto out;
 292                                pmd_val(*pm_dir) = __pa(new_page) | sgt_prot;
 293                                address = (address + PMD_SIZE) & PMD_MASK;
 294                                continue;
 295                        }
 296                        pt_dir = vmem_pte_alloc();
 297                        if (!pt_dir)
 298                                goto out;
 299                        pmd_populate(&init_mm, pm_dir, pt_dir);
 300                } else if (pmd_large(*pm_dir)) {
 301                        address = (address + PMD_SIZE) & PMD_MASK;
 302                        continue;
 303                }
 304
 305                pt_dir = pte_offset_kernel(pm_dir, address);
 306                if (pte_none(*pt_dir)) {
 307                        void *new_page;
 308
 309                        new_page = vmemmap_alloc_block(PAGE_SIZE, node);
 310                        if (!new_page)
 311                                goto out;
 312                        pte_val(*pt_dir) = __pa(new_page) | pgt_prot;
 313                }
 314                address += PAGE_SIZE;
 315        }
 316        ret = 0;
 317out:
 318        return ret;
 319}
 320
 321void vmemmap_free(unsigned long start, unsigned long end)
 322{
 323}
 324
 325/*
 326 * Add memory segment to the segment list if it doesn't overlap with
 327 * an already present segment.
 328 */
 329static int insert_memory_segment(struct memory_segment *seg)
 330{
 331        struct memory_segment *tmp;
 332
 333        if (seg->start + seg->size > VMEM_MAX_PHYS ||
 334            seg->start + seg->size < seg->start)
 335                return -ERANGE;
 336
 337        list_for_each_entry(tmp, &mem_segs, list) {
 338                if (seg->start >= tmp->start + tmp->size)
 339                        continue;
 340                if (seg->start + seg->size <= tmp->start)
 341                        continue;
 342                return -ENOSPC;
 343        }
 344        list_add(&seg->list, &mem_segs);
 345        return 0;
 346}
 347
 348/*
 349 * Remove memory segment from the segment list.
 350 */
 351static void remove_memory_segment(struct memory_segment *seg)
 352{
 353        list_del(&seg->list);
 354}
 355
 356static void __remove_shared_memory(struct memory_segment *seg)
 357{
 358        remove_memory_segment(seg);
 359        vmem_remove_range(seg->start, seg->size);
 360}
 361
 362int vmem_remove_mapping(unsigned long start, unsigned long size)
 363{
 364        struct memory_segment *seg;
 365        int ret;
 366
 367        mutex_lock(&vmem_mutex);
 368
 369        ret = -ENOENT;
 370        list_for_each_entry(seg, &mem_segs, list) {
 371                if (seg->start == start && seg->size == size)
 372                        break;
 373        }
 374
 375        if (seg->start != start || seg->size != size)
 376                goto out;
 377
 378        ret = 0;
 379        __remove_shared_memory(seg);
 380        kfree(seg);
 381out:
 382        mutex_unlock(&vmem_mutex);
 383        return ret;
 384}
 385
 386int vmem_add_mapping(unsigned long start, unsigned long size)
 387{
 388        struct memory_segment *seg;
 389        int ret;
 390
 391        mutex_lock(&vmem_mutex);
 392        ret = -ENOMEM;
 393        seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 394        if (!seg)
 395                goto out;
 396        seg->start = start;
 397        seg->size = size;
 398
 399        ret = insert_memory_segment(seg);
 400        if (ret)
 401                goto out_free;
 402
 403        ret = vmem_add_mem(start, size);
 404        if (ret)
 405                goto out_remove;
 406        goto out;
 407
 408out_remove:
 409        __remove_shared_memory(seg);
 410out_free:
 411        kfree(seg);
 412out:
 413        mutex_unlock(&vmem_mutex);
 414        return ret;
 415}
 416
 417/*
 418 * map whole physical memory to virtual memory (identity mapping)
 419 * we reserve enough space in the vmalloc area for vmemmap to hotplug
 420 * additional memory segments.
 421 */
 422void __init vmem_map_init(void)
 423{
 424        struct memblock_region *reg;
 425
 426        for_each_memblock(memory, reg)
 427                vmem_add_mem(reg->base, reg->size);
 428        __set_memory((unsigned long) _stext,
 429                     (_etext - _stext) >> PAGE_SHIFT,
 430                     SET_MEMORY_RO | SET_MEMORY_X);
 431        __set_memory((unsigned long) _etext,
 432                     (_eshared - _etext) >> PAGE_SHIFT,
 433                     SET_MEMORY_RO);
 434        __set_memory((unsigned long) _sinittext,
 435                     (_einittext - _sinittext) >> PAGE_SHIFT,
 436                     SET_MEMORY_RO | SET_MEMORY_X);
 437        pr_info("Write protected kernel read-only data: %luk\n",
 438                (_eshared - _stext) >> 10);
 439}
 440
 441/*
 442 * Convert memblock.memory  to a memory segment list so there is a single
 443 * list that contains all memory segments.
 444 */
 445static int __init vmem_convert_memory_chunk(void)
 446{
 447        struct memblock_region *reg;
 448        struct memory_segment *seg;
 449
 450        mutex_lock(&vmem_mutex);
 451        for_each_memblock(memory, reg) {
 452                seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 453                if (!seg)
 454                        panic("Out of memory...\n");
 455                seg->start = reg->base;
 456                seg->size = reg->size;
 457                insert_memory_segment(seg);
 458        }
 459        mutex_unlock(&vmem_mutex);
 460        return 0;
 461}
 462
 463core_initcall(vmem_convert_memory_chunk);
 464