LXR linux/arch/s390/mm/vmem.c

   1/*
   2 *    Copyright IBM Corp. 2006
   3 *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   4 */
   5
   6#include <linux/bootmem.h>
   7#include <linux/pfn.h>
   8#include <linux/mm.h>
   9#include <linux/module.h>
  10#include <linux/list.h>
  11#include <linux/hugetlb.h>
  12#include <linux/slab.h>
  13#include <asm/cacheflush.h>
  14#include <asm/pgalloc.h>
  15#include <asm/pgtable.h>
  16#include <asm/setup.h>
  17#include <asm/tlbflush.h>
  18#include <asm/sections.h>
  19
  20static DEFINE_MUTEX(vmem_mutex);
  21
  22struct memory_segment {
  23        struct list_head list;
  24        unsigned long start;
  25        unsigned long size;
  26};
  27
  28static LIST_HEAD(mem_segs);
  29
  30static void __ref *vmem_alloc_pages(unsigned int order)
  31{
  32        unsigned long size = PAGE_SIZE << order;
  33
  34        if (slab_is_available())
  35                return (void *)__get_free_pages(GFP_KERNEL, order);
  36        return alloc_bootmem_align(size, size);
  37}
  38
  39static inline pud_t *vmem_pud_alloc(void)
  40{
  41        pud_t *pud = NULL;
  42
  43#ifdef CONFIG_64BIT
  44        pud = vmem_alloc_pages(2);
  45        if (!pud)
  46                return NULL;
  47        clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  48#endif
  49        return pud;
  50}
  51
  52pmd_t *vmem_pmd_alloc(unsigned long address)
  53{
  54        pmd_t *pmd = NULL;
  55
  56#ifdef CONFIG_64BIT
  57        pmd = vmem_alloc_pages(2);
  58        if (!pmd)
  59                return NULL;
  60        clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  61#endif
  62        return pmd;
  63}
  64
  65pte_t __ref *vmem_pte_alloc(unsigned long address)
  66{
  67        pte_t *pte;
  68
  69        if (slab_is_available())
  70                pte = (pte_t *) page_table_alloc(&init_mm, address);
  71        else
  72                pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t));
  73        if (!pte)
  74                return NULL;
  75        clear_table((unsigned long *) pte, _PAGE_INVALID,
  76                    PTRS_PER_PTE * sizeof(pte_t));
  77        return pte;
  78}
  79
  80/*
  81 * Add a physical memory range to the 1:1 mapping.
  82 */
  83static int vmem_add_mem(unsigned long start, unsigned long size)
  84{
  85        unsigned long pgt_prot, sgt_prot, r3_prot;
  86        unsigned long end = start + size;
  87        unsigned long address = start;
  88        pgd_t *pg_dir;
  89        pud_t *pu_dir;
  90        pmd_t *pm_dir;
  91        pte_t *pt_dir;
  92        int ret = -ENOMEM;
  93
  94        pgt_prot = pgprot_val(PAGE_KERNEL);
  95        sgt_prot = pgprot_val(SEGMENT_KERNEL);
  96        r3_prot = pgprot_val(REGION3_KERNEL);
  97        if (!MACHINE_HAS_NX) {
  98                pgt_prot &= ~_PAGE_NOEXEC;
  99                sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
 100                r3_prot &= ~_REGION_ENTRY_NOEXEC;
 101        }
 102        while (address < end) {
 103                pg_dir = pgd_offset_k(address);
 104                if (pgd_none(*pg_dir)) {
 105                        pu_dir = vmem_pud_alloc();
 106                        if (!pu_dir)
 107                                goto out;
 108                        pgd_populate(&init_mm, pg_dir, pu_dir);
 109                }
 110                pu_dir = pud_offset(pg_dir, address);
 111#ifdef CONFIG_64BIT
 112                if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
 113                    !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) &&
 114                    !debug_pagealloc_enabled()) {
 115                        pud_val(*pu_dir) = address | r3_prot;
 116                        address += PUD_SIZE;
 117                        continue;
 118                }
 119#endif
 120                if (pud_none(*pu_dir)) {
 121                        pm_dir = vmem_pmd_alloc(address);
 122                        if (!pm_dir)
 123                                goto out;
 124                        pud_populate(&init_mm, pu_dir, pm_dir);
 125                }
 126                pm_dir = pmd_offset(pu_dir, address);
 127#ifdef CONFIG_64BIT
 128                if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
 129                    !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) &&
 130                    !debug_pagealloc_enabled()) {
 131                        pmd_val(*pm_dir) = address | sgt_prot;
 132                        address += PMD_SIZE;
 133                        continue;
 134                }
 135#endif
 136                if (pmd_none(*pm_dir)) {
 137                        pt_dir = vmem_pte_alloc(address);
 138                        if (!pt_dir)
 139                                goto out;
 140                        pmd_populate(&init_mm, pm_dir, pt_dir);
 141                }
 142
 143                pt_dir = pte_offset_kernel(pm_dir, address);
 144                pte_val(*pt_dir) = address | pgt_prot;
 145                address += PAGE_SIZE;
 146        }
 147        ret = 0;
 148out:
 149        flush_tlb_kernel_range(start, end);
 150        return ret;
 151}
 152
 153/*
 154 * Remove a physical memory range from the 1:1 mapping.
 155 * Currently only invalidates page table entries.
 156 */
 157static void vmem_remove_range(unsigned long start, unsigned long size)
 158{
 159        unsigned long end = start + size;
 160        unsigned long address = start;
 161        pgd_t *pg_dir;
 162        pud_t *pu_dir;
 163        pmd_t *pm_dir;
 164        pte_t *pt_dir;
 165        pte_t  pte;
 166
 167        pte_val(pte) = _PAGE_INVALID;
 168        while (address < end) {
 169                pg_dir = pgd_offset_k(address);
 170                if (pgd_none(*pg_dir)) {
 171                        address += PGDIR_SIZE;
 172                        continue;
 173                }
 174                pu_dir = pud_offset(pg_dir, address);
 175                if (pud_none(*pu_dir)) {
 176                        address += PUD_SIZE;
 177                        continue;
 178                }
 179                if (pud_large(*pu_dir)) {
 180                        pud_clear(pu_dir);
 181                        address += PUD_SIZE;
 182                        continue;
 183                }
 184                pm_dir = pmd_offset(pu_dir, address);
 185                if (pmd_none(*pm_dir)) {
 186                        address += PMD_SIZE;
 187                        continue;
 188                }
 189                if (pmd_large(*pm_dir)) {
 190                        pmd_clear(pm_dir);
 191                        address += PMD_SIZE;
 192                        continue;
 193                }
 194                pt_dir = pte_offset_kernel(pm_dir, address);
 195                *pt_dir = pte;
 196                address += PAGE_SIZE;
 197        }
 198        flush_tlb_kernel_range(start, end);
 199}
 200
 201/*
 202 * Add a backed mem_map array to the virtual mem_map array.
 203 */
 204int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 205{
 206        unsigned long pgt_prot, sgt_prot;
 207        unsigned long address = start;
 208        pgd_t *pg_dir;
 209        pud_t *pu_dir;
 210        pmd_t *pm_dir;
 211        pte_t *pt_dir;
 212        int ret = -ENOMEM;
 213
 214        pgt_prot = pgprot_val(PAGE_KERNEL);
 215        sgt_prot = pgprot_val(SEGMENT_KERNEL);
 216        if (!MACHINE_HAS_NX) {
 217                pgt_prot &= ~_PAGE_NOEXEC;
 218                sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
 219        }
 220        for (address = start; address < end;) {
 221                pg_dir = pgd_offset_k(address);
 222                if (pgd_none(*pg_dir)) {
 223                        pu_dir = vmem_pud_alloc();
 224                        if (!pu_dir)
 225                                goto out;
 226                        pgd_populate(&init_mm, pg_dir, pu_dir);
 227                }
 228
 229                pu_dir = pud_offset(pg_dir, address);
 230                if (pud_none(*pu_dir)) {
 231                        pm_dir = vmem_pmd_alloc(address);
 232                        if (!pm_dir)
 233                                goto out;
 234                        pud_populate(&init_mm, pu_dir, pm_dir);
 235                }
 236
 237                pm_dir = pmd_offset(pu_dir, address);
 238                if (pmd_none(*pm_dir)) {
 239#ifdef CONFIG_64BIT
 240                        /* Use 1MB frames for vmemmap if available. We always
 241                         * use large frames even if they are only partially
 242                         * used.
 243                         * Otherwise we would have also page tables since
 244                         * vmemmap_populate gets called for each section
 245                         * separately. */
 246                        if (MACHINE_HAS_EDAT1) {
 247                                void *new_page;
 248
 249                                new_page = vmemmap_alloc_block(PMD_SIZE, node);
 250                                if (!new_page)
 251                                        goto out;
 252                                pmd_val(*pm_dir) = __pa(new_page) | sgt_prot;
 253                                address = (address + PMD_SIZE) & PMD_MASK;
 254                                continue;
 255                        }
 256#endif
 257                        pt_dir = vmem_pte_alloc(address);
 258                        if (!pt_dir)
 259                                goto out;
 260                        pmd_populate(&init_mm, pm_dir, pt_dir);
 261                } else if (pmd_large(*pm_dir)) {
 262                        address = (address + PMD_SIZE) & PMD_MASK;
 263                        continue;
 264                }
 265
 266                pt_dir = pte_offset_kernel(pm_dir, address);
 267                if (pte_none(*pt_dir)) {
 268                        unsigned long new_page;
 269
 270                        new_page =__pa(vmem_alloc_pages(0));
 271                        if (!new_page)
 272                                goto out;
 273                        pte_val(*pt_dir) = __pa(new_page) | pgt_prot;
 274                }
 275                address += PAGE_SIZE;
 276        }
 277        memset((void *)start, 0, end - start);
 278        ret = 0;
 279out:
 280        flush_tlb_kernel_range(start, end);
 281        return ret;
 282}
 283
 284void vmemmap_free(unsigned long start, unsigned long end)
 285{
 286}
 287
 288/*
 289 * Add memory segment to the segment list if it doesn't overlap with
 290 * an already present segment.
 291 */
 292static int insert_memory_segment(struct memory_segment *seg)
 293{
 294        struct memory_segment *tmp;
 295
 296        if (seg->start + seg->size > VMEM_MAX_PHYS ||
 297            seg->start + seg->size < seg->start)
 298                return -ERANGE;
 299
 300        list_for_each_entry(tmp, &mem_segs, list) {
 301                if (seg->start >= tmp->start + tmp->size)
 302                        continue;
 303                if (seg->start + seg->size <= tmp->start)
 304                        continue;
 305                return -ENOSPC;
 306        }
 307        list_add(&seg->list, &mem_segs);
 308        return 0;
 309}
 310
 311/*
 312 * Remove memory segment from the segment list.
 313 */
 314static void remove_memory_segment(struct memory_segment *seg)
 315{
 316        list_del(&seg->list);
 317}
 318
 319static void __remove_shared_memory(struct memory_segment *seg)
 320{
 321        remove_memory_segment(seg);
 322        vmem_remove_range(seg->start, seg->size);
 323}
 324
 325int vmem_remove_mapping(unsigned long start, unsigned long size)
 326{
 327        struct memory_segment *seg;
 328        int ret;
 329
 330        mutex_lock(&vmem_mutex);
 331
 332        ret = -ENOENT;
 333        list_for_each_entry(seg, &mem_segs, list) {
 334                if (seg->start == start && seg->size == size)
 335                        break;
 336        }
 337
 338        if (seg->start != start || seg->size != size)
 339                goto out;
 340
 341        ret = 0;
 342        __remove_shared_memory(seg);
 343        kfree(seg);
 344out:
 345        mutex_unlock(&vmem_mutex);
 346        return ret;
 347}
 348
 349int vmem_add_mapping(unsigned long start, unsigned long size)
 350{
 351        struct memory_segment *seg;
 352        int ret;
 353
 354        mutex_lock(&vmem_mutex);
 355        ret = -ENOMEM;
 356        seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 357        if (!seg)
 358                goto out;
 359        seg->start = start;
 360        seg->size = size;
 361
 362        ret = insert_memory_segment(seg);
 363        if (ret)
 364                goto out_free;
 365
 366        ret = vmem_add_mem(start, size);
 367        if (ret)
 368                goto out_remove;
 369        goto out;
 370
 371out_remove:
 372        __remove_shared_memory(seg);
 373out_free:
 374        kfree(seg);
 375out:
 376        mutex_unlock(&vmem_mutex);
 377        return ret;
 378}
 379
 380/*
 381 * map whole physical memory to virtual memory (identity mapping)
 382 * we reserve enough space in the vmalloc area for vmemmap to hotplug
 383 * additional memory segments.
 384 */
 385void __init vmem_map_init(void)
 386{
 387        int i;
 388
 389        for (i = 0; i < MEMORY_CHUNKS; i++) {
 390                if (!memory_chunk[i].size)
 391                        continue;
 392                vmem_add_mem(memory_chunk[i].addr, memory_chunk[i].size);
 393        }
 394        __set_memory((unsigned long) _stext,
 395                     (_etext - _stext) >> PAGE_SHIFT,
 396                     SET_MEMORY_RO | SET_MEMORY_X);
 397        __set_memory((unsigned long) _etext,
 398                     (_eshared - _etext) >> PAGE_SHIFT,
 399                     SET_MEMORY_RO);
 400        __set_memory((unsigned long) _sinittext,
 401                     (_einittext - _sinittext) >> PAGE_SHIFT,
 402                     SET_MEMORY_RO | SET_MEMORY_X);
 403        pr_info("Write protected kernel read-only data: %luk\n",
 404                (_eshared - _stext) >> 10);
 405}
 406
 407/*
 408 * Convert memory chunk array to a memory segment list so there is a single
 409 * list that contains both r/w memory and shared memory segments.
 410 */
 411static int __init vmem_convert_memory_chunk(void)
 412{
 413        struct memory_segment *seg;
 414        int i;
 415
 416        mutex_lock(&vmem_mutex);
 417        for (i = 0; i < MEMORY_CHUNKS; i++) {
 418                if (!memory_chunk[i].size)
 419                        continue;
 420                seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 421                if (!seg)
 422                        panic("Out of memory...\n");
 423                seg->start = memory_chunk[i].addr;
 424                seg->size = memory_chunk[i].size;
 425                insert_memory_segment(seg);
 426        }
 427        mutex_unlock(&vmem_mutex);
 428        return 0;
 429}
 430
 431core_initcall(vmem_convert_memory_chunk);
 432