LXR linux/arch/s390/mm/vmem.c

   1/*
   2 *    Copyright IBM Corp. 2006
   3 *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   4 */
   5
   6#include <linux/bootmem.h>
   7#include <linux/pfn.h>
   8#include <linux/mm.h>
   9#include <linux/module.h>
  10#include <linux/list.h>
  11#include <linux/hugetlb.h>
  12#include <linux/slab.h>
  13#include <asm/pgalloc.h>
  14#include <asm/pgtable.h>
  15#include <asm/setup.h>
  16#include <asm/tlbflush.h>
  17#include <asm/sections.h>
  18
  19static DEFINE_MUTEX(vmem_mutex);
  20
  21struct memory_segment {
  22        struct list_head list;
  23        unsigned long start;
  24        unsigned long size;
  25};
  26
  27static LIST_HEAD(mem_segs);
  28
  29static void __ref *vmem_alloc_pages(unsigned int order)
  30{
  31        if (slab_is_available())
  32                return (void *)__get_free_pages(GFP_KERNEL, order);
  33        return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
  34}
  35
  36static inline pud_t *vmem_pud_alloc(void)
  37{
  38        pud_t *pud = NULL;
  39
  40#ifdef CONFIG_64BIT
  41        pud = vmem_alloc_pages(2);
  42        if (!pud)
  43                return NULL;
  44        clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  45#endif
  46        return pud;
  47}
  48
  49static inline pmd_t *vmem_pmd_alloc(void)
  50{
  51        pmd_t *pmd = NULL;
  52
  53#ifdef CONFIG_64BIT
  54        pmd = vmem_alloc_pages(2);
  55        if (!pmd)
  56                return NULL;
  57        clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  58#endif
  59        return pmd;
  60}
  61
  62static pte_t __ref *vmem_pte_alloc(unsigned long address)
  63{
  64        pte_t *pte;
  65
  66        if (slab_is_available())
  67                pte = (pte_t *) page_table_alloc(&init_mm, address);
  68        else
  69                pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t));
  70        if (!pte)
  71                return NULL;
  72        clear_table((unsigned long *) pte, _PAGE_INVALID,
  73                    PTRS_PER_PTE * sizeof(pte_t));
  74        return pte;
  75}
  76
  77/*
  78 * Add a physical memory range to the 1:1 mapping.
  79 */
  80static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
  81{
  82        unsigned long end = start + size;
  83        unsigned long address = start;
  84        pgd_t *pg_dir;
  85        pud_t *pu_dir;
  86        pmd_t *pm_dir;
  87        pte_t *pt_dir;
  88        int ret = -ENOMEM;
  89
  90        while (address < end) {
  91                pg_dir = pgd_offset_k(address);
  92                if (pgd_none(*pg_dir)) {
  93                        pu_dir = vmem_pud_alloc();
  94                        if (!pu_dir)
  95                                goto out;
  96                        pgd_populate(&init_mm, pg_dir, pu_dir);
  97                }
  98                pu_dir = pud_offset(pg_dir, address);
  99#if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC)
 100                if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
 101                    !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) {
 102                        pud_val(*pu_dir) = __pa(address) |
 103                                _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE |
 104                                (ro ? _REGION_ENTRY_PROTECT : 0);
 105                        address += PUD_SIZE;
 106                        continue;
 107                }
 108#endif
 109                if (pud_none(*pu_dir)) {
 110                        pm_dir = vmem_pmd_alloc();
 111                        if (!pm_dir)
 112                                goto out;
 113                        pud_populate(&init_mm, pu_dir, pm_dir);
 114                }
 115                pm_dir = pmd_offset(pu_dir, address);
 116#if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC)
 117                if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
 118                    !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) {
 119                        pmd_val(*pm_dir) = __pa(address) |
 120                                _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
 121                                _SEGMENT_ENTRY_YOUNG |
 122                                (ro ? _SEGMENT_ENTRY_PROTECT : 0);
 123                        address += PMD_SIZE;
 124                        continue;
 125                }
 126#endif
 127                if (pmd_none(*pm_dir)) {
 128                        pt_dir = vmem_pte_alloc(address);
 129                        if (!pt_dir)
 130                                goto out;
 131                        pmd_populate(&init_mm, pm_dir, pt_dir);
 132                }
 133
 134                pt_dir = pte_offset_kernel(pm_dir, address);
 135                pte_val(*pt_dir) = __pa(address) |
 136                        pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
 137                address += PAGE_SIZE;
 138        }
 139        ret = 0;
 140out:
 141        flush_tlb_kernel_range(start, end);
 142        return ret;
 143}
 144
 145/*
 146 * Remove a physical memory range from the 1:1 mapping.
 147 * Currently only invalidates page table entries.
 148 */
 149static void vmem_remove_range(unsigned long start, unsigned long size)
 150{
 151        unsigned long end = start + size;
 152        unsigned long address = start;
 153        pgd_t *pg_dir;
 154        pud_t *pu_dir;
 155        pmd_t *pm_dir;
 156        pte_t *pt_dir;
 157        pte_t  pte;
 158
 159        pte_val(pte) = _PAGE_INVALID;
 160        while (address < end) {
 161                pg_dir = pgd_offset_k(address);
 162                if (pgd_none(*pg_dir)) {
 163                        address += PGDIR_SIZE;
 164                        continue;
 165                }
 166                pu_dir = pud_offset(pg_dir, address);
 167                if (pud_none(*pu_dir)) {
 168                        address += PUD_SIZE;
 169                        continue;
 170                }
 171                if (pud_large(*pu_dir)) {
 172                        pud_clear(pu_dir);
 173                        address += PUD_SIZE;
 174                        continue;
 175                }
 176                pm_dir = pmd_offset(pu_dir, address);
 177                if (pmd_none(*pm_dir)) {
 178                        address += PMD_SIZE;
 179                        continue;
 180                }
 181                if (pmd_large(*pm_dir)) {
 182                        pmd_clear(pm_dir);
 183                        address += PMD_SIZE;
 184                        continue;
 185                }
 186                pt_dir = pte_offset_kernel(pm_dir, address);
 187                *pt_dir = pte;
 188                address += PAGE_SIZE;
 189        }
 190        flush_tlb_kernel_range(start, end);
 191}
 192
 193/*
 194 * Add a backed mem_map array to the virtual mem_map array.
 195 */
 196int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 197{
 198        unsigned long address = start;
 199        pgd_t *pg_dir;
 200        pud_t *pu_dir;
 201        pmd_t *pm_dir;
 202        pte_t *pt_dir;
 203        int ret = -ENOMEM;
 204
 205        for (address = start; address < end;) {
 206                pg_dir = pgd_offset_k(address);
 207                if (pgd_none(*pg_dir)) {
 208                        pu_dir = vmem_pud_alloc();
 209                        if (!pu_dir)
 210                                goto out;
 211                        pgd_populate(&init_mm, pg_dir, pu_dir);
 212                }
 213
 214                pu_dir = pud_offset(pg_dir, address);
 215                if (pud_none(*pu_dir)) {
 216                        pm_dir = vmem_pmd_alloc();
 217                        if (!pm_dir)
 218                                goto out;
 219                        pud_populate(&init_mm, pu_dir, pm_dir);
 220                }
 221
 222                pm_dir = pmd_offset(pu_dir, address);
 223                if (pmd_none(*pm_dir)) {
 224#ifdef CONFIG_64BIT
 225                        /* Use 1MB frames for vmemmap if available. We always
 226                         * use large frames even if they are only partially
 227                         * used.
 228                         * Otherwise we would have also page tables since
 229                         * vmemmap_populate gets called for each section
 230                         * separately. */
 231                        if (MACHINE_HAS_EDAT1) {
 232                                void *new_page;
 233
 234                                new_page = vmemmap_alloc_block(PMD_SIZE, node);
 235                                if (!new_page)
 236                                        goto out;
 237                                pmd_val(*pm_dir) = __pa(new_page) |
 238                                        _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
 239                                        _SEGMENT_ENTRY_CO;
 240                                address = (address + PMD_SIZE) & PMD_MASK;
 241                                continue;
 242                        }
 243#endif
 244                        pt_dir = vmem_pte_alloc(address);
 245                        if (!pt_dir)
 246                                goto out;
 247                        pmd_populate(&init_mm, pm_dir, pt_dir);
 248                } else if (pmd_large(*pm_dir)) {
 249                        address = (address + PMD_SIZE) & PMD_MASK;
 250                        continue;
 251                }
 252
 253                pt_dir = pte_offset_kernel(pm_dir, address);
 254                if (pte_none(*pt_dir)) {
 255                        unsigned long new_page;
 256
 257                        new_page =__pa(vmem_alloc_pages(0));
 258                        if (!new_page)
 259                                goto out;
 260                        pte_val(*pt_dir) =
 261                                __pa(new_page) | pgprot_val(PAGE_KERNEL);
 262                }
 263                address += PAGE_SIZE;
 264        }
 265        memset((void *)start, 0, end - start);
 266        ret = 0;
 267out:
 268        flush_tlb_kernel_range(start, end);
 269        return ret;
 270}
 271
 272void vmemmap_free(unsigned long start, unsigned long end)
 273{
 274}
 275
 276/*
 277 * Add memory segment to the segment list if it doesn't overlap with
 278 * an already present segment.
 279 */
 280static int insert_memory_segment(struct memory_segment *seg)
 281{
 282        struct memory_segment *tmp;
 283
 284        if (seg->start + seg->size > VMEM_MAX_PHYS ||
 285            seg->start + seg->size < seg->start)
 286                return -ERANGE;
 287
 288        list_for_each_entry(tmp, &mem_segs, list) {
 289                if (seg->start >= tmp->start + tmp->size)
 290                        continue;
 291                if (seg->start + seg->size <= tmp->start)
 292                        continue;
 293                return -ENOSPC;
 294        }
 295        list_add(&seg->list, &mem_segs);
 296        return 0;
 297}
 298
 299/*
 300 * Remove memory segment from the segment list.
 301 */
 302static void remove_memory_segment(struct memory_segment *seg)
 303{
 304        list_del(&seg->list);
 305}
 306
 307static void __remove_shared_memory(struct memory_segment *seg)
 308{
 309        remove_memory_segment(seg);
 310        vmem_remove_range(seg->start, seg->size);
 311}
 312
 313int vmem_remove_mapping(unsigned long start, unsigned long size)
 314{
 315        struct memory_segment *seg;
 316        int ret;
 317
 318        mutex_lock(&vmem_mutex);
 319
 320        ret = -ENOENT;
 321        list_for_each_entry(seg, &mem_segs, list) {
 322                if (seg->start == start && seg->size == size)
 323                        break;
 324        }
 325
 326        if (seg->start != start || seg->size != size)
 327                goto out;
 328
 329        ret = 0;
 330        __remove_shared_memory(seg);
 331        kfree(seg);
 332out:
 333        mutex_unlock(&vmem_mutex);
 334        return ret;
 335}
 336
 337int vmem_add_mapping(unsigned long start, unsigned long size)
 338{
 339        struct memory_segment *seg;
 340        int ret;
 341
 342        mutex_lock(&vmem_mutex);
 343        ret = -ENOMEM;
 344        seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 345        if (!seg)
 346                goto out;
 347        seg->start = start;
 348        seg->size = size;
 349
 350        ret = insert_memory_segment(seg);
 351        if (ret)
 352                goto out_free;
 353
 354        ret = vmem_add_mem(start, size, 0);
 355        if (ret)
 356                goto out_remove;
 357        goto out;
 358
 359out_remove:
 360        __remove_shared_memory(seg);
 361out_free:
 362        kfree(seg);
 363out:
 364        mutex_unlock(&vmem_mutex);
 365        return ret;
 366}
 367
 368/*
 369 * map whole physical memory to virtual memory (identity mapping)
 370 * we reserve enough space in the vmalloc area for vmemmap to hotplug
 371 * additional memory segments.
 372 */
 373void __init vmem_map_init(void)
 374{
 375        unsigned long ro_start, ro_end;
 376        unsigned long start, end;
 377        int i;
 378
 379        ro_start = PFN_ALIGN((unsigned long)&_stext);
 380        ro_end = (unsigned long)&_eshared & PAGE_MASK;
 381        for (i = 0; i < MEMORY_CHUNKS; i++) {
 382                if (!memory_chunk[i].size)
 383                        continue;
 384                start = memory_chunk[i].addr;
 385                end = memory_chunk[i].addr + memory_chunk[i].size;
 386                if (start >= ro_end || end <= ro_start)
 387                        vmem_add_mem(start, end - start, 0);
 388                else if (start >= ro_start && end <= ro_end)
 389                        vmem_add_mem(start, end - start, 1);
 390                else if (start >= ro_start) {
 391                        vmem_add_mem(start, ro_end - start, 1);
 392                        vmem_add_mem(ro_end, end - ro_end, 0);
 393                } else if (end < ro_end) {
 394                        vmem_add_mem(start, ro_start - start, 0);
 395                        vmem_add_mem(ro_start, end - ro_start, 1);
 396                } else {
 397                        vmem_add_mem(start, ro_start - start, 0);
 398                        vmem_add_mem(ro_start, ro_end - ro_start, 1);
 399                        vmem_add_mem(ro_end, end - ro_end, 0);
 400                }
 401        }
 402}
 403
 404/*
 405 * Convert memory chunk array to a memory segment list so there is a single
 406 * list that contains both r/w memory and shared memory segments.
 407 */
 408static int __init vmem_convert_memory_chunk(void)
 409{
 410        struct memory_segment *seg;
 411        int i;
 412
 413        mutex_lock(&vmem_mutex);
 414        for (i = 0; i < MEMORY_CHUNKS; i++) {
 415                if (!memory_chunk[i].size)
 416                        continue;
 417                seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 418                if (!seg)
 419                        panic("Out of memory...\n");
 420                seg->start = memory_chunk[i].addr;
 421                seg->size = memory_chunk[i].size;
 422                insert_memory_segment(seg);
 423        }
 424        mutex_unlock(&vmem_mutex);
 425        return 0;
 426}
 427
 428core_initcall(vmem_convert_memory_chunk);
 429