LXR linux/mm/bootmem

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Bootmem core functions.
   4 *
   5 * Copyright (c) 2020, Bytedance.
   6 *
   7 *     Author: Muchun Song <songmuchun@bytedance.com>
   8 *
   9 */
  10#include <linux/mm.h>
  11#include <linux/compiler.h>
  12#include <linux/memblock.h>
  13#include <linux/bootmem_info.h>
  14#include <linux/memory_hotplug.h>
  15
  16void get_page_bootmem(unsigned long info, struct page *page, unsigned long type)
  17{
  18        page->freelist = (void *)type;
  19        SetPagePrivate(page);
  20        set_page_private(page, info);
  21        page_ref_inc(page);
  22}
  23
  24void put_page_bootmem(struct page *page)
  25{
  26        unsigned long type;
  27
  28        type = (unsigned long) page->freelist;
  29        BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
  30               type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
  31
  32        if (page_ref_dec_return(page) == 1) {
  33                page->freelist = NULL;
  34                ClearPagePrivate(page);
  35                set_page_private(page, 0);
  36                INIT_LIST_HEAD(&page->lru);
  37                free_reserved_page(page);
  38        }
  39}
  40
  41#ifndef CONFIG_SPARSEMEM_VMEMMAP
  42static void __init register_page_bootmem_info_section(unsigned long start_pfn)
  43{
  44        unsigned long mapsize, section_nr, i;
  45        struct mem_section *ms;
  46        struct page *page, *memmap;
  47        struct mem_section_usage *usage;
  48
  49        section_nr = pfn_to_section_nr(start_pfn);
  50        ms = __nr_to_section(section_nr);
  51
  52        /* Get section's memmap address */
  53        memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
  54
  55        /*
  56         * Get page for the memmap's phys address
  57         * XXX: need more consideration for sparse_vmemmap...
  58         */
  59        page = virt_to_page(memmap);
  60        mapsize = sizeof(struct page) * PAGES_PER_SECTION;
  61        mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
  62
  63        /* remember memmap's page */
  64        for (i = 0; i < mapsize; i++, page++)
  65                get_page_bootmem(section_nr, page, SECTION_INFO);
  66
  67        usage = ms->usage;
  68        page = virt_to_page(usage);
  69
  70        mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
  71
  72        for (i = 0; i < mapsize; i++, page++)
  73                get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
  74
  75}
  76#else /* CONFIG_SPARSEMEM_VMEMMAP */
  77static void __init register_page_bootmem_info_section(unsigned long start_pfn)
  78{
  79        unsigned long mapsize, section_nr, i;
  80        struct mem_section *ms;
  81        struct page *page, *memmap;
  82        struct mem_section_usage *usage;
  83
  84        section_nr = pfn_to_section_nr(start_pfn);
  85        ms = __nr_to_section(section_nr);
  86
  87        memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
  88
  89        register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
  90
  91        usage = ms->usage;
  92        page = virt_to_page(usage);
  93
  94        mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
  95
  96        for (i = 0; i < mapsize; i++, page++)
  97                get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
  98}
  99#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
 100
 101void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
 102{
 103        unsigned long i, pfn, end_pfn, nr_pages;
 104        int node = pgdat->node_id;
 105        struct page *page;
 106
 107        nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
 108        page = virt_to_page(pgdat);
 109
 110        for (i = 0; i < nr_pages; i++, page++)
 111                get_page_bootmem(node, page, NODE_INFO);
 112
 113        pfn = pgdat->node_start_pfn;
 114        end_pfn = pgdat_end_pfn(pgdat);
 115
 116        /* register section info */
 117        for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
 118                /*
 119                 * Some platforms can assign the same pfn to multiple nodes - on
 120                 * node0 as well as nodeN.  To avoid registering a pfn against
 121                 * multiple nodes we check that this pfn does not already
 122                 * reside in some other nodes.
 123                 */
 124                if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
 125                        register_page_bootmem_info_section(pfn);
 126        }
 127}
 128