1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _LINUX_MEMREMAP_H_ 3#define _LINUX_MEMREMAP_H_ 4#include <linux/mm.h> 5#include <linux/ioport.h> 6#include <linux/percpu-refcount.h> 7 8#include <asm/pgtable.h> 9 10struct resource; 11struct device; 12 13/** 14 * struct vmem_altmap - pre-allocated storage for vmemmap_populate 15 * @base_pfn: base of the entire dev_pagemap mapping 16 * @reserve: pages mapped, but reserved for driver use (relative to @base) 17 * @free: free pages set aside in the mapping for memmap storage 18 * @align: pages reserved to meet allocation alignments 19 * @alloc: track pages consumed, private to vmemmap_populate() 20 */ 21struct vmem_altmap { 22 const unsigned long base_pfn; 23 const unsigned long reserve; 24 unsigned long free; 25 unsigned long align; 26 unsigned long alloc; 27}; 28 29/* 30 * Specialize ZONE_DEVICE memory into multiple types each having differents 31 * usage. 32 * 33 * MEMORY_DEVICE_HOST: 34 * Persistent device memory (pmem): struct page might be allocated in different 35 * memory and architecture might want to perform special actions. It is similar 36 * to regular memory, in that the CPU can access it transparently. However, 37 * it is likely to have different bandwidth and latency than regular memory. 38 * See Documentation/nvdimm/nvdimm.txt for more information. 39 * 40 * MEMORY_DEVICE_PRIVATE: 41 * Device memory that is not directly addressable by the CPU: CPU can neither 42 * read nor write private memory. In this case, we do still have struct pages 43 * backing the device memory. Doing so simplifies the implementation, but it is 44 * important to remember that there are certain points at which the struct page 45 * must be treated as an opaque object, rather than a "normal" struct page. 46 * 47 * A more complete discussion of unaddressable memory may be found in 48 * include/linux/hmm.h and Documentation/vm/hmm.txt. 49 * 50 * MEMORY_DEVICE_PUBLIC: 51 * Device memory that is cache coherent from device and CPU point of view. This 52 * is use on platform that have an advance system bus (like CAPI or CCIX). A 53 * driver can hotplug the device memory using ZONE_DEVICE and with that memory 54 * type. Any page of a process can be migrated to such memory. However no one 55 * should be allow to pin such memory so that it can always be evicted. 56 */ 57enum memory_type { 58 MEMORY_DEVICE_HOST = 0, 59 MEMORY_DEVICE_PRIVATE, 60 MEMORY_DEVICE_PUBLIC, 61}; 62 63/* 64 * For MEMORY_DEVICE_PRIVATE we use ZONE_DEVICE and extend it with two 65 * callbacks: 66 * page_fault() 67 * page_free() 68 * 69 * Additional notes about MEMORY_DEVICE_PRIVATE may be found in 70 * include/linux/hmm.h and Documentation/vm/hmm.txt. There is also a brief 71 * explanation in include/linux/memory_hotplug.h. 72 * 73 * The page_fault() callback must migrate page back, from device memory to 74 * system memory, so that the CPU can access it. This might fail for various 75 * reasons (device issues, device have been unplugged, ...). When such error 76 * conditions happen, the page_fault() callback must return VM_FAULT_SIGBUS and 77 * set the CPU page table entry to "poisoned". 78 * 79 * Note that because memory cgroup charges are transferred to the device memory, 80 * this should never fail due to memory restrictions. However, allocation 81 * of a regular system page might still fail because we are out of memory. If 82 * that happens, the page_fault() callback must return VM_FAULT_OOM. 83 * 84 * The page_fault() callback can also try to migrate back multiple pages in one 85 * chunk, as an optimization. It must, however, prioritize the faulting address 86 * over all the others. 87 * 88 * 89 * The page_free() callback is called once the page refcount reaches 1 90 * (ZONE_DEVICE pages never reach 0 refcount unless there is a refcount bug. 91 * This allows the device driver to implement its own memory management.) 92 * 93 * For MEMORY_DEVICE_PUBLIC only the page_free() callback matter. 94 */ 95typedef int (*dev_page_fault_t)(struct vm_area_struct *vma, 96 unsigned long addr, 97 const struct page *page, 98 unsigned int flags, 99 pmd_t *pmdp); 100typedef void (*dev_page_free_t)(struct page *page, void *data); 101 102/** 103 * struct dev_pagemap - metadata for ZONE_DEVICE mappings 104 * @page_fault: callback when CPU fault on an unaddressable device page 105 * @page_free: free page callback when page refcount reaches 1 106 * @altmap: pre-allocated/reserved memory for vmemmap allocations 107 * @res: physical address range covered by @ref 108 * @ref: reference count that pins the devm_memremap_pages() mapping 109 * @dev: host device of the mapping for debug 110 * @data: private data pointer for page_free() 111 * @type: memory type: see MEMORY_* in memory_hotplug.h 112 */ 113struct dev_pagemap { 114 dev_page_fault_t page_fault; 115 dev_page_free_t page_free; 116 struct vmem_altmap altmap; 117 bool altmap_valid; 118 struct resource res; 119 struct percpu_ref *ref; 120 struct device *dev; 121 void *data; 122 enum memory_type type; 123}; 124 125#ifdef CONFIG_ZONE_DEVICE 126void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap); 127struct dev_pagemap *get_dev_pagemap(unsigned long pfn, 128 struct dev_pagemap *pgmap); 129 130unsigned long vmem_altmap_offset(struct vmem_altmap *altmap); 131void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns); 132 133static inline bool is_zone_device_page(const struct page *page); 134#else 135static inline void *devm_memremap_pages(struct device *dev, 136 struct dev_pagemap *pgmap) 137{ 138 /* 139 * Fail attempts to call devm_memremap_pages() without 140 * ZONE_DEVICE support enabled, this requires callers to fall 141 * back to plain devm_memremap() based on config 142 */ 143 WARN_ON_ONCE(1); 144 return ERR_PTR(-ENXIO); 145} 146 147static inline struct dev_pagemap *get_dev_pagemap(unsigned long pfn, 148 struct dev_pagemap *pgmap) 149{ 150 return NULL; 151} 152 153static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap) 154{ 155 return 0; 156} 157 158static inline void vmem_altmap_free(struct vmem_altmap *altmap, 159 unsigned long nr_pfns) 160{ 161} 162#endif /* CONFIG_ZONE_DEVICE */ 163 164#if defined(CONFIG_DEVICE_PRIVATE) || defined(CONFIG_DEVICE_PUBLIC) 165static inline bool is_device_private_page(const struct page *page) 166{ 167 return is_zone_device_page(page) && 168 page->pgmap->type == MEMORY_DEVICE_PRIVATE; 169} 170 171static inline bool is_device_public_page(const struct page *page) 172{ 173 return is_zone_device_page(page) && 174 page->pgmap->type == MEMORY_DEVICE_PUBLIC; 175} 176#endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */ 177 178static inline void put_dev_pagemap(struct dev_pagemap *pgmap) 179{ 180 if (pgmap) 181 percpu_ref_put(pgmap->ref); 182} 183#endif /* _LINUX_MEMREMAP_H_ */ 184