1
2
3
4
5
6
7
8
9
10
11
12
13#ifndef __ND_H__
14#define __ND_H__
15#include <linux/libnvdimm.h>
16#include <linux/badblocks.h>
17#include <linux/blkdev.h>
18#include <linux/device.h>
19#include <linux/mutex.h>
20#include <linux/ndctl.h>
21#include <linux/types.h>
22#include <linux/nd.h>
23#include "label.h"
24
25enum {
26
27
28
29
30
31 ND_MAX_LANES = 256,
32 INT_LBASIZE_ALIGNMENT = 64,
33 NVDIMM_IO_ATOMIC = 1,
34};
35
36struct nvdimm_drvdata {
37 struct device *dev;
38 int nslabel_size;
39 struct nd_cmd_get_config_size nsarea;
40 void *data;
41 int ns_current, ns_next;
42 struct resource dpa;
43 struct kref kref;
44};
45
46struct nd_region_data {
47 int ns_count;
48 int ns_active;
49 unsigned int hints_shift;
50 void __iomem *flush_wpq[0];
51};
52
53static inline void __iomem *ndrd_get_flush_wpq(struct nd_region_data *ndrd,
54 int dimm, int hint)
55{
56 unsigned int num = 1 << ndrd->hints_shift;
57 unsigned int mask = num - 1;
58
59 return ndrd->flush_wpq[dimm * num + (hint & mask)];
60}
61
62static inline void ndrd_set_flush_wpq(struct nd_region_data *ndrd, int dimm,
63 int hint, void __iomem *flush)
64{
65 unsigned int num = 1 << ndrd->hints_shift;
66 unsigned int mask = num - 1;
67
68 ndrd->flush_wpq[dimm * num + (hint & mask)] = flush;
69}
70
71static inline struct nd_namespace_index *to_namespace_index(
72 struct nvdimm_drvdata *ndd, int i)
73{
74 if (i < 0)
75 return NULL;
76
77 return ndd->data + sizeof_namespace_index(ndd) * i;
78}
79
80static inline struct nd_namespace_index *to_current_namespace_index(
81 struct nvdimm_drvdata *ndd)
82{
83 return to_namespace_index(ndd, ndd->ns_current);
84}
85
86static inline struct nd_namespace_index *to_next_namespace_index(
87 struct nvdimm_drvdata *ndd)
88{
89 return to_namespace_index(ndd, ndd->ns_next);
90}
91
92unsigned sizeof_namespace_label(struct nvdimm_drvdata *ndd);
93
94#define namespace_label_has(ndd, field) \
95 (offsetof(struct nd_namespace_label, field) \
96 < sizeof_namespace_label(ndd))
97
98#define nd_dbg_dpa(r, d, res, fmt, arg...) \
99 dev_dbg((r) ? &(r)->dev : (d)->dev, "%s: %.13s: %#llx @ %#llx " fmt, \
100 (r) ? dev_name((d)->dev) : "", res ? res->name : "null", \
101 (unsigned long long) (res ? resource_size(res) : 0), \
102 (unsigned long long) (res ? res->start : 0), ##arg)
103
104#define for_each_dpa_resource(ndd, res) \
105 for (res = (ndd)->dpa.child; res; res = res->sibling)
106
107#define for_each_dpa_resource_safe(ndd, res, next) \
108 for (res = (ndd)->dpa.child, next = res ? res->sibling : NULL; \
109 res; res = next, next = next ? next->sibling : NULL)
110
111struct nd_percpu_lane {
112 int count;
113 spinlock_t lock;
114};
115
116enum nd_label_flags {
117 ND_LABEL_REAP,
118};
119struct nd_label_ent {
120 struct list_head list;
121 unsigned long flags;
122 struct nd_namespace_label *label;
123};
124
125enum nd_mapping_lock_class {
126 ND_MAPPING_CLASS0,
127 ND_MAPPING_UUID_SCAN,
128};
129
130struct nd_mapping {
131 struct nvdimm *nvdimm;
132 u64 start;
133 u64 size;
134 int position;
135 struct list_head labels;
136 struct mutex lock;
137
138
139
140
141
142
143 struct nvdimm_drvdata *ndd;
144};
145
146struct nd_region {
147 struct device dev;
148 struct ida ns_ida;
149 struct ida btt_ida;
150 struct ida pfn_ida;
151 struct ida dax_ida;
152 unsigned long flags;
153 struct device *ns_seed;
154 struct device *btt_seed;
155 struct device *pfn_seed;
156 struct device *dax_seed;
157 unsigned long align;
158 u16 ndr_mappings;
159 u64 ndr_size;
160 u64 ndr_start;
161 int id, num_lanes, ro, numa_node, target_node;
162 void *provider_data;
163 struct kernfs_node *bb_state;
164 struct badblocks bb;
165 struct nd_interleave_set *nd_set;
166 struct nd_percpu_lane __percpu *lane;
167 int (*flush)(struct nd_region *nd_region, struct bio *bio);
168 struct nd_mapping mapping[0];
169};
170
171struct nd_blk_region {
172 int (*enable)(struct nvdimm_bus *nvdimm_bus, struct device *dev);
173 int (*do_io)(struct nd_blk_region *ndbr, resource_size_t dpa,
174 void *iobuf, u64 len, int rw);
175 void *blk_provider_data;
176 struct nd_region nd_region;
177};
178
179
180
181
182static inline unsigned nd_inc_seq(unsigned seq)
183{
184 static const unsigned next[] = { 0, 2, 3, 1 };
185
186 return next[seq & 3];
187}
188
189struct btt;
190struct nd_btt {
191 struct device dev;
192 struct nd_namespace_common *ndns;
193 struct btt *btt;
194 unsigned long lbasize;
195 u64 size;
196 u8 *uuid;
197 int id;
198 int initial_offset;
199 u16 version_major;
200 u16 version_minor;
201};
202
203enum nd_pfn_mode {
204 PFN_MODE_NONE,
205 PFN_MODE_RAM,
206 PFN_MODE_PMEM,
207};
208
209struct nd_pfn {
210 int id;
211 u8 *uuid;
212 struct device dev;
213 unsigned long align;
214 unsigned long npfns;
215 enum nd_pfn_mode mode;
216 struct nd_pfn_sb *pfn_sb;
217 struct nd_namespace_common *ndns;
218};
219
220struct nd_dax {
221 struct nd_pfn nd_pfn;
222};
223
224static inline u32 nd_info_block_reserve(void)
225{
226 return ALIGN(SZ_8K, PAGE_SIZE);
227}
228
229enum nd_async_mode {
230 ND_SYNC,
231 ND_ASYNC,
232};
233
234int nd_integrity_init(struct gendisk *disk, unsigned long meta_size);
235void wait_nvdimm_bus_probe_idle(struct device *dev);
236void nd_device_register(struct device *dev);
237void nd_device_unregister(struct device *dev, enum nd_async_mode mode);
238void nd_device_notify(struct device *dev, enum nvdimm_event event);
239int nd_uuid_store(struct device *dev, u8 **uuid_out, const char *buf,
240 size_t len);
241ssize_t nd_size_select_show(unsigned long current_size,
242 const unsigned long *supported, char *buf);
243ssize_t nd_size_select_store(struct device *dev, const char *buf,
244 unsigned long *current_size, const unsigned long *supported);
245int __init nvdimm_init(void);
246int __init nd_region_init(void);
247int __init nd_label_init(void);
248void nvdimm_exit(void);
249void nd_region_exit(void);
250struct nvdimm;
251extern const struct attribute_group nd_device_attribute_group;
252extern const struct attribute_group nd_numa_attribute_group;
253extern const struct attribute_group *nvdimm_bus_attribute_groups[];
254struct nvdimm_drvdata *to_ndd(struct nd_mapping *nd_mapping);
255int nvdimm_check_config_data(struct device *dev);
256int nvdimm_init_nsarea(struct nvdimm_drvdata *ndd);
257int nvdimm_init_config_data(struct nvdimm_drvdata *ndd);
258int nvdimm_get_config_data(struct nvdimm_drvdata *ndd, void *buf,
259 size_t offset, size_t len);
260int nvdimm_set_config_data(struct nvdimm_drvdata *ndd, size_t offset,
261 void *buf, size_t len);
262long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
263 unsigned int len);
264void nvdimm_set_labeling(struct device *dev);
265void nvdimm_set_locked(struct device *dev);
266void nvdimm_clear_locked(struct device *dev);
267int nvdimm_security_setup_events(struct device *dev);
268#if IS_ENABLED(CONFIG_NVDIMM_KEYS)
269int nvdimm_security_unlock(struct device *dev);
270#else
271static inline int nvdimm_security_unlock(struct device *dev)
272{
273 return 0;
274}
275#endif
276struct nd_btt *to_nd_btt(struct device *dev);
277
278struct nd_gen_sb {
279 char reserved[SZ_4K - 8];
280 __le64 checksum;
281};
282
283u64 nd_sb_checksum(struct nd_gen_sb *sb);
284#if IS_ENABLED(CONFIG_BTT)
285int nd_btt_probe(struct device *dev, struct nd_namespace_common *ndns);
286bool is_nd_btt(struct device *dev);
287struct device *nd_btt_create(struct nd_region *nd_region);
288#else
289static inline int nd_btt_probe(struct device *dev,
290 struct nd_namespace_common *ndns)
291{
292 return -ENODEV;
293}
294
295static inline bool is_nd_btt(struct device *dev)
296{
297 return false;
298}
299
300static inline struct device *nd_btt_create(struct nd_region *nd_region)
301{
302 return NULL;
303}
304#endif
305
306struct nd_pfn *to_nd_pfn(struct device *dev);
307#if IS_ENABLED(CONFIG_NVDIMM_PFN)
308
309#ifdef CONFIG_TRANSPARENT_HUGEPAGE
310#define PFN_DEFAULT_ALIGNMENT HPAGE_PMD_SIZE
311#else
312#define PFN_DEFAULT_ALIGNMENT PAGE_SIZE
313#endif
314
315int nd_pfn_probe(struct device *dev, struct nd_namespace_common *ndns);
316bool is_nd_pfn(struct device *dev);
317struct device *nd_pfn_create(struct nd_region *nd_region);
318struct device *nd_pfn_devinit(struct nd_pfn *nd_pfn,
319 struct nd_namespace_common *ndns);
320int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig);
321extern const struct attribute_group *nd_pfn_attribute_groups[];
322#else
323static inline int nd_pfn_probe(struct device *dev,
324 struct nd_namespace_common *ndns)
325{
326 return -ENODEV;
327}
328
329static inline bool is_nd_pfn(struct device *dev)
330{
331 return false;
332}
333
334static inline struct device *nd_pfn_create(struct nd_region *nd_region)
335{
336 return NULL;
337}
338
339static inline int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig)
340{
341 return -ENODEV;
342}
343#endif
344
345struct nd_dax *to_nd_dax(struct device *dev);
346#if IS_ENABLED(CONFIG_NVDIMM_DAX)
347int nd_dax_probe(struct device *dev, struct nd_namespace_common *ndns);
348bool is_nd_dax(struct device *dev);
349struct device *nd_dax_create(struct nd_region *nd_region);
350#else
351static inline int nd_dax_probe(struct device *dev,
352 struct nd_namespace_common *ndns)
353{
354 return -ENODEV;
355}
356
357static inline bool is_nd_dax(struct device *dev)
358{
359 return false;
360}
361
362static inline struct device *nd_dax_create(struct nd_region *nd_region)
363{
364 return NULL;
365}
366#endif
367
368int nd_region_to_nstype(struct nd_region *nd_region);
369int nd_region_register_namespaces(struct nd_region *nd_region, int *err);
370u64 nd_region_interleave_set_cookie(struct nd_region *nd_region,
371 struct nd_namespace_index *nsindex);
372u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region);
373void nvdimm_bus_lock(struct device *dev);
374void nvdimm_bus_unlock(struct device *dev);
375bool is_nvdimm_bus_locked(struct device *dev);
376void nvdimm_check_and_set_ro(struct gendisk *disk);
377void nvdimm_drvdata_release(struct kref *kref);
378void put_ndd(struct nvdimm_drvdata *ndd);
379int nd_label_reserve_dpa(struct nvdimm_drvdata *ndd);
380void nvdimm_free_dpa(struct nvdimm_drvdata *ndd, struct resource *res);
381struct resource *nvdimm_allocate_dpa(struct nvdimm_drvdata *ndd,
382 struct nd_label_id *label_id, resource_size_t start,
383 resource_size_t n);
384resource_size_t nvdimm_namespace_capacity(struct nd_namespace_common *ndns);
385bool nvdimm_namespace_locked(struct nd_namespace_common *ndns);
386struct nd_namespace_common *nvdimm_namespace_common_probe(struct device *dev);
387int nvdimm_namespace_attach_btt(struct nd_namespace_common *ndns);
388int nvdimm_namespace_detach_btt(struct nd_btt *nd_btt);
389const char *nvdimm_namespace_disk_name(struct nd_namespace_common *ndns,
390 char *name);
391unsigned int pmem_sector_size(struct nd_namespace_common *ndns);
392void nvdimm_badblocks_populate(struct nd_region *nd_region,
393 struct badblocks *bb, const struct resource *res);
394int devm_namespace_enable(struct device *dev, struct nd_namespace_common *ndns,
395 resource_size_t size);
396void devm_namespace_disable(struct device *dev,
397 struct nd_namespace_common *ndns);
398#if IS_ENABLED(CONFIG_ND_CLAIM)
399
400#define MAX_STRUCT_PAGE_SIZE 64
401int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap);
402#else
403static inline int nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
404 struct dev_pagemap *pgmap)
405{
406 return -ENXIO;
407}
408#endif
409int nd_blk_region_init(struct nd_region *nd_region);
410int nd_region_activate(struct nd_region *nd_region);
411static inline bool is_bad_pmem(struct badblocks *bb, sector_t sector,
412 unsigned int len)
413{
414 if (bb->count) {
415 sector_t first_bad;
416 int num_bad;
417
418 return !!badblocks_check(bb, sector, len / 512, &first_bad,
419 &num_bad);
420 }
421
422 return false;
423}
424resource_size_t nd_namespace_blk_validate(struct nd_namespace_blk *nsblk);
425const u8 *nd_dev_to_uuid(struct device *dev);
426bool pmem_should_map_pages(struct device *dev);
427#endif
428
