1
2
3
4
5
6
7
8
9
10
11
12
13#ifndef __ND_H__
14#define __ND_H__
15#include <linux/libnvdimm.h>
16#include <linux/badblocks.h>
17#include <linux/blkdev.h>
18#include <linux/device.h>
19#include <linux/mutex.h>
20#include <linux/ndctl.h>
21#include <linux/types.h>
22#include <linux/nd.h>
23#include <linux/idr.h>
24#include "label.h"
25
26enum {
27
28
29
30
31
32 ND_MAX_LANES = 256,
33 SECTOR_SHIFT = 9,
34 INT_LBASIZE_ALIGNMENT = 64,
35 NVDIMM_IO_ATOMIC = 1,
36};
37
38struct nvdimm_drvdata {
39 struct device *dev;
40 int nslabel_size;
41 struct nd_cmd_get_config_size nsarea;
42 void *data;
43 int ns_current, ns_next;
44 struct resource dpa;
45 struct kref kref;
46};
47
48struct nd_region_data {
49 int ns_count;
50 int ns_active;
51 unsigned int hints_shift;
52 void __iomem *flush_wpq[0];
53};
54
55static inline void __iomem *ndrd_get_flush_wpq(struct nd_region_data *ndrd,
56 int dimm, int hint)
57{
58 unsigned int num = 1 << ndrd->hints_shift;
59 unsigned int mask = num - 1;
60
61 return ndrd->flush_wpq[dimm * num + (hint & mask)];
62}
63
64static inline void ndrd_set_flush_wpq(struct nd_region_data *ndrd, int dimm,
65 int hint, void __iomem *flush)
66{
67 unsigned int num = 1 << ndrd->hints_shift;
68 unsigned int mask = num - 1;
69
70 ndrd->flush_wpq[dimm * num + (hint & mask)] = flush;
71}
72
73static inline struct nd_namespace_index *to_namespace_index(
74 struct nvdimm_drvdata *ndd, int i)
75{
76 if (i < 0)
77 return NULL;
78
79 return ndd->data + sizeof_namespace_index(ndd) * i;
80}
81
82static inline struct nd_namespace_index *to_current_namespace_index(
83 struct nvdimm_drvdata *ndd)
84{
85 return to_namespace_index(ndd, ndd->ns_current);
86}
87
88static inline struct nd_namespace_index *to_next_namespace_index(
89 struct nvdimm_drvdata *ndd)
90{
91 return to_namespace_index(ndd, ndd->ns_next);
92}
93
94unsigned sizeof_namespace_label(struct nvdimm_drvdata *ndd);
95
96#define namespace_label_has(ndd, field) \
97 (offsetof(struct nd_namespace_label, field) \
98 < sizeof_namespace_label(ndd))
99
100#define nd_dbg_dpa(r, d, res, fmt, arg...) \
101 dev_dbg((r) ? &(r)->dev : (d)->dev, "%s: %.13s: %#llx @ %#llx " fmt, \
102 (r) ? dev_name((d)->dev) : "", res ? res->name : "null", \
103 (unsigned long long) (res ? resource_size(res) : 0), \
104 (unsigned long long) (res ? res->start : 0), ##arg)
105
106#define for_each_dpa_resource(ndd, res) \
107 for (res = (ndd)->dpa.child; res; res = res->sibling)
108
109#define for_each_dpa_resource_safe(ndd, res, next) \
110 for (res = (ndd)->dpa.child, next = res ? res->sibling : NULL; \
111 res; res = next, next = next ? next->sibling : NULL)
112
113struct nd_percpu_lane {
114 int count;
115 spinlock_t lock;
116};
117
118enum nd_label_flags {
119 ND_LABEL_REAP,
120};
121struct nd_label_ent {
122 struct list_head list;
123 unsigned long flags;
124 struct nd_namespace_label *label;
125};
126
127enum nd_mapping_lock_class {
128 ND_MAPPING_CLASS0,
129 ND_MAPPING_UUID_SCAN,
130};
131
132struct nd_mapping {
133 struct nvdimm *nvdimm;
134 u64 start;
135 u64 size;
136 int position;
137 struct list_head labels;
138 struct mutex lock;
139
140
141
142
143
144
145 struct nvdimm_drvdata *ndd;
146};
147
148struct nd_region {
149 struct device dev;
150 struct ida ns_ida;
151 struct ida btt_ida;
152 struct ida pfn_ida;
153 struct ida dax_ida;
154 unsigned long flags;
155 struct device *ns_seed;
156 struct device *btt_seed;
157 struct device *pfn_seed;
158 struct device *dax_seed;
159 u16 ndr_mappings;
160 u64 ndr_size;
161 u64 ndr_start;
162 int id, num_lanes, ro, numa_node;
163 void *provider_data;
164 struct kernfs_node *bb_state;
165 struct badblocks bb;
166 struct nd_interleave_set *nd_set;
167 struct nd_percpu_lane __percpu *lane;
168 struct nd_mapping mapping[0];
169};
170
171struct nd_blk_region {
172 int (*enable)(struct nvdimm_bus *nvdimm_bus, struct device *dev);
173 int (*do_io)(struct nd_blk_region *ndbr, resource_size_t dpa,
174 void *iobuf, u64 len, int rw);
175 void *blk_provider_data;
176 struct nd_region nd_region;
177};
178
179
180
181
182static inline unsigned nd_inc_seq(unsigned seq)
183{
184 static const unsigned next[] = { 0, 2, 3, 1 };
185
186 return next[seq & 3];
187}
188
189struct btt;
190struct nd_btt {
191 struct device dev;
192 struct nd_namespace_common *ndns;
193 struct btt *btt;
194 unsigned long lbasize;
195 u64 size;
196 u8 *uuid;
197 int id;
198 int initial_offset;
199 u16 version_major;
200 u16 version_minor;
201};
202
203enum nd_pfn_mode {
204 PFN_MODE_NONE,
205 PFN_MODE_RAM,
206 PFN_MODE_PMEM,
207};
208
209struct nd_pfn {
210 int id;
211 u8 *uuid;
212 struct device dev;
213 unsigned long align;
214 unsigned long npfns;
215 enum nd_pfn_mode mode;
216 struct nd_pfn_sb *pfn_sb;
217 struct nd_namespace_common *ndns;
218};
219
220struct nd_dax {
221 struct nd_pfn nd_pfn;
222};
223
224enum nd_async_mode {
225 ND_SYNC,
226 ND_ASYNC,
227};
228
229int nd_integrity_init(struct gendisk *disk, unsigned long meta_size);
230void wait_nvdimm_bus_probe_idle(struct device *dev);
231void nd_device_register(struct device *dev);
232void nd_device_unregister(struct device *dev, enum nd_async_mode mode);
233void nd_device_notify(struct device *dev, enum nvdimm_event event);
234int nd_uuid_store(struct device *dev, u8 **uuid_out, const char *buf,
235 size_t len);
236ssize_t nd_size_select_show(unsigned long current_size,
237 const unsigned long *supported, char *buf);
238ssize_t nd_size_select_store(struct device *dev, const char *buf,
239 unsigned long *current_size, const unsigned long *supported);
240int __init nvdimm_init(void);
241int __init nd_region_init(void);
242int __init nd_label_init(void);
243void nvdimm_exit(void);
244void nd_region_exit(void);
245struct nvdimm;
246struct nvdimm_drvdata *to_ndd(struct nd_mapping *nd_mapping);
247int nvdimm_check_config_data(struct device *dev);
248int nvdimm_init_nsarea(struct nvdimm_drvdata *ndd);
249int nvdimm_init_config_data(struct nvdimm_drvdata *ndd);
250int nvdimm_get_config_data(struct nvdimm_drvdata *ndd, void *buf,
251 size_t offset, size_t len);
252int nvdimm_set_config_data(struct nvdimm_drvdata *ndd, size_t offset,
253 void *buf, size_t len);
254long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
255 unsigned int len);
256void nvdimm_set_aliasing(struct device *dev);
257void nvdimm_set_locked(struct device *dev);
258void nvdimm_clear_locked(struct device *dev);
259int nvdimm_security_setup_events(struct device *dev);
260#if IS_ENABLED(CONFIG_NVDIMM_KEYS)
261int nvdimm_security_unlock(struct device *dev);
262#else
263static inline int nvdimm_security_unlock(struct device *dev)
264{
265 return 0;
266}
267#endif
268struct nd_btt *to_nd_btt(struct device *dev);
269
270struct nd_gen_sb {
271 char reserved[SZ_4K - 8];
272 __le64 checksum;
273};
274
275u64 nd_sb_checksum(struct nd_gen_sb *sb);
276#if IS_ENABLED(CONFIG_BTT)
277int nd_btt_probe(struct device *dev, struct nd_namespace_common *ndns);
278bool is_nd_btt(struct device *dev);
279struct device *nd_btt_create(struct nd_region *nd_region);
280#else
281static inline int nd_btt_probe(struct device *dev,
282 struct nd_namespace_common *ndns)
283{
284 return -ENODEV;
285}
286
287static inline bool is_nd_btt(struct device *dev)
288{
289 return false;
290}
291
292static inline struct device *nd_btt_create(struct nd_region *nd_region)
293{
294 return NULL;
295}
296#endif
297
298struct nd_pfn *to_nd_pfn(struct device *dev);
299#if IS_ENABLED(CONFIG_NVDIMM_PFN)
300
301#ifdef CONFIG_TRANSPARENT_HUGEPAGE
302#define PFN_DEFAULT_ALIGNMENT HPAGE_PMD_SIZE
303#else
304#define PFN_DEFAULT_ALIGNMENT PAGE_SIZE
305#endif
306
307int nd_pfn_probe(struct device *dev, struct nd_namespace_common *ndns);
308bool is_nd_pfn(struct device *dev);
309struct device *nd_pfn_create(struct nd_region *nd_region);
310struct device *nd_pfn_devinit(struct nd_pfn *nd_pfn,
311 struct nd_namespace_common *ndns);
312int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig);
313extern struct attribute_group nd_pfn_attribute_group;
314#else
315static inline int nd_pfn_probe(struct device *dev,
316 struct nd_namespace_common *ndns)
317{
318 return -ENODEV;
319}
320
321static inline bool is_nd_pfn(struct device *dev)
322{
323 return false;
324}
325
326static inline struct device *nd_pfn_create(struct nd_region *nd_region)
327{
328 return NULL;
329}
330
331static inline int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig)
332{
333 return -ENODEV;
334}
335#endif
336
337struct nd_dax *to_nd_dax(struct device *dev);
338#if IS_ENABLED(CONFIG_NVDIMM_DAX)
339int nd_dax_probe(struct device *dev, struct nd_namespace_common *ndns);
340bool is_nd_dax(struct device *dev);
341struct device *nd_dax_create(struct nd_region *nd_region);
342#else
343static inline int nd_dax_probe(struct device *dev,
344 struct nd_namespace_common *ndns)
345{
346 return -ENODEV;
347}
348
349static inline bool is_nd_dax(struct device *dev)
350{
351 return false;
352}
353
354static inline struct device *nd_dax_create(struct nd_region *nd_region)
355{
356 return NULL;
357}
358#endif
359
360int nd_region_to_nstype(struct nd_region *nd_region);
361int nd_region_register_namespaces(struct nd_region *nd_region, int *err);
362u64 nd_region_interleave_set_cookie(struct nd_region *nd_region,
363 struct nd_namespace_index *nsindex);
364u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region);
365void nvdimm_bus_lock(struct device *dev);
366void nvdimm_bus_unlock(struct device *dev);
367bool is_nvdimm_bus_locked(struct device *dev);
368int nvdimm_revalidate_disk(struct gendisk *disk);
369void nvdimm_drvdata_release(struct kref *kref);
370void put_ndd(struct nvdimm_drvdata *ndd);
371int nd_label_reserve_dpa(struct nvdimm_drvdata *ndd);
372void nvdimm_free_dpa(struct nvdimm_drvdata *ndd, struct resource *res);
373struct resource *nvdimm_allocate_dpa(struct nvdimm_drvdata *ndd,
374 struct nd_label_id *label_id, resource_size_t start,
375 resource_size_t n);
376resource_size_t nvdimm_namespace_capacity(struct nd_namespace_common *ndns);
377bool nvdimm_namespace_locked(struct nd_namespace_common *ndns);
378struct nd_namespace_common *nvdimm_namespace_common_probe(struct device *dev);
379int nvdimm_namespace_attach_btt(struct nd_namespace_common *ndns);
380int nvdimm_namespace_detach_btt(struct nd_btt *nd_btt);
381const char *nvdimm_namespace_disk_name(struct nd_namespace_common *ndns,
382 char *name);
383unsigned int pmem_sector_size(struct nd_namespace_common *ndns);
384void nvdimm_badblocks_populate(struct nd_region *nd_region,
385 struct badblocks *bb, const struct resource *res);
386#if IS_ENABLED(CONFIG_ND_CLAIM)
387int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap);
388int devm_nsio_enable(struct device *dev, struct nd_namespace_io *nsio);
389void devm_nsio_disable(struct device *dev, struct nd_namespace_io *nsio);
390#else
391static inline int nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
392 struct dev_pagemap *pgmap)
393{
394 return -ENXIO;
395}
396static inline int devm_nsio_enable(struct device *dev,
397 struct nd_namespace_io *nsio)
398{
399 return -ENXIO;
400}
401static inline void devm_nsio_disable(struct device *dev,
402 struct nd_namespace_io *nsio)
403{
404}
405#endif
406int nd_blk_region_init(struct nd_region *nd_region);
407int nd_region_activate(struct nd_region *nd_region);
408void __nd_iostat_start(struct bio *bio, unsigned long *start);
409static inline bool nd_iostat_start(struct bio *bio, unsigned long *start)
410{
411 struct gendisk *disk = bio->bi_bdev->bd_disk;
412
413 if (!blk_queue_io_stat(disk->queue))
414 return false;
415
416 *start = jiffies;
417 generic_start_io_acct(disk->queue, bio_data_dir(bio),
418 bio_sectors(bio), &disk->part0);
419 return true;
420}
421static inline void nd_iostat_end(struct bio *bio, unsigned long start)
422{
423 struct gendisk *disk = bio->bi_bdev->bd_disk;
424
425 generic_end_io_acct(disk->queue, bio_data_dir(bio), &disk->part0,
426 start);
427}
428static inline bool is_bad_pmem(struct badblocks *bb, sector_t sector,
429 unsigned int len)
430{
431 if (bb->count) {
432 sector_t first_bad;
433 int num_bad;
434
435 return !!badblocks_check(bb, sector, len / 512, &first_bad,
436 &num_bad);
437 }
438
439 return false;
440}
441resource_size_t nd_namespace_blk_validate(struct nd_namespace_blk *nsblk);
442const u8 *nd_dev_to_uuid(struct device *dev);
443bool pmem_should_map_pages(struct device *dev);
444#endif
445
