1
2
3#include <linux/memremap.h>
4#include <linux/device.h>
5#include <linux/mutex.h>
6#include <linux/list.h>
7#include <linux/slab.h>
8#include <linux/dax.h>
9#include <linux/io.h>
10#include "dax-private.h"
11#include "bus.h"
12
13static struct class *dax_class;
14
15static DEFINE_MUTEX(dax_bus_lock);
16
17#define DAX_NAME_LEN 30
18struct dax_id {
19 struct list_head list;
20 char dev_name[DAX_NAME_LEN];
21};
22
23static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
24{
25
26
27
28
29 return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
30}
31
32static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
33{
34 return container_of(drv, struct dax_device_driver, drv);
35}
36
37static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
38 const char *dev_name)
39{
40 struct dax_id *dax_id;
41
42 lockdep_assert_held(&dax_bus_lock);
43
44 list_for_each_entry(dax_id, &dax_drv->ids, list)
45 if (sysfs_streq(dax_id->dev_name, dev_name))
46 return dax_id;
47 return NULL;
48}
49
50static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
51{
52 int match;
53
54 mutex_lock(&dax_bus_lock);
55 match = !!__dax_match_id(dax_drv, dev_name(dev));
56 mutex_unlock(&dax_bus_lock);
57
58 return match;
59}
60
61enum id_action {
62 ID_REMOVE,
63 ID_ADD,
64};
65
66static ssize_t do_id_store(struct device_driver *drv, const char *buf,
67 size_t count, enum id_action action)
68{
69 struct dax_device_driver *dax_drv = to_dax_drv(drv);
70 unsigned int region_id, id;
71 char devname[DAX_NAME_LEN];
72 struct dax_id *dax_id;
73 ssize_t rc = count;
74 int fields;
75
76 fields = sscanf(buf, "dax%d.%d", ®ion_id, &id);
77 if (fields != 2)
78 return -EINVAL;
79 sprintf(devname, "dax%d.%d", region_id, id);
80 if (!sysfs_streq(buf, devname))
81 return -EINVAL;
82
83 mutex_lock(&dax_bus_lock);
84 dax_id = __dax_match_id(dax_drv, buf);
85 if (!dax_id) {
86 if (action == ID_ADD) {
87 dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
88 if (dax_id) {
89 strncpy(dax_id->dev_name, buf, DAX_NAME_LEN);
90 list_add(&dax_id->list, &dax_drv->ids);
91 } else
92 rc = -ENOMEM;
93 }
94 } else if (action == ID_REMOVE) {
95 list_del(&dax_id->list);
96 kfree(dax_id);
97 }
98 mutex_unlock(&dax_bus_lock);
99
100 if (rc < 0)
101 return rc;
102 if (action == ID_ADD)
103 rc = driver_attach(drv);
104 if (rc)
105 return rc;
106 return count;
107}
108
109static ssize_t new_id_store(struct device_driver *drv, const char *buf,
110 size_t count)
111{
112 return do_id_store(drv, buf, count, ID_ADD);
113}
114static DRIVER_ATTR_WO(new_id);
115
116static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
117 size_t count)
118{
119 return do_id_store(drv, buf, count, ID_REMOVE);
120}
121static DRIVER_ATTR_WO(remove_id);
122
123static struct attribute *dax_drv_attrs[] = {
124 &driver_attr_new_id.attr,
125 &driver_attr_remove_id.attr,
126 NULL,
127};
128ATTRIBUTE_GROUPS(dax_drv);
129
130static int dax_bus_match(struct device *dev, struct device_driver *drv);
131
132static bool is_static(struct dax_region *dax_region)
133{
134 return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0;
135}
136
137static u64 dev_dax_size(struct dev_dax *dev_dax)
138{
139 u64 size = 0;
140 int i;
141
142 device_lock_assert(&dev_dax->dev);
143
144 for (i = 0; i < dev_dax->nr_range; i++)
145 size += range_len(&dev_dax->ranges[i].range);
146
147 return size;
148}
149
150static int dax_bus_probe(struct device *dev)
151{
152 struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
153 struct dev_dax *dev_dax = to_dev_dax(dev);
154 struct dax_region *dax_region = dev_dax->region;
155 int rc;
156
157 if (dev_dax_size(dev_dax) == 0 || dev_dax->id < 0)
158 return -ENXIO;
159
160 rc = dax_drv->probe(dev_dax);
161
162 if (rc || is_static(dax_region))
163 return rc;
164
165
166
167
168
169 if (dax_region->seed == dev)
170 dax_region->seed = NULL;
171
172 return 0;
173}
174
175static void dax_bus_remove(struct device *dev)
176{
177 struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
178 struct dev_dax *dev_dax = to_dev_dax(dev);
179
180 if (dax_drv->remove)
181 dax_drv->remove(dev_dax);
182}
183
184static struct bus_type dax_bus_type = {
185 .name = "dax",
186 .uevent = dax_bus_uevent,
187 .match = dax_bus_match,
188 .probe = dax_bus_probe,
189 .remove = dax_bus_remove,
190 .drv_groups = dax_drv_groups,
191};
192
193static int dax_bus_match(struct device *dev, struct device_driver *drv)
194{
195 struct dax_device_driver *dax_drv = to_dax_drv(drv);
196
197
198
199
200
201 if (dax_drv->match_always)
202 return 1;
203
204 return dax_match_id(dax_drv, dev);
205}
206
207
208
209
210
211
212static ssize_t id_show(struct device *dev,
213 struct device_attribute *attr, char *buf)
214{
215 struct dax_region *dax_region = dev_get_drvdata(dev);
216
217 return sprintf(buf, "%d\n", dax_region->id);
218}
219static DEVICE_ATTR_RO(id);
220
221static ssize_t region_size_show(struct device *dev,
222 struct device_attribute *attr, char *buf)
223{
224 struct dax_region *dax_region = dev_get_drvdata(dev);
225
226 return sprintf(buf, "%llu\n", (unsigned long long)
227 resource_size(&dax_region->res));
228}
229static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
230 region_size_show, NULL);
231
232static ssize_t region_align_show(struct device *dev,
233 struct device_attribute *attr, char *buf)
234{
235 struct dax_region *dax_region = dev_get_drvdata(dev);
236
237 return sprintf(buf, "%u\n", dax_region->align);
238}
239static struct device_attribute dev_attr_region_align =
240 __ATTR(align, 0400, region_align_show, NULL);
241
242#define for_each_dax_region_resource(dax_region, res) \
243 for (res = (dax_region)->res.child; res; res = res->sibling)
244
245static unsigned long long dax_region_avail_size(struct dax_region *dax_region)
246{
247 resource_size_t size = resource_size(&dax_region->res);
248 struct resource *res;
249
250 device_lock_assert(dax_region->dev);
251
252 for_each_dax_region_resource(dax_region, res)
253 size -= resource_size(res);
254 return size;
255}
256
257static ssize_t available_size_show(struct device *dev,
258 struct device_attribute *attr, char *buf)
259{
260 struct dax_region *dax_region = dev_get_drvdata(dev);
261 unsigned long long size;
262
263 device_lock(dev);
264 size = dax_region_avail_size(dax_region);
265 device_unlock(dev);
266
267 return sprintf(buf, "%llu\n", size);
268}
269static DEVICE_ATTR_RO(available_size);
270
271static ssize_t seed_show(struct device *dev,
272 struct device_attribute *attr, char *buf)
273{
274 struct dax_region *dax_region = dev_get_drvdata(dev);
275 struct device *seed;
276 ssize_t rc;
277
278 if (is_static(dax_region))
279 return -EINVAL;
280
281 device_lock(dev);
282 seed = dax_region->seed;
283 rc = sprintf(buf, "%s\n", seed ? dev_name(seed) : "");
284 device_unlock(dev);
285
286 return rc;
287}
288static DEVICE_ATTR_RO(seed);
289
290static ssize_t create_show(struct device *dev,
291 struct device_attribute *attr, char *buf)
292{
293 struct dax_region *dax_region = dev_get_drvdata(dev);
294 struct device *youngest;
295 ssize_t rc;
296
297 if (is_static(dax_region))
298 return -EINVAL;
299
300 device_lock(dev);
301 youngest = dax_region->youngest;
302 rc = sprintf(buf, "%s\n", youngest ? dev_name(youngest) : "");
303 device_unlock(dev);
304
305 return rc;
306}
307
308static ssize_t create_store(struct device *dev, struct device_attribute *attr,
309 const char *buf, size_t len)
310{
311 struct dax_region *dax_region = dev_get_drvdata(dev);
312 unsigned long long avail;
313 ssize_t rc;
314 int val;
315
316 if (is_static(dax_region))
317 return -EINVAL;
318
319 rc = kstrtoint(buf, 0, &val);
320 if (rc)
321 return rc;
322 if (val != 1)
323 return -EINVAL;
324
325 device_lock(dev);
326 avail = dax_region_avail_size(dax_region);
327 if (avail == 0)
328 rc = -ENOSPC;
329 else {
330 struct dev_dax_data data = {
331 .dax_region = dax_region,
332 .size = 0,
333 .id = -1,
334 };
335 struct dev_dax *dev_dax = devm_create_dev_dax(&data);
336
337 if (IS_ERR(dev_dax))
338 rc = PTR_ERR(dev_dax);
339 else {
340
341
342
343
344
345
346
347 if (!dax_region->seed)
348 dax_region->seed = &dev_dax->dev;
349 dax_region->youngest = &dev_dax->dev;
350 rc = len;
351 }
352 }
353 device_unlock(dev);
354
355 return rc;
356}
357static DEVICE_ATTR_RW(create);
358
359void kill_dev_dax(struct dev_dax *dev_dax)
360{
361 struct dax_device *dax_dev = dev_dax->dax_dev;
362 struct inode *inode = dax_inode(dax_dev);
363
364 kill_dax(dax_dev);
365 unmap_mapping_range(inode->i_mapping, 0, 0, 1);
366}
367EXPORT_SYMBOL_GPL(kill_dev_dax);
368
369static void trim_dev_dax_range(struct dev_dax *dev_dax)
370{
371 int i = dev_dax->nr_range - 1;
372 struct range *range = &dev_dax->ranges[i].range;
373 struct dax_region *dax_region = dev_dax->region;
374
375 device_lock_assert(dax_region->dev);
376 dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i,
377 (unsigned long long)range->start,
378 (unsigned long long)range->end);
379
380 __release_region(&dax_region->res, range->start, range_len(range));
381 if (--dev_dax->nr_range == 0) {
382 kfree(dev_dax->ranges);
383 dev_dax->ranges = NULL;
384 }
385}
386
387static void free_dev_dax_ranges(struct dev_dax *dev_dax)
388{
389 while (dev_dax->nr_range)
390 trim_dev_dax_range(dev_dax);
391}
392
393static void unregister_dev_dax(void *dev)
394{
395 struct dev_dax *dev_dax = to_dev_dax(dev);
396
397 dev_dbg(dev, "%s\n", __func__);
398
399 kill_dev_dax(dev_dax);
400 free_dev_dax_ranges(dev_dax);
401 device_del(dev);
402 put_device(dev);
403}
404
405
406static int __free_dev_dax_id(struct dev_dax *dev_dax)
407{
408 struct dax_region *dax_region = dev_dax->region;
409 struct device *dev = &dev_dax->dev;
410 int rc = dev_dax->id;
411
412 device_lock_assert(dev);
413
414 if (is_static(dax_region) || dev_dax->id < 0)
415 return -1;
416 ida_free(&dax_region->ida, dev_dax->id);
417 dev_dax->id = -1;
418 return rc;
419}
420
421static int free_dev_dax_id(struct dev_dax *dev_dax)
422{
423 struct device *dev = &dev_dax->dev;
424 int rc;
425
426 device_lock(dev);
427 rc = __free_dev_dax_id(dev_dax);
428 device_unlock(dev);
429 return rc;
430}
431
432static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
433 const char *buf, size_t len)
434{
435 struct dax_region *dax_region = dev_get_drvdata(dev);
436 struct dev_dax *dev_dax;
437 struct device *victim;
438 bool do_del = false;
439 int rc;
440
441 if (is_static(dax_region))
442 return -EINVAL;
443
444 victim = device_find_child_by_name(dax_region->dev, buf);
445 if (!victim)
446 return -ENXIO;
447
448 device_lock(dev);
449 device_lock(victim);
450 dev_dax = to_dev_dax(victim);
451 if (victim->driver || dev_dax_size(dev_dax))
452 rc = -EBUSY;
453 else {
454
455
456
457
458
459
460 if (dev_dax->id > 0) {
461 do_del = __free_dev_dax_id(dev_dax) >= 0;
462 rc = len;
463 if (dax_region->seed == victim)
464 dax_region->seed = NULL;
465 if (dax_region->youngest == victim)
466 dax_region->youngest = NULL;
467 } else
468 rc = -EBUSY;
469 }
470 device_unlock(victim);
471
472
473 if (do_del)
474 devm_release_action(dev, unregister_dev_dax, victim);
475 device_unlock(dev);
476 put_device(victim);
477
478 return rc;
479}
480static DEVICE_ATTR_WO(delete);
481
482static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a,
483 int n)
484{
485 struct device *dev = container_of(kobj, struct device, kobj);
486 struct dax_region *dax_region = dev_get_drvdata(dev);
487
488 if (is_static(dax_region))
489 if (a == &dev_attr_available_size.attr
490 || a == &dev_attr_create.attr
491 || a == &dev_attr_seed.attr
492 || a == &dev_attr_delete.attr)
493 return 0;
494 return a->mode;
495}
496
497static struct attribute *dax_region_attributes[] = {
498 &dev_attr_available_size.attr,
499 &dev_attr_region_size.attr,
500 &dev_attr_region_align.attr,
501 &dev_attr_create.attr,
502 &dev_attr_seed.attr,
503 &dev_attr_delete.attr,
504 &dev_attr_id.attr,
505 NULL,
506};
507
508static const struct attribute_group dax_region_attribute_group = {
509 .name = "dax_region",
510 .attrs = dax_region_attributes,
511 .is_visible = dax_region_visible,
512};
513
514static const struct attribute_group *dax_region_attribute_groups[] = {
515 &dax_region_attribute_group,
516 NULL,
517};
518
519static void dax_region_free(struct kref *kref)
520{
521 struct dax_region *dax_region;
522
523 dax_region = container_of(kref, struct dax_region, kref);
524 kfree(dax_region);
525}
526
527void dax_region_put(struct dax_region *dax_region)
528{
529 kref_put(&dax_region->kref, dax_region_free);
530}
531EXPORT_SYMBOL_GPL(dax_region_put);
532
533static void dax_region_unregister(void *region)
534{
535 struct dax_region *dax_region = region;
536
537 sysfs_remove_groups(&dax_region->dev->kobj,
538 dax_region_attribute_groups);
539 dax_region_put(dax_region);
540}
541
542struct dax_region *alloc_dax_region(struct device *parent, int region_id,
543 struct range *range, int target_node, unsigned int align,
544 unsigned long flags)
545{
546 struct dax_region *dax_region;
547
548
549
550
551
552
553 if (dev_get_drvdata(parent)) {
554 dev_WARN(parent, "dax core failed to setup private data\n");
555 return NULL;
556 }
557
558 if (!IS_ALIGNED(range->start, align)
559 || !IS_ALIGNED(range_len(range), align))
560 return NULL;
561
562 dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
563 if (!dax_region)
564 return NULL;
565
566 dev_set_drvdata(parent, dax_region);
567 kref_init(&dax_region->kref);
568 dax_region->id = region_id;
569 dax_region->align = align;
570 dax_region->dev = parent;
571 dax_region->target_node = target_node;
572 ida_init(&dax_region->ida);
573 dax_region->res = (struct resource) {
574 .start = range->start,
575 .end = range->end,
576 .flags = IORESOURCE_MEM | flags,
577 };
578
579 if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
580 kfree(dax_region);
581 return NULL;
582 }
583
584 kref_get(&dax_region->kref);
585 if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
586 return NULL;
587 return dax_region;
588}
589EXPORT_SYMBOL_GPL(alloc_dax_region);
590
591static void dax_mapping_release(struct device *dev)
592{
593 struct dax_mapping *mapping = to_dax_mapping(dev);
594 struct dev_dax *dev_dax = to_dev_dax(dev->parent);
595
596 ida_free(&dev_dax->ida, mapping->id);
597 kfree(mapping);
598}
599
600static void unregister_dax_mapping(void *data)
601{
602 struct device *dev = data;
603 struct dax_mapping *mapping = to_dax_mapping(dev);
604 struct dev_dax *dev_dax = to_dev_dax(dev->parent);
605 struct dax_region *dax_region = dev_dax->region;
606
607 dev_dbg(dev, "%s\n", __func__);
608
609 device_lock_assert(dax_region->dev);
610
611 dev_dax->ranges[mapping->range_id].mapping = NULL;
612 mapping->range_id = -1;
613
614 device_del(dev);
615 put_device(dev);
616}
617
618static struct dev_dax_range *get_dax_range(struct device *dev)
619{
620 struct dax_mapping *mapping = to_dax_mapping(dev);
621 struct dev_dax *dev_dax = to_dev_dax(dev->parent);
622 struct dax_region *dax_region = dev_dax->region;
623
624 device_lock(dax_region->dev);
625 if (mapping->range_id < 0) {
626 device_unlock(dax_region->dev);
627 return NULL;
628 }
629
630 return &dev_dax->ranges[mapping->range_id];
631}
632
633static void put_dax_range(struct dev_dax_range *dax_range)
634{
635 struct dax_mapping *mapping = dax_range->mapping;
636 struct dev_dax *dev_dax = to_dev_dax(mapping->dev.parent);
637 struct dax_region *dax_region = dev_dax->region;
638
639 device_unlock(dax_region->dev);
640}
641
642static ssize_t start_show(struct device *dev,
643 struct device_attribute *attr, char *buf)
644{
645 struct dev_dax_range *dax_range;
646 ssize_t rc;
647
648 dax_range = get_dax_range(dev);
649 if (!dax_range)
650 return -ENXIO;
651 rc = sprintf(buf, "%#llx\n", dax_range->range.start);
652 put_dax_range(dax_range);
653
654 return rc;
655}
656static DEVICE_ATTR(start, 0400, start_show, NULL);
657
658static ssize_t end_show(struct device *dev,
659 struct device_attribute *attr, char *buf)
660{
661 struct dev_dax_range *dax_range;
662 ssize_t rc;
663
664 dax_range = get_dax_range(dev);
665 if (!dax_range)
666 return -ENXIO;
667 rc = sprintf(buf, "%#llx\n", dax_range->range.end);
668 put_dax_range(dax_range);
669
670 return rc;
671}
672static DEVICE_ATTR(end, 0400, end_show, NULL);
673
674static ssize_t pgoff_show(struct device *dev,
675 struct device_attribute *attr, char *buf)
676{
677 struct dev_dax_range *dax_range;
678 ssize_t rc;
679
680 dax_range = get_dax_range(dev);
681 if (!dax_range)
682 return -ENXIO;
683 rc = sprintf(buf, "%#lx\n", dax_range->pgoff);
684 put_dax_range(dax_range);
685
686 return rc;
687}
688static DEVICE_ATTR(page_offset, 0400, pgoff_show, NULL);
689
690static struct attribute *dax_mapping_attributes[] = {
691 &dev_attr_start.attr,
692 &dev_attr_end.attr,
693 &dev_attr_page_offset.attr,
694 NULL,
695};
696
697static const struct attribute_group dax_mapping_attribute_group = {
698 .attrs = dax_mapping_attributes,
699};
700
701static const struct attribute_group *dax_mapping_attribute_groups[] = {
702 &dax_mapping_attribute_group,
703 NULL,
704};
705
706static struct device_type dax_mapping_type = {
707 .release = dax_mapping_release,
708 .groups = dax_mapping_attribute_groups,
709};
710
711static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
712{
713 struct dax_region *dax_region = dev_dax->region;
714 struct dax_mapping *mapping;
715 struct device *dev;
716 int rc;
717
718 device_lock_assert(dax_region->dev);
719
720 if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver,
721 "region disabled\n"))
722 return -ENXIO;
723
724 mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
725 if (!mapping)
726 return -ENOMEM;
727 mapping->range_id = range_id;
728 mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL);
729 if (mapping->id < 0) {
730 kfree(mapping);
731 return -ENOMEM;
732 }
733 dev_dax->ranges[range_id].mapping = mapping;
734 dev = &mapping->dev;
735 device_initialize(dev);
736 dev->parent = &dev_dax->dev;
737 dev->type = &dax_mapping_type;
738 dev_set_name(dev, "mapping%d", mapping->id);
739 rc = device_add(dev);
740 if (rc) {
741 put_device(dev);
742 return rc;
743 }
744
745 rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_mapping,
746 dev);
747 if (rc)
748 return rc;
749 return 0;
750}
751
752static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start,
753 resource_size_t size)
754{
755 struct dax_region *dax_region = dev_dax->region;
756 struct resource *res = &dax_region->res;
757 struct device *dev = &dev_dax->dev;
758 struct dev_dax_range *ranges;
759 unsigned long pgoff = 0;
760 struct resource *alloc;
761 int i, rc;
762
763 device_lock_assert(dax_region->dev);
764
765
766 if (!size) {
767 if (dev_WARN_ONCE(dev, dev_dax->nr_range,
768 "0-size allocation must be first\n"))
769 return -EBUSY;
770
771 return 0;
772 }
773
774 alloc = __request_region(res, start, size, dev_name(dev), 0);
775 if (!alloc)
776 return -ENOMEM;
777
778 ranges = krealloc(dev_dax->ranges, sizeof(*ranges)
779 * (dev_dax->nr_range + 1), GFP_KERNEL);
780 if (!ranges) {
781 __release_region(res, alloc->start, resource_size(alloc));
782 return -ENOMEM;
783 }
784
785 for (i = 0; i < dev_dax->nr_range; i++)
786 pgoff += PHYS_PFN(range_len(&ranges[i].range));
787 dev_dax->ranges = ranges;
788 ranges[dev_dax->nr_range++] = (struct dev_dax_range) {
789 .pgoff = pgoff,
790 .range = {
791 .start = alloc->start,
792 .end = alloc->end,
793 },
794 };
795
796 dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
797 &alloc->start, &alloc->end);
798
799
800
801
802
803 if (!device_is_registered(&dev_dax->dev))
804 return 0;
805
806 rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1);
807 if (rc)
808 trim_dev_dax_range(dev_dax);
809
810 return rc;
811}
812
813static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size)
814{
815 int last_range = dev_dax->nr_range - 1;
816 struct dev_dax_range *dax_range = &dev_dax->ranges[last_range];
817 struct dax_region *dax_region = dev_dax->region;
818 bool is_shrink = resource_size(res) > size;
819 struct range *range = &dax_range->range;
820 struct device *dev = &dev_dax->dev;
821 int rc;
822
823 device_lock_assert(dax_region->dev);
824
825 if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n"))
826 return -EINVAL;
827
828 rc = adjust_resource(res, range->start, size);
829 if (rc)
830 return rc;
831
832 *range = (struct range) {
833 .start = range->start,
834 .end = range->start + size - 1,
835 };
836
837 dev_dbg(dev, "%s range[%d]: %#llx:%#llx\n", is_shrink ? "shrink" : "extend",
838 last_range, (unsigned long long) range->start,
839 (unsigned long long) range->end);
840
841 return 0;
842}
843
844static ssize_t size_show(struct device *dev,
845 struct device_attribute *attr, char *buf)
846{
847 struct dev_dax *dev_dax = to_dev_dax(dev);
848 unsigned long long size;
849
850 device_lock(dev);
851 size = dev_dax_size(dev_dax);
852 device_unlock(dev);
853
854 return sprintf(buf, "%llu\n", size);
855}
856
857static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size)
858{
859
860
861
862
863 return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align()));
864}
865
866static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size)
867{
868 resource_size_t to_shrink = dev_dax_size(dev_dax) - size;
869 struct dax_region *dax_region = dev_dax->region;
870 struct device *dev = &dev_dax->dev;
871 int i;
872
873 for (i = dev_dax->nr_range - 1; i >= 0; i--) {
874 struct range *range = &dev_dax->ranges[i].range;
875 struct dax_mapping *mapping = dev_dax->ranges[i].mapping;
876 struct resource *adjust = NULL, *res;
877 resource_size_t shrink;
878
879 shrink = min_t(u64, to_shrink, range_len(range));
880 if (shrink >= range_len(range)) {
881 devm_release_action(dax_region->dev,
882 unregister_dax_mapping, &mapping->dev);
883 trim_dev_dax_range(dev_dax);
884 to_shrink -= shrink;
885 if (!to_shrink)
886 break;
887 continue;
888 }
889
890 for_each_dax_region_resource(dax_region, res)
891 if (strcmp(res->name, dev_name(dev)) == 0
892 && res->start == range->start) {
893 adjust = res;
894 break;
895 }
896
897 if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1,
898 "failed to find matching resource\n"))
899 return -ENXIO;
900 return adjust_dev_dax_range(dev_dax, adjust, range_len(range)
901 - shrink);
902 }
903 return 0;
904}
905
906
907
908
909
910static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res)
911{
912 struct dev_dax_range *last;
913 int i;
914
915 if (dev_dax->nr_range == 0)
916 return false;
917 if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0)
918 return false;
919 last = &dev_dax->ranges[dev_dax->nr_range - 1];
920 if (last->range.start != res->start || last->range.end != res->end)
921 return false;
922 for (i = 0; i < dev_dax->nr_range - 1; i++) {
923 struct dev_dax_range *dax_range = &dev_dax->ranges[i];
924
925 if (dax_range->pgoff > last->pgoff)
926 return false;
927 }
928
929 return true;
930}
931
932static ssize_t dev_dax_resize(struct dax_region *dax_region,
933 struct dev_dax *dev_dax, resource_size_t size)
934{
935 resource_size_t avail = dax_region_avail_size(dax_region), to_alloc;
936 resource_size_t dev_size = dev_dax_size(dev_dax);
937 struct resource *region_res = &dax_region->res;
938 struct device *dev = &dev_dax->dev;
939 struct resource *res, *first;
940 resource_size_t alloc = 0;
941 int rc;
942
943 if (dev->driver)
944 return -EBUSY;
945 if (size == dev_size)
946 return 0;
947 if (size > dev_size && size - dev_size > avail)
948 return -ENOSPC;
949 if (size < dev_size)
950 return dev_dax_shrink(dev_dax, size);
951
952 to_alloc = size - dev_size;
953 if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc),
954 "resize of %pa misaligned\n", &to_alloc))
955 return -ENXIO;
956
957
958
959
960
961
962retry:
963 first = region_res->child;
964 if (!first)
965 return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc);
966
967 rc = -ENOSPC;
968 for (res = first; res; res = res->sibling) {
969 struct resource *next = res->sibling;
970
971
972 if (res == first && res->start > dax_region->res.start) {
973 alloc = min(res->start - dax_region->res.start, to_alloc);
974 rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc);
975 break;
976 }
977
978 alloc = 0;
979
980 if (next && next->start > res->end + 1)
981 alloc = min(next->start - (res->end + 1), to_alloc);
982
983
984 if (!alloc && !next && res->end < region_res->end)
985 alloc = min(region_res->end - res->end, to_alloc);
986
987 if (!alloc)
988 continue;
989
990 if (adjust_ok(dev_dax, res)) {
991 rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc);
992 break;
993 }
994 rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc);
995 break;
996 }
997 if (rc)
998 return rc;
999 to_alloc -= alloc;
1000 if (to_alloc)
1001 goto retry;
1002 return 0;
1003}
1004
1005static ssize_t size_store(struct device *dev, struct device_attribute *attr,
1006 const char *buf, size_t len)
1007{
1008 ssize_t rc;
1009 unsigned long long val;
1010 struct dev_dax *dev_dax = to_dev_dax(dev);
1011 struct dax_region *dax_region = dev_dax->region;
1012
1013 rc = kstrtoull(buf, 0, &val);
1014 if (rc)
1015 return rc;
1016
1017 if (!alloc_is_aligned(dev_dax, val)) {
1018 dev_dbg(dev, "%s: size: %lld misaligned\n", __func__, val);
1019 return -EINVAL;
1020 }
1021
1022 device_lock(dax_region->dev);
1023 if (!dax_region->dev->driver) {
1024 device_unlock(dax_region->dev);
1025 return -ENXIO;
1026 }
1027 device_lock(dev);
1028 rc = dev_dax_resize(dax_region, dev_dax, val);
1029 device_unlock(dev);
1030 device_unlock(dax_region->dev);
1031
1032 return rc == 0 ? len : rc;
1033}
1034static DEVICE_ATTR_RW(size);
1035
1036static ssize_t range_parse(const char *opt, size_t len, struct range *range)
1037{
1038 unsigned long long addr = 0;
1039 char *start, *end, *str;
1040 ssize_t rc = -EINVAL;
1041
1042 str = kstrdup(opt, GFP_KERNEL);
1043 if (!str)
1044 return rc;
1045
1046 end = str;
1047 start = strsep(&end, "-");
1048 if (!start || !end)
1049 goto err;
1050
1051 rc = kstrtoull(start, 16, &addr);
1052 if (rc)
1053 goto err;
1054 range->start = addr;
1055
1056 rc = kstrtoull(end, 16, &addr);
1057 if (rc)
1058 goto err;
1059 range->end = addr;
1060
1061err:
1062 kfree(str);
1063 return rc;
1064}
1065
1066static ssize_t mapping_store(struct device *dev, struct device_attribute *attr,
1067 const char *buf, size_t len)
1068{
1069 struct dev_dax *dev_dax = to_dev_dax(dev);
1070 struct dax_region *dax_region = dev_dax->region;
1071 size_t to_alloc;
1072 struct range r;
1073 ssize_t rc;
1074
1075 rc = range_parse(buf, len, &r);
1076 if (rc)
1077 return rc;
1078
1079 rc = -ENXIO;
1080 device_lock(dax_region->dev);
1081 if (!dax_region->dev->driver) {
1082 device_unlock(dax_region->dev);
1083 return rc;
1084 }
1085 device_lock(dev);
1086
1087 to_alloc = range_len(&r);
1088 if (alloc_is_aligned(dev_dax, to_alloc))
1089 rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc);
1090 device_unlock(dev);
1091 device_unlock(dax_region->dev);
1092
1093 return rc == 0 ? len : rc;
1094}
1095static DEVICE_ATTR_WO(mapping);
1096
1097static ssize_t align_show(struct device *dev,
1098 struct device_attribute *attr, char *buf)
1099{
1100 struct dev_dax *dev_dax = to_dev_dax(dev);
1101
1102 return sprintf(buf, "%d\n", dev_dax->align);
1103}
1104
1105static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax)
1106{
1107 struct device *dev = &dev_dax->dev;
1108 int i;
1109
1110 for (i = 0; i < dev_dax->nr_range; i++) {
1111 size_t len = range_len(&dev_dax->ranges[i].range);
1112
1113 if (!alloc_is_aligned(dev_dax, len)) {
1114 dev_dbg(dev, "%s: align %u invalid for range %d\n",
1115 __func__, dev_dax->align, i);
1116 return -EINVAL;
1117 }
1118 }
1119
1120 return 0;
1121}
1122
1123static ssize_t align_store(struct device *dev, struct device_attribute *attr,
1124 const char *buf, size_t len)
1125{
1126 struct dev_dax *dev_dax = to_dev_dax(dev);
1127 struct dax_region *dax_region = dev_dax->region;
1128 unsigned long val, align_save;
1129 ssize_t rc;
1130
1131 rc = kstrtoul(buf, 0, &val);
1132 if (rc)
1133 return -ENXIO;
1134
1135 if (!dax_align_valid(val))
1136 return -EINVAL;
1137
1138 device_lock(dax_region->dev);
1139 if (!dax_region->dev->driver) {
1140 device_unlock(dax_region->dev);
1141 return -ENXIO;
1142 }
1143
1144 device_lock(dev);
1145 if (dev->driver) {
1146 rc = -EBUSY;
1147 goto out_unlock;
1148 }
1149
1150 align_save = dev_dax->align;
1151 dev_dax->align = val;
1152 rc = dev_dax_validate_align(dev_dax);
1153 if (rc)
1154 dev_dax->align = align_save;
1155out_unlock:
1156 device_unlock(dev);
1157 device_unlock(dax_region->dev);
1158 return rc == 0 ? len : rc;
1159}
1160static DEVICE_ATTR_RW(align);
1161
1162static int dev_dax_target_node(struct dev_dax *dev_dax)
1163{
1164 struct dax_region *dax_region = dev_dax->region;
1165
1166 return dax_region->target_node;
1167}
1168
1169static ssize_t target_node_show(struct device *dev,
1170 struct device_attribute *attr, char *buf)
1171{
1172 struct dev_dax *dev_dax = to_dev_dax(dev);
1173
1174 return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax));
1175}
1176static DEVICE_ATTR_RO(target_node);
1177
1178static ssize_t resource_show(struct device *dev,
1179 struct device_attribute *attr, char *buf)
1180{
1181 struct dev_dax *dev_dax = to_dev_dax(dev);
1182 struct dax_region *dax_region = dev_dax->region;
1183 unsigned long long start;
1184
1185 if (dev_dax->nr_range < 1)
1186 start = dax_region->res.start;
1187 else
1188 start = dev_dax->ranges[0].range.start;
1189
1190 return sprintf(buf, "%#llx\n", start);
1191}
1192static DEVICE_ATTR(resource, 0400, resource_show, NULL);
1193
1194static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
1195 char *buf)
1196{
1197
1198
1199
1200
1201 return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
1202}
1203static DEVICE_ATTR_RO(modalias);
1204
1205static ssize_t numa_node_show(struct device *dev,
1206 struct device_attribute *attr, char *buf)
1207{
1208 return sprintf(buf, "%d\n", dev_to_node(dev));
1209}
1210static DEVICE_ATTR_RO(numa_node);
1211
1212static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
1213{
1214 struct device *dev = container_of(kobj, struct device, kobj);
1215 struct dev_dax *dev_dax = to_dev_dax(dev);
1216 struct dax_region *dax_region = dev_dax->region;
1217
1218 if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
1219 return 0;
1220 if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA))
1221 return 0;
1222 if (a == &dev_attr_mapping.attr && is_static(dax_region))
1223 return 0;
1224 if ((a == &dev_attr_align.attr ||
1225 a == &dev_attr_size.attr) && is_static(dax_region))
1226 return 0444;
1227 return a->mode;
1228}
1229
1230static struct attribute *dev_dax_attributes[] = {
1231 &dev_attr_modalias.attr,
1232 &dev_attr_size.attr,
1233 &dev_attr_mapping.attr,
1234 &dev_attr_target_node.attr,
1235 &dev_attr_align.attr,
1236 &dev_attr_resource.attr,
1237 &dev_attr_numa_node.attr,
1238 NULL,
1239};
1240
1241static const struct attribute_group dev_dax_attribute_group = {
1242 .attrs = dev_dax_attributes,
1243 .is_visible = dev_dax_visible,
1244};
1245
1246static const struct attribute_group *dax_attribute_groups[] = {
1247 &dev_dax_attribute_group,
1248 NULL,
1249};
1250
1251static void dev_dax_release(struct device *dev)
1252{
1253 struct dev_dax *dev_dax = to_dev_dax(dev);
1254 struct dax_region *dax_region = dev_dax->region;
1255 struct dax_device *dax_dev = dev_dax->dax_dev;
1256
1257 put_dax(dax_dev);
1258 free_dev_dax_id(dev_dax);
1259 dax_region_put(dax_region);
1260 kfree(dev_dax->pgmap);
1261 kfree(dev_dax);
1262}
1263
1264static const struct device_type dev_dax_type = {
1265 .release = dev_dax_release,
1266 .groups = dax_attribute_groups,
1267};
1268
1269struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
1270{
1271 struct dax_region *dax_region = data->dax_region;
1272 struct device *parent = dax_region->dev;
1273 struct dax_device *dax_dev;
1274 struct dev_dax *dev_dax;
1275 struct inode *inode;
1276 struct device *dev;
1277 int rc;
1278
1279 dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
1280 if (!dev_dax)
1281 return ERR_PTR(-ENOMEM);
1282
1283 if (is_static(dax_region)) {
1284 if (dev_WARN_ONCE(parent, data->id < 0,
1285 "dynamic id specified to static region\n")) {
1286 rc = -EINVAL;
1287 goto err_id;
1288 }
1289
1290 dev_dax->id = data->id;
1291 } else {
1292 if (dev_WARN_ONCE(parent, data->id >= 0,
1293 "static id specified to dynamic region\n")) {
1294 rc = -EINVAL;
1295 goto err_id;
1296 }
1297
1298 rc = ida_alloc(&dax_region->ida, GFP_KERNEL);
1299 if (rc < 0)
1300 goto err_id;
1301 dev_dax->id = rc;
1302 }
1303
1304 dev_dax->region = dax_region;
1305 dev = &dev_dax->dev;
1306 device_initialize(dev);
1307 dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
1308
1309 rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size);
1310 if (rc)
1311 goto err_range;
1312
1313 if (data->pgmap) {
1314 dev_WARN_ONCE(parent, !is_static(dax_region),
1315 "custom dev_pagemap requires a static dax_region\n");
1316
1317 dev_dax->pgmap = kmemdup(data->pgmap,
1318 sizeof(struct dev_pagemap), GFP_KERNEL);
1319 if (!dev_dax->pgmap) {
1320 rc = -ENOMEM;
1321 goto err_pgmap;
1322 }
1323 }
1324
1325
1326
1327
1328
1329 dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC);
1330 if (IS_ERR(dax_dev)) {
1331 rc = PTR_ERR(dax_dev);
1332 goto err_alloc_dax;
1333 }
1334
1335
1336 kill_dax(dax_dev);
1337
1338 dev_dax->dax_dev = dax_dev;
1339 dev_dax->target_node = dax_region->target_node;
1340 dev_dax->align = dax_region->align;
1341 ida_init(&dev_dax->ida);
1342 kref_get(&dax_region->kref);
1343
1344 inode = dax_inode(dax_dev);
1345 dev->devt = inode->i_rdev;
1346 if (data->subsys == DEV_DAX_BUS)
1347 dev->bus = &dax_bus_type;
1348 else
1349 dev->class = dax_class;
1350 dev->parent = parent;
1351 dev->type = &dev_dax_type;
1352
1353 rc = device_add(dev);
1354 if (rc) {
1355 kill_dev_dax(dev_dax);
1356 put_device(dev);
1357 return ERR_PTR(rc);
1358 }
1359
1360 rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
1361 if (rc)
1362 return ERR_PTR(rc);
1363
1364
1365 if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) {
1366 rc = devm_register_dax_mapping(dev_dax, 0);
1367 if (rc)
1368 return ERR_PTR(rc);
1369 }
1370
1371 return dev_dax;
1372
1373err_alloc_dax:
1374 kfree(dev_dax->pgmap);
1375err_pgmap:
1376 free_dev_dax_ranges(dev_dax);
1377err_range:
1378 free_dev_dax_id(dev_dax);
1379err_id:
1380 kfree(dev_dax);
1381
1382 return ERR_PTR(rc);
1383}
1384EXPORT_SYMBOL_GPL(devm_create_dev_dax);
1385
1386static int match_always_count;
1387
1388int __dax_driver_register(struct dax_device_driver *dax_drv,
1389 struct module *module, const char *mod_name)
1390{
1391 struct device_driver *drv = &dax_drv->drv;
1392 int rc = 0;
1393
1394
1395
1396
1397
1398 if (!dax_drv->probe)
1399 return -EINVAL;
1400
1401 INIT_LIST_HEAD(&dax_drv->ids);
1402 drv->owner = module;
1403 drv->name = mod_name;
1404 drv->mod_name = mod_name;
1405 drv->bus = &dax_bus_type;
1406
1407
1408 mutex_lock(&dax_bus_lock);
1409 match_always_count += dax_drv->match_always;
1410 if (match_always_count > 1) {
1411 match_always_count--;
1412 WARN_ON(1);
1413 rc = -EINVAL;
1414 }
1415 mutex_unlock(&dax_bus_lock);
1416 if (rc)
1417 return rc;
1418
1419 rc = driver_register(drv);
1420 if (rc && dax_drv->match_always) {
1421 mutex_lock(&dax_bus_lock);
1422 match_always_count -= dax_drv->match_always;
1423 mutex_unlock(&dax_bus_lock);
1424 }
1425
1426 return rc;
1427}
1428EXPORT_SYMBOL_GPL(__dax_driver_register);
1429
1430void dax_driver_unregister(struct dax_device_driver *dax_drv)
1431{
1432 struct device_driver *drv = &dax_drv->drv;
1433 struct dax_id *dax_id, *_id;
1434
1435 mutex_lock(&dax_bus_lock);
1436 match_always_count -= dax_drv->match_always;
1437 list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
1438 list_del(&dax_id->list);
1439 kfree(dax_id);
1440 }
1441 mutex_unlock(&dax_bus_lock);
1442 driver_unregister(drv);
1443}
1444EXPORT_SYMBOL_GPL(dax_driver_unregister);
1445
1446int __init dax_bus_init(void)
1447{
1448 int rc;
1449
1450 if (IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)) {
1451 dax_class = class_create(THIS_MODULE, "dax");
1452 if (IS_ERR(dax_class))
1453 return PTR_ERR(dax_class);
1454 }
1455
1456 rc = bus_register(&dax_bus_type);
1457 if (rc)
1458 class_destroy(dax_class);
1459 return rc;
1460}
1461
1462void __exit dax_bus_exit(void)
1463{
1464 bus_unregister(&dax_bus_type);
1465 class_destroy(dax_class);
1466}
1467