1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
32
33#include <linux/module.h>
34#include <linux/kernel.h>
35#include <linux/sched.h>
36#include <linux/magic.h>
37#include <linux/bitops.h>
38#include <linux/errno.h>
39#include <linux/highmem.h>
40#include <linux/string.h>
41#include <linux/slab.h>
42#include <asm/tlbflush.h>
43#include <asm/pgtable.h>
44#include <linux/cpumask.h>
45#include <linux/cpu.h>
46#include <linux/vmalloc.h>
47#include <linux/preempt.h>
48#include <linux/spinlock.h>
49#include <linux/shrinker.h>
50#include <linux/types.h>
51#include <linux/debugfs.h>
52#include <linux/zsmalloc.h>
53#include <linux/zpool.h>
54#include <linux/mount.h>
55#include <linux/pseudo_fs.h>
56#include <linux/migrate.h>
57#include <linux/wait.h>
58#include <linux/pagemap.h>
59#include <linux/fs.h>
60
61#define ZSPAGE_MAGIC 0x58
62
63
64
65
66
67
68
69#define ZS_ALIGN 8
70
71
72
73
74
75#define ZS_MAX_ZSPAGE_ORDER 2
76#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)
77
78#define ZS_HANDLE_SIZE (sizeof(unsigned long))
79
80
81
82
83
84
85
86
87
88
89#ifndef MAX_POSSIBLE_PHYSMEM_BITS
90#ifdef MAX_PHYSMEM_BITS
91#define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS
92#else
93
94
95
96
97#define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG
98#endif
99#endif
100
101#define _PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
102
103
104
105
106
107
108
109
110#define HANDLE_PIN_BIT 0
111
112
113
114
115
116
117
118
119#define OBJ_ALLOCATED_TAG 1
120#define OBJ_TAG_BITS 1
121#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS - OBJ_TAG_BITS)
122#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
123
124#define FULLNESS_BITS 2
125#define CLASS_BITS 8
126#define ISOLATED_BITS 3
127#define MAGIC_VAL_BITS 8
128
129#define MAX(a, b) ((a) >= (b) ? (a) : (b))
130
131#define ZS_MIN_ALLOC_SIZE \
132 MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
133
134#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149#define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> CLASS_BITS)
150#define ZS_SIZE_CLASSES (DIV_ROUND_UP(ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE, \
151 ZS_SIZE_CLASS_DELTA) + 1)
152
153enum fullness_group {
154 ZS_EMPTY,
155 ZS_ALMOST_EMPTY,
156 ZS_ALMOST_FULL,
157 ZS_FULL,
158 NR_ZS_FULLNESS,
159};
160
161enum zs_stat_type {
162 CLASS_EMPTY,
163 CLASS_ALMOST_EMPTY,
164 CLASS_ALMOST_FULL,
165 CLASS_FULL,
166 OBJ_ALLOCATED,
167 OBJ_USED,
168 NR_ZS_STAT_TYPE,
169};
170
171struct zs_size_stat {
172 unsigned long objs[NR_ZS_STAT_TYPE];
173};
174
175#ifdef CONFIG_ZSMALLOC_STAT
176static struct dentry *zs_stat_root;
177#endif
178
179#ifdef CONFIG_COMPACTION
180static struct vfsmount *zsmalloc_mnt;
181#endif
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197static const int fullness_threshold_frac = 4;
198static size_t huge_class_size;
199
200struct size_class {
201 spinlock_t lock;
202 struct list_head fullness_list[NR_ZS_FULLNESS];
203
204
205
206
207 int size;
208 int objs_per_zspage;
209
210 int pages_per_zspage;
211
212 unsigned int index;
213 struct zs_size_stat stats;
214};
215
216
217static void SetPageHugeObject(struct page *page)
218{
219 SetPageOwnerPriv1(page);
220}
221
222static void ClearPageHugeObject(struct page *page)
223{
224 ClearPageOwnerPriv1(page);
225}
226
227static int PageHugeObject(struct page *page)
228{
229 return PageOwnerPriv1(page);
230}
231
232
233
234
235
236
237
238struct link_free {
239 union {
240
241
242
243
244 unsigned long next;
245
246
247
248 unsigned long handle;
249 };
250};
251
252struct zs_pool {
253 const char *name;
254
255 struct size_class *size_class[ZS_SIZE_CLASSES];
256 struct kmem_cache *handle_cachep;
257 struct kmem_cache *zspage_cachep;
258
259 atomic_long_t pages_allocated;
260
261 struct zs_pool_stats stats;
262
263
264 struct shrinker shrinker;
265
266#ifdef CONFIG_ZSMALLOC_STAT
267 struct dentry *stat_dentry;
268#endif
269#ifdef CONFIG_COMPACTION
270 struct inode *inode;
271 struct work_struct free_work;
272
273 struct wait_queue_head migration_wait;
274 atomic_long_t isolated_pages;
275 bool destroying;
276#endif
277};
278
279struct zspage {
280 struct {
281 unsigned int fullness:FULLNESS_BITS;
282 unsigned int class:CLASS_BITS + 1;
283 unsigned int isolated:ISOLATED_BITS;
284 unsigned int magic:MAGIC_VAL_BITS;
285 };
286 unsigned int inuse;
287 unsigned int freeobj;
288 struct page *first_page;
289 struct list_head list;
290#ifdef CONFIG_COMPACTION
291 rwlock_t lock;
292#endif
293};
294
295struct mapping_area {
296#ifdef CONFIG_PGTABLE_MAPPING
297 struct vm_struct *vm;
298#else
299 char *vm_buf;
300#endif
301 char *vm_addr;
302 enum zs_mapmode vm_mm;
303};
304
305#ifdef CONFIG_COMPACTION
306static int zs_register_migration(struct zs_pool *pool);
307static void zs_unregister_migration(struct zs_pool *pool);
308static void migrate_lock_init(struct zspage *zspage);
309static void migrate_read_lock(struct zspage *zspage);
310static void migrate_read_unlock(struct zspage *zspage);
311static void kick_deferred_free(struct zs_pool *pool);
312static void init_deferred_free(struct zs_pool *pool);
313static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
314#else
315static int zsmalloc_mount(void) { return 0; }
316static void zsmalloc_unmount(void) {}
317static int zs_register_migration(struct zs_pool *pool) { return 0; }
318static void zs_unregister_migration(struct zs_pool *pool) {}
319static void migrate_lock_init(struct zspage *zspage) {}
320static void migrate_read_lock(struct zspage *zspage) {}
321static void migrate_read_unlock(struct zspage *zspage) {}
322static void kick_deferred_free(struct zs_pool *pool) {}
323static void init_deferred_free(struct zs_pool *pool) {}
324static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}
325#endif
326
327static int create_cache(struct zs_pool *pool)
328{
329 pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE,
330 0, 0, NULL);
331 if (!pool->handle_cachep)
332 return 1;
333
334 pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),
335 0, 0, NULL);
336 if (!pool->zspage_cachep) {
337 kmem_cache_destroy(pool->handle_cachep);
338 pool->handle_cachep = NULL;
339 return 1;
340 }
341
342 return 0;
343}
344
345static void destroy_cache(struct zs_pool *pool)
346{
347 kmem_cache_destroy(pool->handle_cachep);
348 kmem_cache_destroy(pool->zspage_cachep);
349}
350
351static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp)
352{
353 return (unsigned long)kmem_cache_alloc(pool->handle_cachep,
354 gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
355}
356
357static void cache_free_handle(struct zs_pool *pool, unsigned long handle)
358{
359 kmem_cache_free(pool->handle_cachep, (void *)handle);
360}
361
362static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)
363{
364 return kmem_cache_alloc(pool->zspage_cachep,
365 flags & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
366}
367
368static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
369{
370 kmem_cache_free(pool->zspage_cachep, zspage);
371}
372
373static void record_obj(unsigned long handle, unsigned long obj)
374{
375
376
377
378
379
380 WRITE_ONCE(*(unsigned long *)handle, obj);
381}
382
383
384
385#ifdef CONFIG_ZPOOL
386
387static void *zs_zpool_create(const char *name, gfp_t gfp,
388 const struct zpool_ops *zpool_ops,
389 struct zpool *zpool)
390{
391
392
393
394
395
396 return zs_create_pool(name);
397}
398
399static void zs_zpool_destroy(void *pool)
400{
401 zs_destroy_pool(pool);
402}
403
404static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp,
405 unsigned long *handle)
406{
407 *handle = zs_malloc(pool, size, gfp);
408 return *handle ? 0 : -1;
409}
410static void zs_zpool_free(void *pool, unsigned long handle)
411{
412 zs_free(pool, handle);
413}
414
415static void *zs_zpool_map(void *pool, unsigned long handle,
416 enum zpool_mapmode mm)
417{
418 enum zs_mapmode zs_mm;
419
420 switch (mm) {
421 case ZPOOL_MM_RO:
422 zs_mm = ZS_MM_RO;
423 break;
424 case ZPOOL_MM_WO:
425 zs_mm = ZS_MM_WO;
426 break;
427 case ZPOOL_MM_RW:
428 default:
429 zs_mm = ZS_MM_RW;
430 break;
431 }
432
433 return zs_map_object(pool, handle, zs_mm);
434}
435static void zs_zpool_unmap(void *pool, unsigned long handle)
436{
437 zs_unmap_object(pool, handle);
438}
439
440static u64 zs_zpool_total_size(void *pool)
441{
442 return zs_get_total_pages(pool) << PAGE_SHIFT;
443}
444
445static struct zpool_driver zs_zpool_driver = {
446 .type = "zsmalloc",
447 .owner = THIS_MODULE,
448 .create = zs_zpool_create,
449 .destroy = zs_zpool_destroy,
450 .malloc_support_movable = true,
451 .malloc = zs_zpool_malloc,
452 .free = zs_zpool_free,
453 .map = zs_zpool_map,
454 .unmap = zs_zpool_unmap,
455 .total_size = zs_zpool_total_size,
456};
457
458MODULE_ALIAS("zpool-zsmalloc");
459#endif
460
461
462static DEFINE_PER_CPU(struct mapping_area, zs_map_area);
463
464static bool is_zspage_isolated(struct zspage *zspage)
465{
466 return zspage->isolated;
467}
468
469static __maybe_unused int is_first_page(struct page *page)
470{
471 return PagePrivate(page);
472}
473
474
475static inline int get_zspage_inuse(struct zspage *zspage)
476{
477 return zspage->inuse;
478}
479
480
481static inline void mod_zspage_inuse(struct zspage *zspage, int val)
482{
483 zspage->inuse += val;
484}
485
486static inline struct page *get_first_page(struct zspage *zspage)
487{
488 struct page *first_page = zspage->first_page;
489
490 VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
491 return first_page;
492}
493
494static inline int get_first_obj_offset(struct page *page)
495{
496 return page->units;
497}
498
499static inline void set_first_obj_offset(struct page *page, int offset)
500{
501 page->units = offset;
502}
503
504static inline unsigned int get_freeobj(struct zspage *zspage)
505{
506 return zspage->freeobj;
507}
508
509static inline void set_freeobj(struct zspage *zspage, unsigned int obj)
510{
511 zspage->freeobj = obj;
512}
513
514static void get_zspage_mapping(struct zspage *zspage,
515 unsigned int *class_idx,
516 enum fullness_group *fullness)
517{
518 BUG_ON(zspage->magic != ZSPAGE_MAGIC);
519
520 *fullness = zspage->fullness;
521 *class_idx = zspage->class;
522}
523
524static void set_zspage_mapping(struct zspage *zspage,
525 unsigned int class_idx,
526 enum fullness_group fullness)
527{
528 zspage->class = class_idx;
529 zspage->fullness = fullness;
530}
531
532
533
534
535
536
537
538
539static int get_size_class_index(int size)
540{
541 int idx = 0;
542
543 if (likely(size > ZS_MIN_ALLOC_SIZE))
544 idx = DIV_ROUND_UP(size - ZS_MIN_ALLOC_SIZE,
545 ZS_SIZE_CLASS_DELTA);
546
547 return min_t(int, ZS_SIZE_CLASSES - 1, idx);
548}
549
550
551static inline void zs_stat_inc(struct size_class *class,
552 int type, unsigned long cnt)
553{
554 class->stats.objs[type] += cnt;
555}
556
557
558static inline void zs_stat_dec(struct size_class *class,
559 int type, unsigned long cnt)
560{
561 class->stats.objs[type] -= cnt;
562}
563
564
565static inline unsigned long zs_stat_get(struct size_class *class,
566 int type)
567{
568 return class->stats.objs[type];
569}
570
571#ifdef CONFIG_ZSMALLOC_STAT
572
573static void __init zs_stat_init(void)
574{
575 if (!debugfs_initialized()) {
576 pr_warn("debugfs not available, stat dir not created\n");
577 return;
578 }
579
580 zs_stat_root = debugfs_create_dir("zsmalloc", NULL);
581}
582
583static void __exit zs_stat_exit(void)
584{
585 debugfs_remove_recursive(zs_stat_root);
586}
587
588static unsigned long zs_can_compact(struct size_class *class);
589
590static int zs_stats_size_show(struct seq_file *s, void *v)
591{
592 int i;
593 struct zs_pool *pool = s->private;
594 struct size_class *class;
595 int objs_per_zspage;
596 unsigned long class_almost_full, class_almost_empty;
597 unsigned long obj_allocated, obj_used, pages_used, freeable;
598 unsigned long total_class_almost_full = 0, total_class_almost_empty = 0;
599 unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0;
600 unsigned long total_freeable = 0;
601
602 seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n",
603 "class", "size", "almost_full", "almost_empty",
604 "obj_allocated", "obj_used", "pages_used",
605 "pages_per_zspage", "freeable");
606
607 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
608 class = pool->size_class[i];
609
610 if (class->index != i)
611 continue;
612
613 spin_lock(&class->lock);
614 class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL);
615 class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
616 obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
617 obj_used = zs_stat_get(class, OBJ_USED);
618 freeable = zs_can_compact(class);
619 spin_unlock(&class->lock);
620
621 objs_per_zspage = class->objs_per_zspage;
622 pages_used = obj_allocated / objs_per_zspage *
623 class->pages_per_zspage;
624
625 seq_printf(s, " %5u %5u %11lu %12lu %13lu"
626 " %10lu %10lu %16d %8lu\n",
627 i, class->size, class_almost_full, class_almost_empty,
628 obj_allocated, obj_used, pages_used,
629 class->pages_per_zspage, freeable);
630
631 total_class_almost_full += class_almost_full;
632 total_class_almost_empty += class_almost_empty;
633 total_objs += obj_allocated;
634 total_used_objs += obj_used;
635 total_pages += pages_used;
636 total_freeable += freeable;
637 }
638
639 seq_puts(s, "\n");
640 seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n",
641 "Total", "", total_class_almost_full,
642 total_class_almost_empty, total_objs,
643 total_used_objs, total_pages, "", total_freeable);
644
645 return 0;
646}
647DEFINE_SHOW_ATTRIBUTE(zs_stats_size);
648
649static void zs_pool_stat_create(struct zs_pool *pool, const char *name)
650{
651 if (!zs_stat_root) {
652 pr_warn("no root stat dir, not creating <%s> stat dir\n", name);
653 return;
654 }
655
656 pool->stat_dentry = debugfs_create_dir(name, zs_stat_root);
657
658 debugfs_create_file("classes", S_IFREG | 0444, pool->stat_dentry, pool,
659 &zs_stats_size_fops);
660}
661
662static void zs_pool_stat_destroy(struct zs_pool *pool)
663{
664 debugfs_remove_recursive(pool->stat_dentry);
665}
666
667#else
668static void __init zs_stat_init(void)
669{
670}
671
672static void __exit zs_stat_exit(void)
673{
674}
675
676static inline void zs_pool_stat_create(struct zs_pool *pool, const char *name)
677{
678}
679
680static inline void zs_pool_stat_destroy(struct zs_pool *pool)
681{
682}
683#endif
684
685
686
687
688
689
690
691
692
693static enum fullness_group get_fullness_group(struct size_class *class,
694 struct zspage *zspage)
695{
696 int inuse, objs_per_zspage;
697 enum fullness_group fg;
698
699 inuse = get_zspage_inuse(zspage);
700 objs_per_zspage = class->objs_per_zspage;
701
702 if (inuse == 0)
703 fg = ZS_EMPTY;
704 else if (inuse == objs_per_zspage)
705 fg = ZS_FULL;
706 else if (inuse <= 3 * objs_per_zspage / fullness_threshold_frac)
707 fg = ZS_ALMOST_EMPTY;
708 else
709 fg = ZS_ALMOST_FULL;
710
711 return fg;
712}
713
714
715
716
717
718
719
720static void insert_zspage(struct size_class *class,
721 struct zspage *zspage,
722 enum fullness_group fullness)
723{
724 struct zspage *head;
725
726 zs_stat_inc(class, fullness, 1);
727 head = list_first_entry_or_null(&class->fullness_list[fullness],
728 struct zspage, list);
729
730
731
732
733 if (head) {
734 if (get_zspage_inuse(zspage) < get_zspage_inuse(head)) {
735 list_add(&zspage->list, &head->list);
736 return;
737 }
738 }
739 list_add(&zspage->list, &class->fullness_list[fullness]);
740}
741
742
743
744
745
746static void remove_zspage(struct size_class *class,
747 struct zspage *zspage,
748 enum fullness_group fullness)
749{
750 VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
751 VM_BUG_ON(is_zspage_isolated(zspage));
752
753 list_del_init(&zspage->list);
754 zs_stat_dec(class, fullness, 1);
755}
756
757
758
759
760
761
762
763
764
765
766static enum fullness_group fix_fullness_group(struct size_class *class,
767 struct zspage *zspage)
768{
769 int class_idx;
770 enum fullness_group currfg, newfg;
771
772 get_zspage_mapping(zspage, &class_idx, &currfg);
773 newfg = get_fullness_group(class, zspage);
774 if (newfg == currfg)
775 goto out;
776
777 if (!is_zspage_isolated(zspage)) {
778 remove_zspage(class, zspage, currfg);
779 insert_zspage(class, zspage, newfg);
780 }
781
782 set_zspage_mapping(zspage, class_idx, newfg);
783
784out:
785 return newfg;
786}
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801static int get_pages_per_zspage(int class_size)
802{
803 int i, max_usedpc = 0;
804
805 int max_usedpc_order = 1;
806
807 for (i = 1; i <= ZS_MAX_PAGES_PER_ZSPAGE; i++) {
808 int zspage_size;
809 int waste, usedpc;
810
811 zspage_size = i * PAGE_SIZE;
812 waste = zspage_size % class_size;
813 usedpc = (zspage_size - waste) * 100 / zspage_size;
814
815 if (usedpc > max_usedpc) {
816 max_usedpc = usedpc;
817 max_usedpc_order = i;
818 }
819 }
820
821 return max_usedpc_order;
822}
823
824static struct zspage *get_zspage(struct page *page)
825{
826 struct zspage *zspage = (struct zspage *)page->private;
827
828 BUG_ON(zspage->magic != ZSPAGE_MAGIC);
829 return zspage;
830}
831
832static struct page *get_next_page(struct page *page)
833{
834 if (unlikely(PageHugeObject(page)))
835 return NULL;
836
837 return page->freelist;
838}
839
840
841
842
843
844
845
846static void obj_to_location(unsigned long obj, struct page **page,
847 unsigned int *obj_idx)
848{
849 obj >>= OBJ_TAG_BITS;
850 *page = pfn_to_page(obj >> OBJ_INDEX_BITS);
851 *obj_idx = (obj & OBJ_INDEX_MASK);
852}
853
854
855
856
857
858
859static unsigned long location_to_obj(struct page *page, unsigned int obj_idx)
860{
861 unsigned long obj;
862
863 obj = page_to_pfn(page) << OBJ_INDEX_BITS;
864 obj |= obj_idx & OBJ_INDEX_MASK;
865 obj <<= OBJ_TAG_BITS;
866
867 return obj;
868}
869
870static unsigned long handle_to_obj(unsigned long handle)
871{
872 return *(unsigned long *)handle;
873}
874
875static unsigned long obj_to_head(struct page *page, void *obj)
876{
877 if (unlikely(PageHugeObject(page))) {
878 VM_BUG_ON_PAGE(!is_first_page(page), page);
879 return page->index;
880 } else
881 return *(unsigned long *)obj;
882}
883
884static inline int testpin_tag(unsigned long handle)
885{
886 return bit_spin_is_locked(HANDLE_PIN_BIT, (unsigned long *)handle);
887}
888
889static inline int trypin_tag(unsigned long handle)
890{
891 return bit_spin_trylock(HANDLE_PIN_BIT, (unsigned long *)handle);
892}
893
894static void pin_tag(unsigned long handle)
895{
896 bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle);
897}
898
899static void unpin_tag(unsigned long handle)
900{
901 bit_spin_unlock(HANDLE_PIN_BIT, (unsigned long *)handle);
902}
903
904static void reset_page(struct page *page)
905{
906 __ClearPageMovable(page);
907 ClearPagePrivate(page);
908 set_page_private(page, 0);
909 page_mapcount_reset(page);
910 ClearPageHugeObject(page);
911 page->freelist = NULL;
912}
913
914static int trylock_zspage(struct zspage *zspage)
915{
916 struct page *cursor, *fail;
917
918 for (cursor = get_first_page(zspage); cursor != NULL; cursor =
919 get_next_page(cursor)) {
920 if (!trylock_page(cursor)) {
921 fail = cursor;
922 goto unlock;
923 }
924 }
925
926 return 1;
927unlock:
928 for (cursor = get_first_page(zspage); cursor != fail; cursor =
929 get_next_page(cursor))
930 unlock_page(cursor);
931
932 return 0;
933}
934
935static void __free_zspage(struct zs_pool *pool, struct size_class *class,
936 struct zspage *zspage)
937{
938 struct page *page, *next;
939 enum fullness_group fg;
940 unsigned int class_idx;
941
942 get_zspage_mapping(zspage, &class_idx, &fg);
943
944 assert_spin_locked(&class->lock);
945
946 VM_BUG_ON(get_zspage_inuse(zspage));
947 VM_BUG_ON(fg != ZS_EMPTY);
948
949 next = page = get_first_page(zspage);
950 do {
951 VM_BUG_ON_PAGE(!PageLocked(page), page);
952 next = get_next_page(page);
953 reset_page(page);
954 unlock_page(page);
955 dec_zone_page_state(page, NR_ZSPAGES);
956 put_page(page);
957 page = next;
958 } while (page != NULL);
959
960 cache_free_zspage(pool, zspage);
961
962 zs_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage);
963 atomic_long_sub(class->pages_per_zspage,
964 &pool->pages_allocated);
965}
966
967static void free_zspage(struct zs_pool *pool, struct size_class *class,
968 struct zspage *zspage)
969{
970 VM_BUG_ON(get_zspage_inuse(zspage));
971 VM_BUG_ON(list_empty(&zspage->list));
972
973 if (!trylock_zspage(zspage)) {
974 kick_deferred_free(pool);
975 return;
976 }
977
978 remove_zspage(class, zspage, ZS_EMPTY);
979 __free_zspage(pool, class, zspage);
980}
981
982
983static void init_zspage(struct size_class *class, struct zspage *zspage)
984{
985 unsigned int freeobj = 1;
986 unsigned long off = 0;
987 struct page *page = get_first_page(zspage);
988
989 while (page) {
990 struct page *next_page;
991 struct link_free *link;
992 void *vaddr;
993
994 set_first_obj_offset(page, off);
995
996 vaddr = kmap_atomic(page);
997 link = (struct link_free *)vaddr + off / sizeof(*link);
998
999 while ((off += class->size) < PAGE_SIZE) {
1000 link->next = freeobj++ << OBJ_TAG_BITS;
1001 link += class->size / sizeof(*link);
1002 }
1003
1004
1005
1006
1007
1008
1009 next_page = get_next_page(page);
1010 if (next_page) {
1011 link->next = freeobj++ << OBJ_TAG_BITS;
1012 } else {
1013
1014
1015
1016
1017 link->next = -1UL << OBJ_TAG_BITS;
1018 }
1019 kunmap_atomic(vaddr);
1020 page = next_page;
1021 off %= PAGE_SIZE;
1022 }
1023
1024 set_freeobj(zspage, 0);
1025}
1026
1027static void create_page_chain(struct size_class *class, struct zspage *zspage,
1028 struct page *pages[])
1029{
1030 int i;
1031 struct page *page;
1032 struct page *prev_page = NULL;
1033 int nr_pages = class->pages_per_zspage;
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043 for (i = 0; i < nr_pages; i++) {
1044 page = pages[i];
1045 set_page_private(page, (unsigned long)zspage);
1046 page->freelist = NULL;
1047 if (i == 0) {
1048 zspage->first_page = page;
1049 SetPagePrivate(page);
1050 if (unlikely(class->objs_per_zspage == 1 &&
1051 class->pages_per_zspage == 1))
1052 SetPageHugeObject(page);
1053 } else {
1054 prev_page->freelist = page;
1055 }
1056 prev_page = page;
1057 }
1058}
1059
1060
1061
1062
1063static struct zspage *alloc_zspage(struct zs_pool *pool,
1064 struct size_class *class,
1065 gfp_t gfp)
1066{
1067 int i;
1068 struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE];
1069 struct zspage *zspage = cache_alloc_zspage(pool, gfp);
1070
1071 if (!zspage)
1072 return NULL;
1073
1074 memset(zspage, 0, sizeof(struct zspage));
1075 zspage->magic = ZSPAGE_MAGIC;
1076 migrate_lock_init(zspage);
1077
1078 for (i = 0; i < class->pages_per_zspage; i++) {
1079 struct page *page;
1080
1081 page = alloc_page(gfp);
1082 if (!page) {
1083 while (--i >= 0) {
1084 dec_zone_page_state(pages[i], NR_ZSPAGES);
1085 __free_page(pages[i]);
1086 }
1087 cache_free_zspage(pool, zspage);
1088 return NULL;
1089 }
1090
1091 inc_zone_page_state(page, NR_ZSPAGES);
1092 pages[i] = page;
1093 }
1094
1095 create_page_chain(class, zspage, pages);
1096 init_zspage(class, zspage);
1097
1098 return zspage;
1099}
1100
1101static struct zspage *find_get_zspage(struct size_class *class)
1102{
1103 int i;
1104 struct zspage *zspage;
1105
1106 for (i = ZS_ALMOST_FULL; i >= ZS_EMPTY; i--) {
1107 zspage = list_first_entry_or_null(&class->fullness_list[i],
1108 struct zspage, list);
1109 if (zspage)
1110 break;
1111 }
1112
1113 return zspage;
1114}
1115
1116#ifdef CONFIG_PGTABLE_MAPPING
1117static inline int __zs_cpu_up(struct mapping_area *area)
1118{
1119
1120
1121
1122
1123 if (area->vm)
1124 return 0;
1125 area->vm = alloc_vm_area(PAGE_SIZE * 2, NULL);
1126 if (!area->vm)
1127 return -ENOMEM;
1128 return 0;
1129}
1130
1131static inline void __zs_cpu_down(struct mapping_area *area)
1132{
1133 if (area->vm)
1134 free_vm_area(area->vm);
1135 area->vm = NULL;
1136}
1137
1138static inline void *__zs_map_object(struct mapping_area *area,
1139 struct page *pages[2], int off, int size)
1140{
1141 BUG_ON(map_vm_area(area->vm, PAGE_KERNEL, pages));
1142 area->vm_addr = area->vm->addr;
1143 return area->vm_addr + off;
1144}
1145
1146static inline void __zs_unmap_object(struct mapping_area *area,
1147 struct page *pages[2], int off, int size)
1148{
1149 unsigned long addr = (unsigned long)area->vm_addr;
1150
1151 unmap_kernel_range(addr, PAGE_SIZE * 2);
1152}
1153
1154#else
1155
1156static inline int __zs_cpu_up(struct mapping_area *area)
1157{
1158
1159
1160
1161
1162 if (area->vm_buf)
1163 return 0;
1164 area->vm_buf = kmalloc(ZS_MAX_ALLOC_SIZE, GFP_KERNEL);
1165 if (!area->vm_buf)
1166 return -ENOMEM;
1167 return 0;
1168}
1169
1170static inline void __zs_cpu_down(struct mapping_area *area)
1171{
1172 kfree(area->vm_buf);
1173 area->vm_buf = NULL;
1174}
1175
1176static void *__zs_map_object(struct mapping_area *area,
1177 struct page *pages[2], int off, int size)
1178{
1179 int sizes[2];
1180 void *addr;
1181 char *buf = area->vm_buf;
1182
1183
1184 pagefault_disable();
1185
1186
1187 if (area->vm_mm == ZS_MM_WO)
1188 goto out;
1189
1190 sizes[0] = PAGE_SIZE - off;
1191 sizes[1] = size - sizes[0];
1192
1193
1194 addr = kmap_atomic(pages[0]);
1195 memcpy(buf, addr + off, sizes[0]);
1196 kunmap_atomic(addr);
1197 addr = kmap_atomic(pages[1]);
1198 memcpy(buf + sizes[0], addr, sizes[1]);
1199 kunmap_atomic(addr);
1200out:
1201 return area->vm_buf;
1202}
1203
1204static void __zs_unmap_object(struct mapping_area *area,
1205 struct page *pages[2], int off, int size)
1206{
1207 int sizes[2];
1208 void *addr;
1209 char *buf;
1210
1211
1212 if (area->vm_mm == ZS_MM_RO)
1213 goto out;
1214
1215 buf = area->vm_buf;
1216 buf = buf + ZS_HANDLE_SIZE;
1217 size -= ZS_HANDLE_SIZE;
1218 off += ZS_HANDLE_SIZE;
1219
1220 sizes[0] = PAGE_SIZE - off;
1221 sizes[1] = size - sizes[0];
1222
1223
1224 addr = kmap_atomic(pages[0]);
1225 memcpy(addr + off, buf, sizes[0]);
1226 kunmap_atomic(addr);
1227 addr = kmap_atomic(pages[1]);
1228 memcpy(addr, buf + sizes[0], sizes[1]);
1229 kunmap_atomic(addr);
1230
1231out:
1232
1233 pagefault_enable();
1234}
1235
1236#endif
1237
1238static int zs_cpu_prepare(unsigned int cpu)
1239{
1240 struct mapping_area *area;
1241
1242 area = &per_cpu(zs_map_area, cpu);
1243 return __zs_cpu_up(area);
1244}
1245
1246static int zs_cpu_dead(unsigned int cpu)
1247{
1248 struct mapping_area *area;
1249
1250 area = &per_cpu(zs_map_area, cpu);
1251 __zs_cpu_down(area);
1252 return 0;
1253}
1254
1255static bool can_merge(struct size_class *prev, int pages_per_zspage,
1256 int objs_per_zspage)
1257{
1258 if (prev->pages_per_zspage == pages_per_zspage &&
1259 prev->objs_per_zspage == objs_per_zspage)
1260 return true;
1261
1262 return false;
1263}
1264
1265static bool zspage_full(struct size_class *class, struct zspage *zspage)
1266{
1267 return get_zspage_inuse(zspage) == class->objs_per_zspage;
1268}
1269
1270unsigned long zs_get_total_pages(struct zs_pool *pool)
1271{
1272 return atomic_long_read(&pool->pages_allocated);
1273}
1274EXPORT_SYMBOL_GPL(zs_get_total_pages);
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291void *zs_map_object(struct zs_pool *pool, unsigned long handle,
1292 enum zs_mapmode mm)
1293{
1294 struct zspage *zspage;
1295 struct page *page;
1296 unsigned long obj, off;
1297 unsigned int obj_idx;
1298
1299 unsigned int class_idx;
1300 enum fullness_group fg;
1301 struct size_class *class;
1302 struct mapping_area *area;
1303 struct page *pages[2];
1304 void *ret;
1305
1306
1307
1308
1309
1310
1311 BUG_ON(in_interrupt());
1312
1313
1314 pin_tag(handle);
1315
1316 obj = handle_to_obj(handle);
1317 obj_to_location(obj, &page, &obj_idx);
1318 zspage = get_zspage(page);
1319
1320
1321 migrate_read_lock(zspage);
1322
1323 get_zspage_mapping(zspage, &class_idx, &fg);
1324 class = pool->size_class[class_idx];
1325 off = (class->size * obj_idx) & ~PAGE_MASK;
1326
1327 area = &get_cpu_var(zs_map_area);
1328 area->vm_mm = mm;
1329 if (off + class->size <= PAGE_SIZE) {
1330
1331 area->vm_addr = kmap_atomic(page);
1332 ret = area->vm_addr + off;
1333 goto out;
1334 }
1335
1336
1337 pages[0] = page;
1338 pages[1] = get_next_page(page);
1339 BUG_ON(!pages[1]);
1340
1341 ret = __zs_map_object(area, pages, off, class->size);
1342out:
1343 if (likely(!PageHugeObject(page)))
1344 ret += ZS_HANDLE_SIZE;
1345
1346 return ret;
1347}
1348EXPORT_SYMBOL_GPL(zs_map_object);
1349
1350void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
1351{
1352 struct zspage *zspage;
1353 struct page *page;
1354 unsigned long obj, off;
1355 unsigned int obj_idx;
1356
1357 unsigned int class_idx;
1358 enum fullness_group fg;
1359 struct size_class *class;
1360 struct mapping_area *area;
1361
1362 obj = handle_to_obj(handle);
1363 obj_to_location(obj, &page, &obj_idx);
1364 zspage = get_zspage(page);
1365 get_zspage_mapping(zspage, &class_idx, &fg);
1366 class = pool->size_class[class_idx];
1367 off = (class->size * obj_idx) & ~PAGE_MASK;
1368
1369 area = this_cpu_ptr(&zs_map_area);
1370 if (off + class->size <= PAGE_SIZE)
1371 kunmap_atomic(area->vm_addr);
1372 else {
1373 struct page *pages[2];
1374
1375 pages[0] = page;
1376 pages[1] = get_next_page(page);
1377 BUG_ON(!pages[1]);
1378
1379 __zs_unmap_object(area, pages, off, class->size);
1380 }
1381 put_cpu_var(zs_map_area);
1382
1383 migrate_read_unlock(zspage);
1384 unpin_tag(handle);
1385}
1386EXPORT_SYMBOL_GPL(zs_unmap_object);
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401size_t zs_huge_class_size(struct zs_pool *pool)
1402{
1403 return huge_class_size;
1404}
1405EXPORT_SYMBOL_GPL(zs_huge_class_size);
1406
1407static unsigned long obj_malloc(struct size_class *class,
1408 struct zspage *zspage, unsigned long handle)
1409{
1410 int i, nr_page, offset;
1411 unsigned long obj;
1412 struct link_free *link;
1413
1414 struct page *m_page;
1415 unsigned long m_offset;
1416 void *vaddr;
1417
1418 handle |= OBJ_ALLOCATED_TAG;
1419 obj = get_freeobj(zspage);
1420
1421 offset = obj * class->size;
1422 nr_page = offset >> PAGE_SHIFT;
1423 m_offset = offset & ~PAGE_MASK;
1424 m_page = get_first_page(zspage);
1425
1426 for (i = 0; i < nr_page; i++)
1427 m_page = get_next_page(m_page);
1428
1429 vaddr = kmap_atomic(m_page);
1430 link = (struct link_free *)vaddr + m_offset / sizeof(*link);
1431 set_freeobj(zspage, link->next >> OBJ_TAG_BITS);
1432 if (likely(!PageHugeObject(m_page)))
1433
1434 link->handle = handle;
1435 else
1436
1437 zspage->first_page->index = handle;
1438
1439 kunmap_atomic(vaddr);
1440 mod_zspage_inuse(zspage, 1);
1441 zs_stat_inc(class, OBJ_USED, 1);
1442
1443 obj = location_to_obj(m_page, obj);
1444
1445 return obj;
1446}
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
1460{
1461 unsigned long handle, obj;
1462 struct size_class *class;
1463 enum fullness_group newfg;
1464 struct zspage *zspage;
1465
1466 if (unlikely(!size || size > ZS_MAX_ALLOC_SIZE))
1467 return 0;
1468
1469 handle = cache_alloc_handle(pool, gfp);
1470 if (!handle)
1471 return 0;
1472
1473
1474 size += ZS_HANDLE_SIZE;
1475 class = pool->size_class[get_size_class_index(size)];
1476
1477 spin_lock(&class->lock);
1478 zspage = find_get_zspage(class);
1479 if (likely(zspage)) {
1480 obj = obj_malloc(class, zspage, handle);
1481
1482 fix_fullness_group(class, zspage);
1483 record_obj(handle, obj);
1484 spin_unlock(&class->lock);
1485
1486 return handle;
1487 }
1488
1489 spin_unlock(&class->lock);
1490
1491 zspage = alloc_zspage(pool, class, gfp);
1492 if (!zspage) {
1493 cache_free_handle(pool, handle);
1494 return 0;
1495 }
1496
1497 spin_lock(&class->lock);
1498 obj = obj_malloc(class, zspage, handle);
1499 newfg = get_fullness_group(class, zspage);
1500 insert_zspage(class, zspage, newfg);
1501 set_zspage_mapping(zspage, class->index, newfg);
1502 record_obj(handle, obj);
1503 atomic_long_add(class->pages_per_zspage,
1504 &pool->pages_allocated);
1505 zs_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage);
1506
1507
1508 SetZsPageMovable(pool, zspage);
1509 spin_unlock(&class->lock);
1510
1511 return handle;
1512}
1513EXPORT_SYMBOL_GPL(zs_malloc);
1514
1515static void obj_free(struct size_class *class, unsigned long obj)
1516{
1517 struct link_free *link;
1518 struct zspage *zspage;
1519 struct page *f_page;
1520 unsigned long f_offset;
1521 unsigned int f_objidx;
1522 void *vaddr;
1523
1524 obj &= ~OBJ_ALLOCATED_TAG;
1525 obj_to_location(obj, &f_page, &f_objidx);
1526 f_offset = (class->size * f_objidx) & ~PAGE_MASK;
1527 zspage = get_zspage(f_page);
1528
1529 vaddr = kmap_atomic(f_page);
1530
1531
1532 link = (struct link_free *)(vaddr + f_offset);
1533 link->next = get_freeobj(zspage) << OBJ_TAG_BITS;
1534 kunmap_atomic(vaddr);
1535 set_freeobj(zspage, f_objidx);
1536 mod_zspage_inuse(zspage, -1);
1537 zs_stat_dec(class, OBJ_USED, 1);
1538}
1539
1540void zs_free(struct zs_pool *pool, unsigned long handle)
1541{
1542 struct zspage *zspage;
1543 struct page *f_page;
1544 unsigned long obj;
1545 unsigned int f_objidx;
1546 int class_idx;
1547 struct size_class *class;
1548 enum fullness_group fullness;
1549 bool isolated;
1550
1551 if (unlikely(!handle))
1552 return;
1553
1554 pin_tag(handle);
1555 obj = handle_to_obj(handle);
1556 obj_to_location(obj, &f_page, &f_objidx);
1557 zspage = get_zspage(f_page);
1558
1559 migrate_read_lock(zspage);
1560
1561 get_zspage_mapping(zspage, &class_idx, &fullness);
1562 class = pool->size_class[class_idx];
1563
1564 spin_lock(&class->lock);
1565 obj_free(class, obj);
1566 fullness = fix_fullness_group(class, zspage);
1567 if (fullness != ZS_EMPTY) {
1568 migrate_read_unlock(zspage);
1569 goto out;
1570 }
1571
1572 isolated = is_zspage_isolated(zspage);
1573 migrate_read_unlock(zspage);
1574
1575 if (likely(!isolated))
1576 free_zspage(pool, class, zspage);
1577out:
1578
1579 spin_unlock(&class->lock);
1580 unpin_tag(handle);
1581 cache_free_handle(pool, handle);
1582}
1583EXPORT_SYMBOL_GPL(zs_free);
1584
1585static void zs_object_copy(struct size_class *class, unsigned long dst,
1586 unsigned long src)
1587{
1588 struct page *s_page, *d_page;
1589 unsigned int s_objidx, d_objidx;
1590 unsigned long s_off, d_off;
1591 void *s_addr, *d_addr;
1592 int s_size, d_size, size;
1593 int written = 0;
1594
1595 s_size = d_size = class->size;
1596
1597 obj_to_location(src, &s_page, &s_objidx);
1598 obj_to_location(dst, &d_page, &d_objidx);
1599
1600 s_off = (class->size * s_objidx) & ~PAGE_MASK;
1601 d_off = (class->size * d_objidx) & ~PAGE_MASK;
1602
1603 if (s_off + class->size > PAGE_SIZE)
1604 s_size = PAGE_SIZE - s_off;
1605
1606 if (d_off + class->size > PAGE_SIZE)
1607 d_size = PAGE_SIZE - d_off;
1608
1609 s_addr = kmap_atomic(s_page);
1610 d_addr = kmap_atomic(d_page);
1611
1612 while (1) {
1613 size = min(s_size, d_size);
1614 memcpy(d_addr + d_off, s_addr + s_off, size);
1615 written += size;
1616
1617 if (written == class->size)
1618 break;
1619
1620 s_off += size;
1621 s_size -= size;
1622 d_off += size;
1623 d_size -= size;
1624
1625 if (s_off >= PAGE_SIZE) {
1626 kunmap_atomic(d_addr);
1627 kunmap_atomic(s_addr);
1628 s_page = get_next_page(s_page);
1629 s_addr = kmap_atomic(s_page);
1630 d_addr = kmap_atomic(d_page);
1631 s_size = class->size - written;
1632 s_off = 0;
1633 }
1634
1635 if (d_off >= PAGE_SIZE) {
1636 kunmap_atomic(d_addr);
1637 d_page = get_next_page(d_page);
1638 d_addr = kmap_atomic(d_page);
1639 d_size = class->size - written;
1640 d_off = 0;
1641 }
1642 }
1643
1644 kunmap_atomic(d_addr);
1645 kunmap_atomic(s_addr);
1646}
1647
1648
1649
1650
1651
1652static unsigned long find_alloced_obj(struct size_class *class,
1653 struct page *page, int *obj_idx)
1654{
1655 unsigned long head;
1656 int offset = 0;
1657 int index = *obj_idx;
1658 unsigned long handle = 0;
1659 void *addr = kmap_atomic(page);
1660
1661 offset = get_first_obj_offset(page);
1662 offset += class->size * index;
1663
1664 while (offset < PAGE_SIZE) {
1665 head = obj_to_head(page, addr + offset);
1666 if (head & OBJ_ALLOCATED_TAG) {
1667 handle = head & ~OBJ_ALLOCATED_TAG;
1668 if (trypin_tag(handle))
1669 break;
1670 handle = 0;
1671 }
1672
1673 offset += class->size;
1674 index++;
1675 }
1676
1677 kunmap_atomic(addr);
1678
1679 *obj_idx = index;
1680
1681 return handle;
1682}
1683
1684struct zs_compact_control {
1685
1686 struct page *s_page;
1687
1688
1689 struct page *d_page;
1690
1691
1692 int obj_idx;
1693};
1694
1695static int migrate_zspage(struct zs_pool *pool, struct size_class *class,
1696 struct zs_compact_control *cc)
1697{
1698 unsigned long used_obj, free_obj;
1699 unsigned long handle;
1700 struct page *s_page = cc->s_page;
1701 struct page *d_page = cc->d_page;
1702 int obj_idx = cc->obj_idx;
1703 int ret = 0;
1704
1705 while (1) {
1706 handle = find_alloced_obj(class, s_page, &obj_idx);
1707 if (!handle) {
1708 s_page = get_next_page(s_page);
1709 if (!s_page)
1710 break;
1711 obj_idx = 0;
1712 continue;
1713 }
1714
1715
1716 if (zspage_full(class, get_zspage(d_page))) {
1717 unpin_tag(handle);
1718 ret = -ENOMEM;
1719 break;
1720 }
1721
1722 used_obj = handle_to_obj(handle);
1723 free_obj = obj_malloc(class, get_zspage(d_page), handle);
1724 zs_object_copy(class, free_obj, used_obj);
1725 obj_idx++;
1726
1727
1728
1729
1730
1731
1732 free_obj |= BIT(HANDLE_PIN_BIT);
1733 record_obj(handle, free_obj);
1734 unpin_tag(handle);
1735 obj_free(class, used_obj);
1736 }
1737
1738
1739 cc->s_page = s_page;
1740 cc->obj_idx = obj_idx;
1741
1742 return ret;
1743}
1744
1745static struct zspage *isolate_zspage(struct size_class *class, bool source)
1746{
1747 int i;
1748 struct zspage *zspage;
1749 enum fullness_group fg[2] = {ZS_ALMOST_EMPTY, ZS_ALMOST_FULL};
1750
1751 if (!source) {
1752 fg[0] = ZS_ALMOST_FULL;
1753 fg[1] = ZS_ALMOST_EMPTY;
1754 }
1755
1756 for (i = 0; i < 2; i++) {
1757 zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],
1758 struct zspage, list);
1759 if (zspage) {
1760 VM_BUG_ON(is_zspage_isolated(zspage));
1761 remove_zspage(class, zspage, fg[i]);
1762 return zspage;
1763 }
1764 }
1765
1766 return zspage;
1767}
1768
1769
1770
1771
1772
1773
1774
1775
1776static enum fullness_group putback_zspage(struct size_class *class,
1777 struct zspage *zspage)
1778{
1779 enum fullness_group fullness;
1780
1781 VM_BUG_ON(is_zspage_isolated(zspage));
1782
1783 fullness = get_fullness_group(class, zspage);
1784 insert_zspage(class, zspage, fullness);
1785 set_zspage_mapping(zspage, class->index, fullness);
1786
1787 return fullness;
1788}
1789
1790#ifdef CONFIG_COMPACTION
1791
1792
1793
1794
1795static void lock_zspage(struct zspage *zspage)
1796{
1797 struct page *page = get_first_page(zspage);
1798
1799 do {
1800 lock_page(page);
1801 } while ((page = get_next_page(page)) != NULL);
1802}
1803
1804static int zs_init_fs_context(struct fs_context *fc)
1805{
1806 return init_pseudo(fc, ZSMALLOC_MAGIC) ? 0 : -ENOMEM;
1807}
1808
1809static struct file_system_type zsmalloc_fs = {
1810 .name = "zsmalloc",
1811 .init_fs_context = zs_init_fs_context,
1812 .kill_sb = kill_anon_super,
1813};
1814
1815static int zsmalloc_mount(void)
1816{
1817 int ret = 0;
1818
1819 zsmalloc_mnt = kern_mount(&zsmalloc_fs);
1820 if (IS_ERR(zsmalloc_mnt))
1821 ret = PTR_ERR(zsmalloc_mnt);
1822
1823 return ret;
1824}
1825
1826static void zsmalloc_unmount(void)
1827{
1828 kern_unmount(zsmalloc_mnt);
1829}
1830
1831static void migrate_lock_init(struct zspage *zspage)
1832{
1833 rwlock_init(&zspage->lock);
1834}
1835
1836static void migrate_read_lock(struct zspage *zspage)
1837{
1838 read_lock(&zspage->lock);
1839}
1840
1841static void migrate_read_unlock(struct zspage *zspage)
1842{
1843 read_unlock(&zspage->lock);
1844}
1845
1846static void migrate_write_lock(struct zspage *zspage)
1847{
1848 write_lock(&zspage->lock);
1849}
1850
1851static void migrate_write_unlock(struct zspage *zspage)
1852{
1853 write_unlock(&zspage->lock);
1854}
1855
1856
1857static void inc_zspage_isolation(struct zspage *zspage)
1858{
1859 zspage->isolated++;
1860}
1861
1862static void dec_zspage_isolation(struct zspage *zspage)
1863{
1864 zspage->isolated--;
1865}
1866
1867static void putback_zspage_deferred(struct zs_pool *pool,
1868 struct size_class *class,
1869 struct zspage *zspage)
1870{
1871 enum fullness_group fg;
1872
1873 fg = putback_zspage(class, zspage);
1874 if (fg == ZS_EMPTY)
1875 schedule_work(&pool->free_work);
1876
1877}
1878
1879static inline void zs_pool_dec_isolated(struct zs_pool *pool)
1880{
1881 VM_BUG_ON(atomic_long_read(&pool->isolated_pages) <= 0);
1882 atomic_long_dec(&pool->isolated_pages);
1883
1884
1885
1886
1887
1888 if (atomic_long_read(&pool->isolated_pages) == 0 && pool->destroying)
1889 wake_up_all(&pool->migration_wait);
1890}
1891
1892static void replace_sub_page(struct size_class *class, struct zspage *zspage,
1893 struct page *newpage, struct page *oldpage)
1894{
1895 struct page *page;
1896 struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE] = {NULL, };
1897 int idx = 0;
1898
1899 page = get_first_page(zspage);
1900 do {
1901 if (page == oldpage)
1902 pages[idx] = newpage;
1903 else
1904 pages[idx] = page;
1905 idx++;
1906 } while ((page = get_next_page(page)) != NULL);
1907
1908 create_page_chain(class, zspage, pages);
1909 set_first_obj_offset(newpage, get_first_obj_offset(oldpage));
1910 if (unlikely(PageHugeObject(oldpage)))
1911 newpage->index = oldpage->index;
1912 __SetPageMovable(newpage, page_mapping(oldpage));
1913}
1914
1915static bool zs_page_isolate(struct page *page, isolate_mode_t mode)
1916{
1917 struct zs_pool *pool;
1918 struct size_class *class;
1919 int class_idx;
1920 enum fullness_group fullness;
1921 struct zspage *zspage;
1922 struct address_space *mapping;
1923
1924
1925
1926
1927
1928 VM_BUG_ON_PAGE(!PageMovable(page), page);
1929 VM_BUG_ON_PAGE(PageIsolated(page), page);
1930
1931 zspage = get_zspage(page);
1932
1933
1934
1935
1936
1937
1938 get_zspage_mapping(zspage, &class_idx, &fullness);
1939 mapping = page_mapping(page);
1940 pool = mapping->private_data;
1941 class = pool->size_class[class_idx];
1942
1943 spin_lock(&class->lock);
1944 if (get_zspage_inuse(zspage) == 0) {
1945 spin_unlock(&class->lock);
1946 return false;
1947 }
1948
1949
1950 if (list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1951 spin_unlock(&class->lock);
1952 return false;
1953 }
1954
1955
1956
1957
1958
1959 if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1960 get_zspage_mapping(zspage, &class_idx, &fullness);
1961 atomic_long_inc(&pool->isolated_pages);
1962 remove_zspage(class, zspage, fullness);
1963 }
1964
1965 inc_zspage_isolation(zspage);
1966 spin_unlock(&class->lock);
1967
1968 return true;
1969}
1970
1971static int zs_page_migrate(struct address_space *mapping, struct page *newpage,
1972 struct page *page, enum migrate_mode mode)
1973{
1974 struct zs_pool *pool;
1975 struct size_class *class;
1976 int class_idx;
1977 enum fullness_group fullness;
1978 struct zspage *zspage;
1979 struct page *dummy;
1980 void *s_addr, *d_addr, *addr;
1981 int offset, pos;
1982 unsigned long handle, head;
1983 unsigned long old_obj, new_obj;
1984 unsigned int obj_idx;
1985 int ret = -EAGAIN;
1986
1987
1988
1989
1990
1991
1992 if (mode == MIGRATE_SYNC_NO_COPY)
1993 return -EINVAL;
1994
1995 VM_BUG_ON_PAGE(!PageMovable(page), page);
1996 VM_BUG_ON_PAGE(!PageIsolated(page), page);
1997
1998 zspage = get_zspage(page);
1999
2000
2001 migrate_write_lock(zspage);
2002 get_zspage_mapping(zspage, &class_idx, &fullness);
2003 pool = mapping->private_data;
2004 class = pool->size_class[class_idx];
2005 offset = get_first_obj_offset(page);
2006
2007 spin_lock(&class->lock);
2008 if (!get_zspage_inuse(zspage)) {
2009
2010
2011
2012
2013 offset = PAGE_SIZE;
2014 }
2015
2016 pos = offset;
2017 s_addr = kmap_atomic(page);
2018 while (pos < PAGE_SIZE) {
2019 head = obj_to_head(page, s_addr + pos);
2020 if (head & OBJ_ALLOCATED_TAG) {
2021 handle = head & ~OBJ_ALLOCATED_TAG;
2022 if (!trypin_tag(handle))
2023 goto unpin_objects;
2024 }
2025 pos += class->size;
2026 }
2027
2028
2029
2030
2031 d_addr = kmap_atomic(newpage);
2032 memcpy(d_addr, s_addr, PAGE_SIZE);
2033 kunmap_atomic(d_addr);
2034
2035 for (addr = s_addr + offset; addr < s_addr + pos;
2036 addr += class->size) {
2037 head = obj_to_head(page, addr);
2038 if (head & OBJ_ALLOCATED_TAG) {
2039 handle = head & ~OBJ_ALLOCATED_TAG;
2040 if (!testpin_tag(handle))
2041 BUG();
2042
2043 old_obj = handle_to_obj(handle);
2044 obj_to_location(old_obj, &dummy, &obj_idx);
2045 new_obj = (unsigned long)location_to_obj(newpage,
2046 obj_idx);
2047 new_obj |= BIT(HANDLE_PIN_BIT);
2048 record_obj(handle, new_obj);
2049 }
2050 }
2051
2052 replace_sub_page(class, zspage, newpage, page);
2053 get_page(newpage);
2054
2055 dec_zspage_isolation(zspage);
2056
2057
2058
2059
2060
2061 if (!is_zspage_isolated(zspage)) {
2062
2063
2064
2065
2066
2067
2068 putback_zspage_deferred(pool, class, zspage);
2069 zs_pool_dec_isolated(pool);
2070 }
2071
2072 if (page_zone(newpage) != page_zone(page)) {
2073 dec_zone_page_state(page, NR_ZSPAGES);
2074 inc_zone_page_state(newpage, NR_ZSPAGES);
2075 }
2076
2077 reset_page(page);
2078 put_page(page);
2079 page = newpage;
2080
2081 ret = MIGRATEPAGE_SUCCESS;
2082unpin_objects:
2083 for (addr = s_addr + offset; addr < s_addr + pos;
2084 addr += class->size) {
2085 head = obj_to_head(page, addr);
2086 if (head & OBJ_ALLOCATED_TAG) {
2087 handle = head & ~OBJ_ALLOCATED_TAG;
2088 if (!testpin_tag(handle))
2089 BUG();
2090 unpin_tag(handle);
2091 }
2092 }
2093 kunmap_atomic(s_addr);
2094 spin_unlock(&class->lock);
2095 migrate_write_unlock(zspage);
2096
2097 return ret;
2098}
2099
2100static void zs_page_putback(struct page *page)
2101{
2102 struct zs_pool *pool;
2103 struct size_class *class;
2104 int class_idx;
2105 enum fullness_group fg;
2106 struct address_space *mapping;
2107 struct zspage *zspage;
2108
2109 VM_BUG_ON_PAGE(!PageMovable(page), page);
2110 VM_BUG_ON_PAGE(!PageIsolated(page), page);
2111
2112 zspage = get_zspage(page);
2113 get_zspage_mapping(zspage, &class_idx, &fg);
2114 mapping = page_mapping(page);
2115 pool = mapping->private_data;
2116 class = pool->size_class[class_idx];
2117
2118 spin_lock(&class->lock);
2119 dec_zspage_isolation(zspage);
2120 if (!is_zspage_isolated(zspage)) {
2121
2122
2123
2124
2125 putback_zspage_deferred(pool, class, zspage);
2126 zs_pool_dec_isolated(pool);
2127 }
2128 spin_unlock(&class->lock);
2129}
2130
2131static const struct address_space_operations zsmalloc_aops = {
2132 .isolate_page = zs_page_isolate,
2133 .migratepage = zs_page_migrate,
2134 .putback_page = zs_page_putback,
2135};
2136
2137static int zs_register_migration(struct zs_pool *pool)
2138{
2139 pool->inode = alloc_anon_inode(zsmalloc_mnt->mnt_sb);
2140 if (IS_ERR(pool->inode)) {
2141 pool->inode = NULL;
2142 return 1;
2143 }
2144
2145 pool->inode->i_mapping->private_data = pool;
2146 pool->inode->i_mapping->a_ops = &zsmalloc_aops;
2147 return 0;
2148}
2149
2150static bool pool_isolated_are_drained(struct zs_pool *pool)
2151{
2152 return atomic_long_read(&pool->isolated_pages) == 0;
2153}
2154
2155
2156static void wait_for_isolated_drain(struct zs_pool *pool)
2157{
2158
2159
2160
2161
2162
2163
2164
2165 wait_event(pool->migration_wait,
2166 pool_isolated_are_drained(pool));
2167}
2168
2169static void zs_unregister_migration(struct zs_pool *pool)
2170{
2171 pool->destroying = true;
2172
2173
2174
2175
2176
2177
2178 smp_mb();
2179 wait_for_isolated_drain(pool);
2180 flush_work(&pool->free_work);
2181 iput(pool->inode);
2182}
2183
2184
2185
2186
2187
2188static void async_free_zspage(struct work_struct *work)
2189{
2190 int i;
2191 struct size_class *class;
2192 unsigned int class_idx;
2193 enum fullness_group fullness;
2194 struct zspage *zspage, *tmp;
2195 LIST_HEAD(free_pages);
2196 struct zs_pool *pool = container_of(work, struct zs_pool,
2197 free_work);
2198
2199 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
2200 class = pool->size_class[i];
2201 if (class->index != i)
2202 continue;
2203
2204 spin_lock(&class->lock);
2205 list_splice_init(&class->fullness_list[ZS_EMPTY], &free_pages);
2206 spin_unlock(&class->lock);
2207 }
2208
2209
2210 list_for_each_entry_safe(zspage, tmp, &free_pages, list) {
2211 list_del(&zspage->list);
2212 lock_zspage(zspage);
2213
2214 get_zspage_mapping(zspage, &class_idx, &fullness);
2215 VM_BUG_ON(fullness != ZS_EMPTY);
2216 class = pool->size_class[class_idx];
2217 spin_lock(&class->lock);
2218 __free_zspage(pool, pool->size_class[class_idx], zspage);
2219 spin_unlock(&class->lock);
2220 }
2221};
2222
2223static void kick_deferred_free(struct zs_pool *pool)
2224{
2225 schedule_work(&pool->free_work);
2226}
2227
2228static void init_deferred_free(struct zs_pool *pool)
2229{
2230 INIT_WORK(&pool->free_work, async_free_zspage);
2231}
2232
2233static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)
2234{
2235 struct page *page = get_first_page(zspage);
2236
2237 do {
2238 WARN_ON(!trylock_page(page));
2239 __SetPageMovable(page, pool->inode->i_mapping);
2240 unlock_page(page);
2241 } while ((page = get_next_page(page)) != NULL);
2242}
2243#endif
2244
2245
2246
2247
2248
2249
2250static unsigned long zs_can_compact(struct size_class *class)
2251{
2252 unsigned long obj_wasted;
2253 unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
2254 unsigned long obj_used = zs_stat_get(class, OBJ_USED);
2255
2256 if (obj_allocated <= obj_used)
2257 return 0;
2258
2259 obj_wasted = obj_allocated - obj_used;
2260 obj_wasted /= class->objs_per_zspage;
2261
2262 return obj_wasted * class->pages_per_zspage;
2263}
2264
2265static void __zs_compact(struct zs_pool *pool, struct size_class *class)
2266{
2267 struct zs_compact_control cc;
2268 struct zspage *src_zspage;
2269 struct zspage *dst_zspage = NULL;
2270
2271 spin_lock(&class->lock);
2272 while ((src_zspage = isolate_zspage(class, true))) {
2273
2274 if (!zs_can_compact(class))
2275 break;
2276
2277 cc.obj_idx = 0;
2278 cc.s_page = get_first_page(src_zspage);
2279
2280 while ((dst_zspage = isolate_zspage(class, false))) {
2281 cc.d_page = get_first_page(dst_zspage);
2282
2283
2284
2285
2286 if (!migrate_zspage(pool, class, &cc))
2287 break;
2288
2289 putback_zspage(class, dst_zspage);
2290 }
2291
2292
2293 if (dst_zspage == NULL)
2294 break;
2295
2296 putback_zspage(class, dst_zspage);
2297 if (putback_zspage(class, src_zspage) == ZS_EMPTY) {
2298 free_zspage(pool, class, src_zspage);
2299 pool->stats.pages_compacted += class->pages_per_zspage;
2300 }
2301 spin_unlock(&class->lock);
2302 cond_resched();
2303 spin_lock(&class->lock);
2304 }
2305
2306 if (src_zspage)
2307 putback_zspage(class, src_zspage);
2308
2309 spin_unlock(&class->lock);
2310}
2311
2312unsigned long zs_compact(struct zs_pool *pool)
2313{
2314 int i;
2315 struct size_class *class;
2316
2317 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2318 class = pool->size_class[i];
2319 if (!class)
2320 continue;
2321 if (class->index != i)
2322 continue;
2323 __zs_compact(pool, class);
2324 }
2325
2326 return pool->stats.pages_compacted;
2327}
2328EXPORT_SYMBOL_GPL(zs_compact);
2329
2330void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats)
2331{
2332 memcpy(stats, &pool->stats, sizeof(struct zs_pool_stats));
2333}
2334EXPORT_SYMBOL_GPL(zs_pool_stats);
2335
2336static unsigned long zs_shrinker_scan(struct shrinker *shrinker,
2337 struct shrink_control *sc)
2338{
2339 unsigned long pages_freed;
2340 struct zs_pool *pool = container_of(shrinker, struct zs_pool,
2341 shrinker);
2342
2343 pages_freed = pool->stats.pages_compacted;
2344
2345
2346
2347
2348
2349 pages_freed = zs_compact(pool) - pages_freed;
2350
2351 return pages_freed ? pages_freed : SHRINK_STOP;
2352}
2353
2354static unsigned long zs_shrinker_count(struct shrinker *shrinker,
2355 struct shrink_control *sc)
2356{
2357 int i;
2358 struct size_class *class;
2359 unsigned long pages_to_free = 0;
2360 struct zs_pool *pool = container_of(shrinker, struct zs_pool,
2361 shrinker);
2362
2363 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2364 class = pool->size_class[i];
2365 if (!class)
2366 continue;
2367 if (class->index != i)
2368 continue;
2369
2370 pages_to_free += zs_can_compact(class);
2371 }
2372
2373 return pages_to_free;
2374}
2375
2376static void zs_unregister_shrinker(struct zs_pool *pool)
2377{
2378 unregister_shrinker(&pool->shrinker);
2379}
2380
2381static int zs_register_shrinker(struct zs_pool *pool)
2382{
2383 pool->shrinker.scan_objects = zs_shrinker_scan;
2384 pool->shrinker.count_objects = zs_shrinker_count;
2385 pool->shrinker.batch = 0;
2386 pool->shrinker.seeks = DEFAULT_SEEKS;
2387
2388 return register_shrinker(&pool->shrinker);
2389}
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401struct zs_pool *zs_create_pool(const char *name)
2402{
2403 int i;
2404 struct zs_pool *pool;
2405 struct size_class *prev_class = NULL;
2406
2407 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
2408 if (!pool)
2409 return NULL;
2410
2411 init_deferred_free(pool);
2412
2413 pool->name = kstrdup(name, GFP_KERNEL);
2414 if (!pool->name)
2415 goto err;
2416
2417#ifdef CONFIG_COMPACTION
2418 init_waitqueue_head(&pool->migration_wait);
2419#endif
2420
2421 if (create_cache(pool))
2422 goto err;
2423
2424
2425
2426
2427
2428 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2429 int size;
2430 int pages_per_zspage;
2431 int objs_per_zspage;
2432 struct size_class *class;
2433 int fullness = 0;
2434
2435 size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA;
2436 if (size > ZS_MAX_ALLOC_SIZE)
2437 size = ZS_MAX_ALLOC_SIZE;
2438 pages_per_zspage = get_pages_per_zspage(size);
2439 objs_per_zspage = pages_per_zspage * PAGE_SIZE / size;
2440
2441
2442
2443
2444
2445
2446
2447 if (pages_per_zspage != 1 && objs_per_zspage != 1 &&
2448 !huge_class_size) {
2449 huge_class_size = size;
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459 huge_class_size -= (ZS_HANDLE_SIZE - 1);
2460 }
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471 if (prev_class) {
2472 if (can_merge(prev_class, pages_per_zspage, objs_per_zspage)) {
2473 pool->size_class[i] = prev_class;
2474 continue;
2475 }
2476 }
2477
2478 class = kzalloc(sizeof(struct size_class), GFP_KERNEL);
2479 if (!class)
2480 goto err;
2481
2482 class->size = size;
2483 class->index = i;
2484 class->pages_per_zspage = pages_per_zspage;
2485 class->objs_per_zspage = objs_per_zspage;
2486 spin_lock_init(&class->lock);
2487 pool->size_class[i] = class;
2488 for (fullness = ZS_EMPTY; fullness < NR_ZS_FULLNESS;
2489 fullness++)
2490 INIT_LIST_HEAD(&class->fullness_list[fullness]);
2491
2492 prev_class = class;
2493 }
2494
2495
2496 zs_pool_stat_create(pool, name);
2497
2498 if (zs_register_migration(pool))
2499 goto err;
2500
2501
2502
2503
2504
2505
2506
2507 zs_register_shrinker(pool);
2508
2509 return pool;
2510
2511err:
2512 zs_destroy_pool(pool);
2513 return NULL;
2514}
2515EXPORT_SYMBOL_GPL(zs_create_pool);
2516
2517void zs_destroy_pool(struct zs_pool *pool)
2518{
2519 int i;
2520
2521 zs_unregister_shrinker(pool);
2522 zs_unregister_migration(pool);
2523 zs_pool_stat_destroy(pool);
2524
2525 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
2526 int fg;
2527 struct size_class *class = pool->size_class[i];
2528
2529 if (!class)
2530 continue;
2531
2532 if (class->index != i)
2533 continue;
2534
2535 for (fg = ZS_EMPTY; fg < NR_ZS_FULLNESS; fg++) {
2536 if (!list_empty(&class->fullness_list[fg])) {
2537 pr_info("Freeing non-empty class with size %db, fullness group %d\n",
2538 class->size, fg);
2539 }
2540 }
2541 kfree(class);
2542 }
2543
2544 destroy_cache(pool);
2545 kfree(pool->name);
2546 kfree(pool);
2547}
2548EXPORT_SYMBOL_GPL(zs_destroy_pool);
2549
2550static int __init zs_init(void)
2551{
2552 int ret;
2553
2554 ret = zsmalloc_mount();
2555 if (ret)
2556 goto out;
2557
2558 ret = cpuhp_setup_state(CPUHP_MM_ZS_PREPARE, "mm/zsmalloc:prepare",
2559 zs_cpu_prepare, zs_cpu_dead);
2560 if (ret)
2561 goto hp_setup_fail;
2562
2563#ifdef CONFIG_ZPOOL
2564 zpool_register_driver(&zs_zpool_driver);
2565#endif
2566
2567 zs_stat_init();
2568
2569 return 0;
2570
2571hp_setup_fail:
2572 zsmalloc_unmount();
2573out:
2574 return ret;
2575}
2576
2577static void __exit zs_exit(void)
2578{
2579#ifdef CONFIG_ZPOOL
2580 zpool_unregister_driver(&zs_zpool_driver);
2581#endif
2582 zsmalloc_unmount();
2583 cpuhp_remove_state(CPUHP_MM_ZS_PREPARE);
2584
2585 zs_stat_exit();
2586}
2587
2588module_init(zs_init);
2589module_exit(zs_exit);
2590
2591MODULE_LICENSE("Dual BSD/GPL");
2592MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
2593