1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
32
33#include <linux/module.h>
34#include <linux/kernel.h>
35#include <linux/sched.h>
36#include <linux/magic.h>
37#include <linux/bitops.h>
38#include <linux/errno.h>
39#include <linux/highmem.h>
40#include <linux/string.h>
41#include <linux/slab.h>
42#include <asm/tlbflush.h>
43#include <asm/pgtable.h>
44#include <linux/cpumask.h>
45#include <linux/cpu.h>
46#include <linux/vmalloc.h>
47#include <linux/preempt.h>
48#include <linux/spinlock.h>
49#include <linux/shrinker.h>
50#include <linux/types.h>
51#include <linux/debugfs.h>
52#include <linux/zsmalloc.h>
53#include <linux/zpool.h>
54#include <linux/mount.h>
55#include <linux/pseudo_fs.h>
56#include <linux/migrate.h>
57#include <linux/wait.h>
58#include <linux/pagemap.h>
59#include <linux/fs.h>
60
61#define ZSPAGE_MAGIC 0x58
62
63
64
65
66
67
68
69#define ZS_ALIGN 8
70
71
72
73
74
75#define ZS_MAX_ZSPAGE_ORDER 2
76#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)
77
78#define ZS_HANDLE_SIZE (sizeof(unsigned long))
79
80
81
82
83
84
85
86
87
88
89#ifndef MAX_POSSIBLE_PHYSMEM_BITS
90#ifdef MAX_PHYSMEM_BITS
91#define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS
92#else
93
94
95
96
97#define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG
98#endif
99#endif
100
101#define _PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
102
103
104
105
106
107
108
109
110#define HANDLE_PIN_BIT 0
111
112
113
114
115
116
117
118
119#define OBJ_ALLOCATED_TAG 1
120#define OBJ_TAG_BITS 1
121#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS - OBJ_TAG_BITS)
122#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
123
124#define FULLNESS_BITS 2
125#define CLASS_BITS 8
126#define ISOLATED_BITS 3
127#define MAGIC_VAL_BITS 8
128
129#define MAX(a, b) ((a) >= (b) ? (a) : (b))
130
131#define ZS_MIN_ALLOC_SIZE \
132 MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
133
134#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149#define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> CLASS_BITS)
150#define ZS_SIZE_CLASSES (DIV_ROUND_UP(ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE, \
151 ZS_SIZE_CLASS_DELTA) + 1)
152
153enum fullness_group {
154 ZS_EMPTY,
155 ZS_ALMOST_EMPTY,
156 ZS_ALMOST_FULL,
157 ZS_FULL,
158 NR_ZS_FULLNESS,
159};
160
161enum zs_stat_type {
162 CLASS_EMPTY,
163 CLASS_ALMOST_EMPTY,
164 CLASS_ALMOST_FULL,
165 CLASS_FULL,
166 OBJ_ALLOCATED,
167 OBJ_USED,
168 NR_ZS_STAT_TYPE,
169};
170
171struct zs_size_stat {
172 unsigned long objs[NR_ZS_STAT_TYPE];
173};
174
175#ifdef CONFIG_ZSMALLOC_STAT
176static struct dentry *zs_stat_root;
177#endif
178
179#ifdef CONFIG_COMPACTION
180static struct vfsmount *zsmalloc_mnt;
181#endif
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197static const int fullness_threshold_frac = 4;
198static size_t huge_class_size;
199
200struct size_class {
201 spinlock_t lock;
202 struct list_head fullness_list[NR_ZS_FULLNESS];
203
204
205
206
207 int size;
208 int objs_per_zspage;
209
210 int pages_per_zspage;
211
212 unsigned int index;
213 struct zs_size_stat stats;
214};
215
216
217static void SetPageHugeObject(struct page *page)
218{
219 SetPageOwnerPriv1(page);
220}
221
222static void ClearPageHugeObject(struct page *page)
223{
224 ClearPageOwnerPriv1(page);
225}
226
227static int PageHugeObject(struct page *page)
228{
229 return PageOwnerPriv1(page);
230}
231
232
233
234
235
236
237
238struct link_free {
239 union {
240
241
242
243
244 unsigned long next;
245
246
247
248 unsigned long handle;
249 };
250};
251
252struct zs_pool {
253 const char *name;
254
255 struct size_class *size_class[ZS_SIZE_CLASSES];
256 struct kmem_cache *handle_cachep;
257 struct kmem_cache *zspage_cachep;
258
259 atomic_long_t pages_allocated;
260
261 struct zs_pool_stats stats;
262
263
264 struct shrinker shrinker;
265
266#ifdef CONFIG_ZSMALLOC_STAT
267 struct dentry *stat_dentry;
268#endif
269#ifdef CONFIG_COMPACTION
270 struct inode *inode;
271 struct work_struct free_work;
272
273 struct wait_queue_head migration_wait;
274 atomic_long_t isolated_pages;
275 bool destroying;
276#endif
277};
278
279struct zspage {
280 struct {
281 unsigned int fullness:FULLNESS_BITS;
282 unsigned int class:CLASS_BITS + 1;
283 unsigned int isolated:ISOLATED_BITS;
284 unsigned int magic:MAGIC_VAL_BITS;
285 };
286 unsigned int inuse;
287 unsigned int freeobj;
288 struct page *first_page;
289 struct list_head list;
290#ifdef CONFIG_COMPACTION
291 rwlock_t lock;
292#endif
293};
294
295struct mapping_area {
296#ifdef CONFIG_PGTABLE_MAPPING
297 struct vm_struct *vm;
298#else
299 char *vm_buf;
300#endif
301 char *vm_addr;
302 enum zs_mapmode vm_mm;
303};
304
305#ifdef CONFIG_COMPACTION
306static int zs_register_migration(struct zs_pool *pool);
307static void zs_unregister_migration(struct zs_pool *pool);
308static void migrate_lock_init(struct zspage *zspage);
309static void migrate_read_lock(struct zspage *zspage);
310static void migrate_read_unlock(struct zspage *zspage);
311static void kick_deferred_free(struct zs_pool *pool);
312static void init_deferred_free(struct zs_pool *pool);
313static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
314#else
315static int zsmalloc_mount(void) { return 0; }
316static void zsmalloc_unmount(void) {}
317static int zs_register_migration(struct zs_pool *pool) { return 0; }
318static void zs_unregister_migration(struct zs_pool *pool) {}
319static void migrate_lock_init(struct zspage *zspage) {}
320static void migrate_read_lock(struct zspage *zspage) {}
321static void migrate_read_unlock(struct zspage *zspage) {}
322static void kick_deferred_free(struct zs_pool *pool) {}
323static void init_deferred_free(struct zs_pool *pool) {}
324static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}
325#endif
326
327static int create_cache(struct zs_pool *pool)
328{
329 pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE,
330 0, 0, NULL);
331 if (!pool->handle_cachep)
332 return 1;
333
334 pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),
335 0, 0, NULL);
336 if (!pool->zspage_cachep) {
337 kmem_cache_destroy(pool->handle_cachep);
338 pool->handle_cachep = NULL;
339 return 1;
340 }
341
342 return 0;
343}
344
345static void destroy_cache(struct zs_pool *pool)
346{
347 kmem_cache_destroy(pool->handle_cachep);
348 kmem_cache_destroy(pool->zspage_cachep);
349}
350
351static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp)
352{
353 return (unsigned long)kmem_cache_alloc(pool->handle_cachep,
354 gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
355}
356
357static void cache_free_handle(struct zs_pool *pool, unsigned long handle)
358{
359 kmem_cache_free(pool->handle_cachep, (void *)handle);
360}
361
362static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)
363{
364 return kmem_cache_alloc(pool->zspage_cachep,
365 flags & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
366}
367
368static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
369{
370 kmem_cache_free(pool->zspage_cachep, zspage);
371}
372
373static void record_obj(unsigned long handle, unsigned long obj)
374{
375
376
377
378
379
380 WRITE_ONCE(*(unsigned long *)handle, obj);
381}
382
383
384
385#ifdef CONFIG_ZPOOL
386
387static void *zs_zpool_create(const char *name, gfp_t gfp,
388 const struct zpool_ops *zpool_ops,
389 struct zpool *zpool)
390{
391
392
393
394
395
396 return zs_create_pool(name);
397}
398
399static void zs_zpool_destroy(void *pool)
400{
401 zs_destroy_pool(pool);
402}
403
404static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp,
405 unsigned long *handle)
406{
407 *handle = zs_malloc(pool, size, gfp);
408 return *handle ? 0 : -1;
409}
410static void zs_zpool_free(void *pool, unsigned long handle)
411{
412 zs_free(pool, handle);
413}
414
415static void *zs_zpool_map(void *pool, unsigned long handle,
416 enum zpool_mapmode mm)
417{
418 enum zs_mapmode zs_mm;
419
420 switch (mm) {
421 case ZPOOL_MM_RO:
422 zs_mm = ZS_MM_RO;
423 break;
424 case ZPOOL_MM_WO:
425 zs_mm = ZS_MM_WO;
426 break;
427 case ZPOOL_MM_RW:
428 default:
429 zs_mm = ZS_MM_RW;
430 break;
431 }
432
433 return zs_map_object(pool, handle, zs_mm);
434}
435static void zs_zpool_unmap(void *pool, unsigned long handle)
436{
437 zs_unmap_object(pool, handle);
438}
439
440static u64 zs_zpool_total_size(void *pool)
441{
442 return zs_get_total_pages(pool) << PAGE_SHIFT;
443}
444
445static struct zpool_driver zs_zpool_driver = {
446 .type = "zsmalloc",
447 .owner = THIS_MODULE,
448 .create = zs_zpool_create,
449 .destroy = zs_zpool_destroy,
450 .malloc = zs_zpool_malloc,
451 .free = zs_zpool_free,
452 .map = zs_zpool_map,
453 .unmap = zs_zpool_unmap,
454 .total_size = zs_zpool_total_size,
455};
456
457MODULE_ALIAS("zpool-zsmalloc");
458#endif
459
460
461static DEFINE_PER_CPU(struct mapping_area, zs_map_area);
462
463static bool is_zspage_isolated(struct zspage *zspage)
464{
465 return zspage->isolated;
466}
467
468static __maybe_unused int is_first_page(struct page *page)
469{
470 return PagePrivate(page);
471}
472
473
474static inline int get_zspage_inuse(struct zspage *zspage)
475{
476 return zspage->inuse;
477}
478
479static inline void set_zspage_inuse(struct zspage *zspage, int val)
480{
481 zspage->inuse = val;
482}
483
484static inline void mod_zspage_inuse(struct zspage *zspage, int val)
485{
486 zspage->inuse += val;
487}
488
489static inline struct page *get_first_page(struct zspage *zspage)
490{
491 struct page *first_page = zspage->first_page;
492
493 VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
494 return first_page;
495}
496
497static inline int get_first_obj_offset(struct page *page)
498{
499 return page->units;
500}
501
502static inline void set_first_obj_offset(struct page *page, int offset)
503{
504 page->units = offset;
505}
506
507static inline unsigned int get_freeobj(struct zspage *zspage)
508{
509 return zspage->freeobj;
510}
511
512static inline void set_freeobj(struct zspage *zspage, unsigned int obj)
513{
514 zspage->freeobj = obj;
515}
516
517static void get_zspage_mapping(struct zspage *zspage,
518 unsigned int *class_idx,
519 enum fullness_group *fullness)
520{
521 BUG_ON(zspage->magic != ZSPAGE_MAGIC);
522
523 *fullness = zspage->fullness;
524 *class_idx = zspage->class;
525}
526
527static void set_zspage_mapping(struct zspage *zspage,
528 unsigned int class_idx,
529 enum fullness_group fullness)
530{
531 zspage->class = class_idx;
532 zspage->fullness = fullness;
533}
534
535
536
537
538
539
540
541
542static int get_size_class_index(int size)
543{
544 int idx = 0;
545
546 if (likely(size > ZS_MIN_ALLOC_SIZE))
547 idx = DIV_ROUND_UP(size - ZS_MIN_ALLOC_SIZE,
548 ZS_SIZE_CLASS_DELTA);
549
550 return min_t(int, ZS_SIZE_CLASSES - 1, idx);
551}
552
553
554static inline void zs_stat_inc(struct size_class *class,
555 int type, unsigned long cnt)
556{
557 class->stats.objs[type] += cnt;
558}
559
560
561static inline void zs_stat_dec(struct size_class *class,
562 int type, unsigned long cnt)
563{
564 class->stats.objs[type] -= cnt;
565}
566
567
568static inline unsigned long zs_stat_get(struct size_class *class,
569 int type)
570{
571 return class->stats.objs[type];
572}
573
574#ifdef CONFIG_ZSMALLOC_STAT
575
576static void __init zs_stat_init(void)
577{
578 if (!debugfs_initialized()) {
579 pr_warn("debugfs not available, stat dir not created\n");
580 return;
581 }
582
583 zs_stat_root = debugfs_create_dir("zsmalloc", NULL);
584}
585
586static void __exit zs_stat_exit(void)
587{
588 debugfs_remove_recursive(zs_stat_root);
589}
590
591static unsigned long zs_can_compact(struct size_class *class);
592
593static int zs_stats_size_show(struct seq_file *s, void *v)
594{
595 int i;
596 struct zs_pool *pool = s->private;
597 struct size_class *class;
598 int objs_per_zspage;
599 unsigned long class_almost_full, class_almost_empty;
600 unsigned long obj_allocated, obj_used, pages_used, freeable;
601 unsigned long total_class_almost_full = 0, total_class_almost_empty = 0;
602 unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0;
603 unsigned long total_freeable = 0;
604
605 seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n",
606 "class", "size", "almost_full", "almost_empty",
607 "obj_allocated", "obj_used", "pages_used",
608 "pages_per_zspage", "freeable");
609
610 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
611 class = pool->size_class[i];
612
613 if (class->index != i)
614 continue;
615
616 spin_lock(&class->lock);
617 class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL);
618 class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
619 obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
620 obj_used = zs_stat_get(class, OBJ_USED);
621 freeable = zs_can_compact(class);
622 spin_unlock(&class->lock);
623
624 objs_per_zspage = class->objs_per_zspage;
625 pages_used = obj_allocated / objs_per_zspage *
626 class->pages_per_zspage;
627
628 seq_printf(s, " %5u %5u %11lu %12lu %13lu"
629 " %10lu %10lu %16d %8lu\n",
630 i, class->size, class_almost_full, class_almost_empty,
631 obj_allocated, obj_used, pages_used,
632 class->pages_per_zspage, freeable);
633
634 total_class_almost_full += class_almost_full;
635 total_class_almost_empty += class_almost_empty;
636 total_objs += obj_allocated;
637 total_used_objs += obj_used;
638 total_pages += pages_used;
639 total_freeable += freeable;
640 }
641
642 seq_puts(s, "\n");
643 seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n",
644 "Total", "", total_class_almost_full,
645 total_class_almost_empty, total_objs,
646 total_used_objs, total_pages, "", total_freeable);
647
648 return 0;
649}
650DEFINE_SHOW_ATTRIBUTE(zs_stats_size);
651
652static void zs_pool_stat_create(struct zs_pool *pool, const char *name)
653{
654 if (!zs_stat_root) {
655 pr_warn("no root stat dir, not creating <%s> stat dir\n", name);
656 return;
657 }
658
659 pool->stat_dentry = debugfs_create_dir(name, zs_stat_root);
660
661 debugfs_create_file("classes", S_IFREG | 0444, pool->stat_dentry, pool,
662 &zs_stats_size_fops);
663}
664
665static void zs_pool_stat_destroy(struct zs_pool *pool)
666{
667 debugfs_remove_recursive(pool->stat_dentry);
668}
669
670#else
671static void __init zs_stat_init(void)
672{
673}
674
675static void __exit zs_stat_exit(void)
676{
677}
678
679static inline void zs_pool_stat_create(struct zs_pool *pool, const char *name)
680{
681}
682
683static inline void zs_pool_stat_destroy(struct zs_pool *pool)
684{
685}
686#endif
687
688
689
690
691
692
693
694
695
696static enum fullness_group get_fullness_group(struct size_class *class,
697 struct zspage *zspage)
698{
699 int inuse, objs_per_zspage;
700 enum fullness_group fg;
701
702 inuse = get_zspage_inuse(zspage);
703 objs_per_zspage = class->objs_per_zspage;
704
705 if (inuse == 0)
706 fg = ZS_EMPTY;
707 else if (inuse == objs_per_zspage)
708 fg = ZS_FULL;
709 else if (inuse <= 3 * objs_per_zspage / fullness_threshold_frac)
710 fg = ZS_ALMOST_EMPTY;
711 else
712 fg = ZS_ALMOST_FULL;
713
714 return fg;
715}
716
717
718
719
720
721
722
723static void insert_zspage(struct size_class *class,
724 struct zspage *zspage,
725 enum fullness_group fullness)
726{
727 struct zspage *head;
728
729 zs_stat_inc(class, fullness, 1);
730 head = list_first_entry_or_null(&class->fullness_list[fullness],
731 struct zspage, list);
732
733
734
735
736 if (head) {
737 if (get_zspage_inuse(zspage) < get_zspage_inuse(head)) {
738 list_add(&zspage->list, &head->list);
739 return;
740 }
741 }
742 list_add(&zspage->list, &class->fullness_list[fullness]);
743}
744
745
746
747
748
749static void remove_zspage(struct size_class *class,
750 struct zspage *zspage,
751 enum fullness_group fullness)
752{
753 VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
754 VM_BUG_ON(is_zspage_isolated(zspage));
755
756 list_del_init(&zspage->list);
757 zs_stat_dec(class, fullness, 1);
758}
759
760
761
762
763
764
765
766
767
768
769static enum fullness_group fix_fullness_group(struct size_class *class,
770 struct zspage *zspage)
771{
772 int class_idx;
773 enum fullness_group currfg, newfg;
774
775 get_zspage_mapping(zspage, &class_idx, &currfg);
776 newfg = get_fullness_group(class, zspage);
777 if (newfg == currfg)
778 goto out;
779
780 if (!is_zspage_isolated(zspage)) {
781 remove_zspage(class, zspage, currfg);
782 insert_zspage(class, zspage, newfg);
783 }
784
785 set_zspage_mapping(zspage, class_idx, newfg);
786
787out:
788 return newfg;
789}
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804static int get_pages_per_zspage(int class_size)
805{
806 int i, max_usedpc = 0;
807
808 int max_usedpc_order = 1;
809
810 for (i = 1; i <= ZS_MAX_PAGES_PER_ZSPAGE; i++) {
811 int zspage_size;
812 int waste, usedpc;
813
814 zspage_size = i * PAGE_SIZE;
815 waste = zspage_size % class_size;
816 usedpc = (zspage_size - waste) * 100 / zspage_size;
817
818 if (usedpc > max_usedpc) {
819 max_usedpc = usedpc;
820 max_usedpc_order = i;
821 }
822 }
823
824 return max_usedpc_order;
825}
826
827static struct zspage *get_zspage(struct page *page)
828{
829 struct zspage *zspage = (struct zspage *)page->private;
830
831 BUG_ON(zspage->magic != ZSPAGE_MAGIC);
832 return zspage;
833}
834
835static struct page *get_next_page(struct page *page)
836{
837 if (unlikely(PageHugeObject(page)))
838 return NULL;
839
840 return page->freelist;
841}
842
843
844
845
846
847
848
849static void obj_to_location(unsigned long obj, struct page **page,
850 unsigned int *obj_idx)
851{
852 obj >>= OBJ_TAG_BITS;
853 *page = pfn_to_page(obj >> OBJ_INDEX_BITS);
854 *obj_idx = (obj & OBJ_INDEX_MASK);
855}
856
857
858
859
860
861
862static unsigned long location_to_obj(struct page *page, unsigned int obj_idx)
863{
864 unsigned long obj;
865
866 obj = page_to_pfn(page) << OBJ_INDEX_BITS;
867 obj |= obj_idx & OBJ_INDEX_MASK;
868 obj <<= OBJ_TAG_BITS;
869
870 return obj;
871}
872
873static unsigned long handle_to_obj(unsigned long handle)
874{
875 return *(unsigned long *)handle;
876}
877
878static unsigned long obj_to_head(struct page *page, void *obj)
879{
880 if (unlikely(PageHugeObject(page))) {
881 VM_BUG_ON_PAGE(!is_first_page(page), page);
882 return page->index;
883 } else
884 return *(unsigned long *)obj;
885}
886
887static inline int testpin_tag(unsigned long handle)
888{
889 return bit_spin_is_locked(HANDLE_PIN_BIT, (unsigned long *)handle);
890}
891
892static inline int trypin_tag(unsigned long handle)
893{
894 return bit_spin_trylock(HANDLE_PIN_BIT, (unsigned long *)handle);
895}
896
897static void pin_tag(unsigned long handle)
898{
899 bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle);
900}
901
902static void unpin_tag(unsigned long handle)
903{
904 bit_spin_unlock(HANDLE_PIN_BIT, (unsigned long *)handle);
905}
906
907static void reset_page(struct page *page)
908{
909 __ClearPageMovable(page);
910 ClearPagePrivate(page);
911 set_page_private(page, 0);
912 page_mapcount_reset(page);
913 ClearPageHugeObject(page);
914 page->freelist = NULL;
915}
916
917static int trylock_zspage(struct zspage *zspage)
918{
919 struct page *cursor, *fail;
920
921 for (cursor = get_first_page(zspage); cursor != NULL; cursor =
922 get_next_page(cursor)) {
923 if (!trylock_page(cursor)) {
924 fail = cursor;
925 goto unlock;
926 }
927 }
928
929 return 1;
930unlock:
931 for (cursor = get_first_page(zspage); cursor != fail; cursor =
932 get_next_page(cursor))
933 unlock_page(cursor);
934
935 return 0;
936}
937
938static void __free_zspage(struct zs_pool *pool, struct size_class *class,
939 struct zspage *zspage)
940{
941 struct page *page, *next;
942 enum fullness_group fg;
943 unsigned int class_idx;
944
945 get_zspage_mapping(zspage, &class_idx, &fg);
946
947 assert_spin_locked(&class->lock);
948
949 VM_BUG_ON(get_zspage_inuse(zspage));
950 VM_BUG_ON(fg != ZS_EMPTY);
951
952 next = page = get_first_page(zspage);
953 do {
954 VM_BUG_ON_PAGE(!PageLocked(page), page);
955 next = get_next_page(page);
956 reset_page(page);
957 unlock_page(page);
958 dec_zone_page_state(page, NR_ZSPAGES);
959 put_page(page);
960 page = next;
961 } while (page != NULL);
962
963 cache_free_zspage(pool, zspage);
964
965 zs_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage);
966 atomic_long_sub(class->pages_per_zspage,
967 &pool->pages_allocated);
968}
969
970static void free_zspage(struct zs_pool *pool, struct size_class *class,
971 struct zspage *zspage)
972{
973 VM_BUG_ON(get_zspage_inuse(zspage));
974 VM_BUG_ON(list_empty(&zspage->list));
975
976 if (!trylock_zspage(zspage)) {
977 kick_deferred_free(pool);
978 return;
979 }
980
981 remove_zspage(class, zspage, ZS_EMPTY);
982 __free_zspage(pool, class, zspage);
983}
984
985
986static void init_zspage(struct size_class *class, struct zspage *zspage)
987{
988 unsigned int freeobj = 1;
989 unsigned long off = 0;
990 struct page *page = get_first_page(zspage);
991
992 while (page) {
993 struct page *next_page;
994 struct link_free *link;
995 void *vaddr;
996
997 set_first_obj_offset(page, off);
998
999 vaddr = kmap_atomic(page);
1000 link = (struct link_free *)vaddr + off / sizeof(*link);
1001
1002 while ((off += class->size) < PAGE_SIZE) {
1003 link->next = freeobj++ << OBJ_TAG_BITS;
1004 link += class->size / sizeof(*link);
1005 }
1006
1007
1008
1009
1010
1011
1012 next_page = get_next_page(page);
1013 if (next_page) {
1014 link->next = freeobj++ << OBJ_TAG_BITS;
1015 } else {
1016
1017
1018
1019
1020 link->next = -1UL << OBJ_TAG_BITS;
1021 }
1022 kunmap_atomic(vaddr);
1023 page = next_page;
1024 off %= PAGE_SIZE;
1025 }
1026
1027 set_freeobj(zspage, 0);
1028}
1029
1030static void create_page_chain(struct size_class *class, struct zspage *zspage,
1031 struct page *pages[])
1032{
1033 int i;
1034 struct page *page;
1035 struct page *prev_page = NULL;
1036 int nr_pages = class->pages_per_zspage;
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046 for (i = 0; i < nr_pages; i++) {
1047 page = pages[i];
1048 set_page_private(page, (unsigned long)zspage);
1049 page->freelist = NULL;
1050 if (i == 0) {
1051 zspage->first_page = page;
1052 SetPagePrivate(page);
1053 if (unlikely(class->objs_per_zspage == 1 &&
1054 class->pages_per_zspage == 1))
1055 SetPageHugeObject(page);
1056 } else {
1057 prev_page->freelist = page;
1058 }
1059 prev_page = page;
1060 }
1061}
1062
1063
1064
1065
1066static struct zspage *alloc_zspage(struct zs_pool *pool,
1067 struct size_class *class,
1068 gfp_t gfp)
1069{
1070 int i;
1071 struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE];
1072 struct zspage *zspage = cache_alloc_zspage(pool, gfp);
1073
1074 if (!zspage)
1075 return NULL;
1076
1077 memset(zspage, 0, sizeof(struct zspage));
1078 zspage->magic = ZSPAGE_MAGIC;
1079 migrate_lock_init(zspage);
1080
1081 for (i = 0; i < class->pages_per_zspage; i++) {
1082 struct page *page;
1083
1084 page = alloc_page(gfp);
1085 if (!page) {
1086 while (--i >= 0) {
1087 dec_zone_page_state(pages[i], NR_ZSPAGES);
1088 __free_page(pages[i]);
1089 }
1090 cache_free_zspage(pool, zspage);
1091 return NULL;
1092 }
1093
1094 inc_zone_page_state(page, NR_ZSPAGES);
1095 pages[i] = page;
1096 }
1097
1098 create_page_chain(class, zspage, pages);
1099 init_zspage(class, zspage);
1100
1101 return zspage;
1102}
1103
1104static struct zspage *find_get_zspage(struct size_class *class)
1105{
1106 int i;
1107 struct zspage *zspage;
1108
1109 for (i = ZS_ALMOST_FULL; i >= ZS_EMPTY; i--) {
1110 zspage = list_first_entry_or_null(&class->fullness_list[i],
1111 struct zspage, list);
1112 if (zspage)
1113 break;
1114 }
1115
1116 return zspage;
1117}
1118
1119#ifdef CONFIG_PGTABLE_MAPPING
1120static inline int __zs_cpu_up(struct mapping_area *area)
1121{
1122
1123
1124
1125
1126 if (area->vm)
1127 return 0;
1128 area->vm = alloc_vm_area(PAGE_SIZE * 2, NULL);
1129 if (!area->vm)
1130 return -ENOMEM;
1131 return 0;
1132}
1133
1134static inline void __zs_cpu_down(struct mapping_area *area)
1135{
1136 if (area->vm)
1137 free_vm_area(area->vm);
1138 area->vm = NULL;
1139}
1140
1141static inline void *__zs_map_object(struct mapping_area *area,
1142 struct page *pages[2], int off, int size)
1143{
1144 BUG_ON(map_vm_area(area->vm, PAGE_KERNEL, pages));
1145 area->vm_addr = area->vm->addr;
1146 return area->vm_addr + off;
1147}
1148
1149static inline void __zs_unmap_object(struct mapping_area *area,
1150 struct page *pages[2], int off, int size)
1151{
1152 unsigned long addr = (unsigned long)area->vm_addr;
1153
1154 unmap_kernel_range(addr, PAGE_SIZE * 2);
1155}
1156
1157#else
1158
1159static inline int __zs_cpu_up(struct mapping_area *area)
1160{
1161
1162
1163
1164
1165 if (area->vm_buf)
1166 return 0;
1167 area->vm_buf = kmalloc(ZS_MAX_ALLOC_SIZE, GFP_KERNEL);
1168 if (!area->vm_buf)
1169 return -ENOMEM;
1170 return 0;
1171}
1172
1173static inline void __zs_cpu_down(struct mapping_area *area)
1174{
1175 kfree(area->vm_buf);
1176 area->vm_buf = NULL;
1177}
1178
1179static void *__zs_map_object(struct mapping_area *area,
1180 struct page *pages[2], int off, int size)
1181{
1182 int sizes[2];
1183 void *addr;
1184 char *buf = area->vm_buf;
1185
1186
1187 pagefault_disable();
1188
1189
1190 if (area->vm_mm == ZS_MM_WO)
1191 goto out;
1192
1193 sizes[0] = PAGE_SIZE - off;
1194 sizes[1] = size - sizes[0];
1195
1196
1197 addr = kmap_atomic(pages[0]);
1198 memcpy(buf, addr + off, sizes[0]);
1199 kunmap_atomic(addr);
1200 addr = kmap_atomic(pages[1]);
1201 memcpy(buf + sizes[0], addr, sizes[1]);
1202 kunmap_atomic(addr);
1203out:
1204 return area->vm_buf;
1205}
1206
1207static void __zs_unmap_object(struct mapping_area *area,
1208 struct page *pages[2], int off, int size)
1209{
1210 int sizes[2];
1211 void *addr;
1212 char *buf;
1213
1214
1215 if (area->vm_mm == ZS_MM_RO)
1216 goto out;
1217
1218 buf = area->vm_buf;
1219 buf = buf + ZS_HANDLE_SIZE;
1220 size -= ZS_HANDLE_SIZE;
1221 off += ZS_HANDLE_SIZE;
1222
1223 sizes[0] = PAGE_SIZE - off;
1224 sizes[1] = size - sizes[0];
1225
1226
1227 addr = kmap_atomic(pages[0]);
1228 memcpy(addr + off, buf, sizes[0]);
1229 kunmap_atomic(addr);
1230 addr = kmap_atomic(pages[1]);
1231 memcpy(addr, buf + sizes[0], sizes[1]);
1232 kunmap_atomic(addr);
1233
1234out:
1235
1236 pagefault_enable();
1237}
1238
1239#endif
1240
1241static int zs_cpu_prepare(unsigned int cpu)
1242{
1243 struct mapping_area *area;
1244
1245 area = &per_cpu(zs_map_area, cpu);
1246 return __zs_cpu_up(area);
1247}
1248
1249static int zs_cpu_dead(unsigned int cpu)
1250{
1251 struct mapping_area *area;
1252
1253 area = &per_cpu(zs_map_area, cpu);
1254 __zs_cpu_down(area);
1255 return 0;
1256}
1257
1258static bool can_merge(struct size_class *prev, int pages_per_zspage,
1259 int objs_per_zspage)
1260{
1261 if (prev->pages_per_zspage == pages_per_zspage &&
1262 prev->objs_per_zspage == objs_per_zspage)
1263 return true;
1264
1265 return false;
1266}
1267
1268static bool zspage_full(struct size_class *class, struct zspage *zspage)
1269{
1270 return get_zspage_inuse(zspage) == class->objs_per_zspage;
1271}
1272
1273unsigned long zs_get_total_pages(struct zs_pool *pool)
1274{
1275 return atomic_long_read(&pool->pages_allocated);
1276}
1277EXPORT_SYMBOL_GPL(zs_get_total_pages);
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294void *zs_map_object(struct zs_pool *pool, unsigned long handle,
1295 enum zs_mapmode mm)
1296{
1297 struct zspage *zspage;
1298 struct page *page;
1299 unsigned long obj, off;
1300 unsigned int obj_idx;
1301
1302 unsigned int class_idx;
1303 enum fullness_group fg;
1304 struct size_class *class;
1305 struct mapping_area *area;
1306 struct page *pages[2];
1307 void *ret;
1308
1309
1310
1311
1312
1313
1314 BUG_ON(in_interrupt());
1315
1316
1317 pin_tag(handle);
1318
1319 obj = handle_to_obj(handle);
1320 obj_to_location(obj, &page, &obj_idx);
1321 zspage = get_zspage(page);
1322
1323
1324 migrate_read_lock(zspage);
1325
1326 get_zspage_mapping(zspage, &class_idx, &fg);
1327 class = pool->size_class[class_idx];
1328 off = (class->size * obj_idx) & ~PAGE_MASK;
1329
1330 area = &get_cpu_var(zs_map_area);
1331 area->vm_mm = mm;
1332 if (off + class->size <= PAGE_SIZE) {
1333
1334 area->vm_addr = kmap_atomic(page);
1335 ret = area->vm_addr + off;
1336 goto out;
1337 }
1338
1339
1340 pages[0] = page;
1341 pages[1] = get_next_page(page);
1342 BUG_ON(!pages[1]);
1343
1344 ret = __zs_map_object(area, pages, off, class->size);
1345out:
1346 if (likely(!PageHugeObject(page)))
1347 ret += ZS_HANDLE_SIZE;
1348
1349 return ret;
1350}
1351EXPORT_SYMBOL_GPL(zs_map_object);
1352
1353void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
1354{
1355 struct zspage *zspage;
1356 struct page *page;
1357 unsigned long obj, off;
1358 unsigned int obj_idx;
1359
1360 unsigned int class_idx;
1361 enum fullness_group fg;
1362 struct size_class *class;
1363 struct mapping_area *area;
1364
1365 obj = handle_to_obj(handle);
1366 obj_to_location(obj, &page, &obj_idx);
1367 zspage = get_zspage(page);
1368 get_zspage_mapping(zspage, &class_idx, &fg);
1369 class = pool->size_class[class_idx];
1370 off = (class->size * obj_idx) & ~PAGE_MASK;
1371
1372 area = this_cpu_ptr(&zs_map_area);
1373 if (off + class->size <= PAGE_SIZE)
1374 kunmap_atomic(area->vm_addr);
1375 else {
1376 struct page *pages[2];
1377
1378 pages[0] = page;
1379 pages[1] = get_next_page(page);
1380 BUG_ON(!pages[1]);
1381
1382 __zs_unmap_object(area, pages, off, class->size);
1383 }
1384 put_cpu_var(zs_map_area);
1385
1386 migrate_read_unlock(zspage);
1387 unpin_tag(handle);
1388}
1389EXPORT_SYMBOL_GPL(zs_unmap_object);
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404size_t zs_huge_class_size(struct zs_pool *pool)
1405{
1406 return huge_class_size;
1407}
1408EXPORT_SYMBOL_GPL(zs_huge_class_size);
1409
1410static unsigned long obj_malloc(struct size_class *class,
1411 struct zspage *zspage, unsigned long handle)
1412{
1413 int i, nr_page, offset;
1414 unsigned long obj;
1415 struct link_free *link;
1416
1417 struct page *m_page;
1418 unsigned long m_offset;
1419 void *vaddr;
1420
1421 handle |= OBJ_ALLOCATED_TAG;
1422 obj = get_freeobj(zspage);
1423
1424 offset = obj * class->size;
1425 nr_page = offset >> PAGE_SHIFT;
1426 m_offset = offset & ~PAGE_MASK;
1427 m_page = get_first_page(zspage);
1428
1429 for (i = 0; i < nr_page; i++)
1430 m_page = get_next_page(m_page);
1431
1432 vaddr = kmap_atomic(m_page);
1433 link = (struct link_free *)vaddr + m_offset / sizeof(*link);
1434 set_freeobj(zspage, link->next >> OBJ_TAG_BITS);
1435 if (likely(!PageHugeObject(m_page)))
1436
1437 link->handle = handle;
1438 else
1439
1440 zspage->first_page->index = handle;
1441
1442 kunmap_atomic(vaddr);
1443 mod_zspage_inuse(zspage, 1);
1444 zs_stat_inc(class, OBJ_USED, 1);
1445
1446 obj = location_to_obj(m_page, obj);
1447
1448 return obj;
1449}
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
1463{
1464 unsigned long handle, obj;
1465 struct size_class *class;
1466 enum fullness_group newfg;
1467 struct zspage *zspage;
1468
1469 if (unlikely(!size || size > ZS_MAX_ALLOC_SIZE))
1470 return 0;
1471
1472 handle = cache_alloc_handle(pool, gfp);
1473 if (!handle)
1474 return 0;
1475
1476
1477 size += ZS_HANDLE_SIZE;
1478 class = pool->size_class[get_size_class_index(size)];
1479
1480 spin_lock(&class->lock);
1481 zspage = find_get_zspage(class);
1482 if (likely(zspage)) {
1483 obj = obj_malloc(class, zspage, handle);
1484
1485 fix_fullness_group(class, zspage);
1486 record_obj(handle, obj);
1487 spin_unlock(&class->lock);
1488
1489 return handle;
1490 }
1491
1492 spin_unlock(&class->lock);
1493
1494 zspage = alloc_zspage(pool, class, gfp);
1495 if (!zspage) {
1496 cache_free_handle(pool, handle);
1497 return 0;
1498 }
1499
1500 spin_lock(&class->lock);
1501 obj = obj_malloc(class, zspage, handle);
1502 newfg = get_fullness_group(class, zspage);
1503 insert_zspage(class, zspage, newfg);
1504 set_zspage_mapping(zspage, class->index, newfg);
1505 record_obj(handle, obj);
1506 atomic_long_add(class->pages_per_zspage,
1507 &pool->pages_allocated);
1508 zs_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage);
1509
1510
1511 SetZsPageMovable(pool, zspage);
1512 spin_unlock(&class->lock);
1513
1514 return handle;
1515}
1516EXPORT_SYMBOL_GPL(zs_malloc);
1517
1518static void obj_free(struct size_class *class, unsigned long obj)
1519{
1520 struct link_free *link;
1521 struct zspage *zspage;
1522 struct page *f_page;
1523 unsigned long f_offset;
1524 unsigned int f_objidx;
1525 void *vaddr;
1526
1527 obj &= ~OBJ_ALLOCATED_TAG;
1528 obj_to_location(obj, &f_page, &f_objidx);
1529 f_offset = (class->size * f_objidx) & ~PAGE_MASK;
1530 zspage = get_zspage(f_page);
1531
1532 vaddr = kmap_atomic(f_page);
1533
1534
1535 link = (struct link_free *)(vaddr + f_offset);
1536 link->next = get_freeobj(zspage) << OBJ_TAG_BITS;
1537 kunmap_atomic(vaddr);
1538 set_freeobj(zspage, f_objidx);
1539 mod_zspage_inuse(zspage, -1);
1540 zs_stat_dec(class, OBJ_USED, 1);
1541}
1542
1543void zs_free(struct zs_pool *pool, unsigned long handle)
1544{
1545 struct zspage *zspage;
1546 struct page *f_page;
1547 unsigned long obj;
1548 unsigned int f_objidx;
1549 int class_idx;
1550 struct size_class *class;
1551 enum fullness_group fullness;
1552 bool isolated;
1553
1554 if (unlikely(!handle))
1555 return;
1556
1557 pin_tag(handle);
1558 obj = handle_to_obj(handle);
1559 obj_to_location(obj, &f_page, &f_objidx);
1560 zspage = get_zspage(f_page);
1561
1562 migrate_read_lock(zspage);
1563
1564 get_zspage_mapping(zspage, &class_idx, &fullness);
1565 class = pool->size_class[class_idx];
1566
1567 spin_lock(&class->lock);
1568 obj_free(class, obj);
1569 fullness = fix_fullness_group(class, zspage);
1570 if (fullness != ZS_EMPTY) {
1571 migrate_read_unlock(zspage);
1572 goto out;
1573 }
1574
1575 isolated = is_zspage_isolated(zspage);
1576 migrate_read_unlock(zspage);
1577
1578 if (likely(!isolated))
1579 free_zspage(pool, class, zspage);
1580out:
1581
1582 spin_unlock(&class->lock);
1583 unpin_tag(handle);
1584 cache_free_handle(pool, handle);
1585}
1586EXPORT_SYMBOL_GPL(zs_free);
1587
1588static void zs_object_copy(struct size_class *class, unsigned long dst,
1589 unsigned long src)
1590{
1591 struct page *s_page, *d_page;
1592 unsigned int s_objidx, d_objidx;
1593 unsigned long s_off, d_off;
1594 void *s_addr, *d_addr;
1595 int s_size, d_size, size;
1596 int written = 0;
1597
1598 s_size = d_size = class->size;
1599
1600 obj_to_location(src, &s_page, &s_objidx);
1601 obj_to_location(dst, &d_page, &d_objidx);
1602
1603 s_off = (class->size * s_objidx) & ~PAGE_MASK;
1604 d_off = (class->size * d_objidx) & ~PAGE_MASK;
1605
1606 if (s_off + class->size > PAGE_SIZE)
1607 s_size = PAGE_SIZE - s_off;
1608
1609 if (d_off + class->size > PAGE_SIZE)
1610 d_size = PAGE_SIZE - d_off;
1611
1612 s_addr = kmap_atomic(s_page);
1613 d_addr = kmap_atomic(d_page);
1614
1615 while (1) {
1616 size = min(s_size, d_size);
1617 memcpy(d_addr + d_off, s_addr + s_off, size);
1618 written += size;
1619
1620 if (written == class->size)
1621 break;
1622
1623 s_off += size;
1624 s_size -= size;
1625 d_off += size;
1626 d_size -= size;
1627
1628 if (s_off >= PAGE_SIZE) {
1629 kunmap_atomic(d_addr);
1630 kunmap_atomic(s_addr);
1631 s_page = get_next_page(s_page);
1632 s_addr = kmap_atomic(s_page);
1633 d_addr = kmap_atomic(d_page);
1634 s_size = class->size - written;
1635 s_off = 0;
1636 }
1637
1638 if (d_off >= PAGE_SIZE) {
1639 kunmap_atomic(d_addr);
1640 d_page = get_next_page(d_page);
1641 d_addr = kmap_atomic(d_page);
1642 d_size = class->size - written;
1643 d_off = 0;
1644 }
1645 }
1646
1647 kunmap_atomic(d_addr);
1648 kunmap_atomic(s_addr);
1649}
1650
1651
1652
1653
1654
1655static unsigned long find_alloced_obj(struct size_class *class,
1656 struct page *page, int *obj_idx)
1657{
1658 unsigned long head;
1659 int offset = 0;
1660 int index = *obj_idx;
1661 unsigned long handle = 0;
1662 void *addr = kmap_atomic(page);
1663
1664 offset = get_first_obj_offset(page);
1665 offset += class->size * index;
1666
1667 while (offset < PAGE_SIZE) {
1668 head = obj_to_head(page, addr + offset);
1669 if (head & OBJ_ALLOCATED_TAG) {
1670 handle = head & ~OBJ_ALLOCATED_TAG;
1671 if (trypin_tag(handle))
1672 break;
1673 handle = 0;
1674 }
1675
1676 offset += class->size;
1677 index++;
1678 }
1679
1680 kunmap_atomic(addr);
1681
1682 *obj_idx = index;
1683
1684 return handle;
1685}
1686
1687struct zs_compact_control {
1688
1689 struct page *s_page;
1690
1691
1692 struct page *d_page;
1693
1694
1695 int obj_idx;
1696};
1697
1698static int migrate_zspage(struct zs_pool *pool, struct size_class *class,
1699 struct zs_compact_control *cc)
1700{
1701 unsigned long used_obj, free_obj;
1702 unsigned long handle;
1703 struct page *s_page = cc->s_page;
1704 struct page *d_page = cc->d_page;
1705 int obj_idx = cc->obj_idx;
1706 int ret = 0;
1707
1708 while (1) {
1709 handle = find_alloced_obj(class, s_page, &obj_idx);
1710 if (!handle) {
1711 s_page = get_next_page(s_page);
1712 if (!s_page)
1713 break;
1714 obj_idx = 0;
1715 continue;
1716 }
1717
1718
1719 if (zspage_full(class, get_zspage(d_page))) {
1720 unpin_tag(handle);
1721 ret = -ENOMEM;
1722 break;
1723 }
1724
1725 used_obj = handle_to_obj(handle);
1726 free_obj = obj_malloc(class, get_zspage(d_page), handle);
1727 zs_object_copy(class, free_obj, used_obj);
1728 obj_idx++;
1729
1730
1731
1732
1733
1734
1735 free_obj |= BIT(HANDLE_PIN_BIT);
1736 record_obj(handle, free_obj);
1737 unpin_tag(handle);
1738 obj_free(class, used_obj);
1739 }
1740
1741
1742 cc->s_page = s_page;
1743 cc->obj_idx = obj_idx;
1744
1745 return ret;
1746}
1747
1748static struct zspage *isolate_zspage(struct size_class *class, bool source)
1749{
1750 int i;
1751 struct zspage *zspage;
1752 enum fullness_group fg[2] = {ZS_ALMOST_EMPTY, ZS_ALMOST_FULL};
1753
1754 if (!source) {
1755 fg[0] = ZS_ALMOST_FULL;
1756 fg[1] = ZS_ALMOST_EMPTY;
1757 }
1758
1759 for (i = 0; i < 2; i++) {
1760 zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],
1761 struct zspage, list);
1762 if (zspage) {
1763 VM_BUG_ON(is_zspage_isolated(zspage));
1764 remove_zspage(class, zspage, fg[i]);
1765 return zspage;
1766 }
1767 }
1768
1769 return zspage;
1770}
1771
1772
1773
1774
1775
1776
1777
1778
1779static enum fullness_group putback_zspage(struct size_class *class,
1780 struct zspage *zspage)
1781{
1782 enum fullness_group fullness;
1783
1784 VM_BUG_ON(is_zspage_isolated(zspage));
1785
1786 fullness = get_fullness_group(class, zspage);
1787 insert_zspage(class, zspage, fullness);
1788 set_zspage_mapping(zspage, class->index, fullness);
1789
1790 return fullness;
1791}
1792
1793#ifdef CONFIG_COMPACTION
1794
1795
1796
1797
1798static void lock_zspage(struct zspage *zspage)
1799{
1800 struct page *page = get_first_page(zspage);
1801
1802 do {
1803 lock_page(page);
1804 } while ((page = get_next_page(page)) != NULL);
1805}
1806
1807static int zs_init_fs_context(struct fs_context *fc)
1808{
1809 return init_pseudo(fc, ZSMALLOC_MAGIC) ? 0 : -ENOMEM;
1810}
1811
1812static struct file_system_type zsmalloc_fs = {
1813 .name = "zsmalloc",
1814 .init_fs_context = zs_init_fs_context,
1815 .kill_sb = kill_anon_super,
1816};
1817
1818static int zsmalloc_mount(void)
1819{
1820 int ret = 0;
1821
1822 zsmalloc_mnt = kern_mount(&zsmalloc_fs);
1823 if (IS_ERR(zsmalloc_mnt))
1824 ret = PTR_ERR(zsmalloc_mnt);
1825
1826 return ret;
1827}
1828
1829static void zsmalloc_unmount(void)
1830{
1831 kern_unmount(zsmalloc_mnt);
1832}
1833
1834static void migrate_lock_init(struct zspage *zspage)
1835{
1836 rwlock_init(&zspage->lock);
1837}
1838
1839static void migrate_read_lock(struct zspage *zspage)
1840{
1841 read_lock(&zspage->lock);
1842}
1843
1844static void migrate_read_unlock(struct zspage *zspage)
1845{
1846 read_unlock(&zspage->lock);
1847}
1848
1849static void migrate_write_lock(struct zspage *zspage)
1850{
1851 write_lock(&zspage->lock);
1852}
1853
1854static void migrate_write_unlock(struct zspage *zspage)
1855{
1856 write_unlock(&zspage->lock);
1857}
1858
1859
1860static void inc_zspage_isolation(struct zspage *zspage)
1861{
1862 zspage->isolated++;
1863}
1864
1865static void dec_zspage_isolation(struct zspage *zspage)
1866{
1867 zspage->isolated--;
1868}
1869
1870static void putback_zspage_deferred(struct zs_pool *pool,
1871 struct size_class *class,
1872 struct zspage *zspage)
1873{
1874 enum fullness_group fg;
1875
1876 fg = putback_zspage(class, zspage);
1877 if (fg == ZS_EMPTY)
1878 schedule_work(&pool->free_work);
1879
1880}
1881
1882static inline void zs_pool_dec_isolated(struct zs_pool *pool)
1883{
1884 VM_BUG_ON(atomic_long_read(&pool->isolated_pages) <= 0);
1885 atomic_long_dec(&pool->isolated_pages);
1886
1887
1888
1889
1890
1891 if (atomic_long_read(&pool->isolated_pages) == 0 && pool->destroying)
1892 wake_up_all(&pool->migration_wait);
1893}
1894
1895static void replace_sub_page(struct size_class *class, struct zspage *zspage,
1896 struct page *newpage, struct page *oldpage)
1897{
1898 struct page *page;
1899 struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE] = {NULL, };
1900 int idx = 0;
1901
1902 page = get_first_page(zspage);
1903 do {
1904 if (page == oldpage)
1905 pages[idx] = newpage;
1906 else
1907 pages[idx] = page;
1908 idx++;
1909 } while ((page = get_next_page(page)) != NULL);
1910
1911 create_page_chain(class, zspage, pages);
1912 set_first_obj_offset(newpage, get_first_obj_offset(oldpage));
1913 if (unlikely(PageHugeObject(oldpage)))
1914 newpage->index = oldpage->index;
1915 __SetPageMovable(newpage, page_mapping(oldpage));
1916}
1917
1918static bool zs_page_isolate(struct page *page, isolate_mode_t mode)
1919{
1920 struct zs_pool *pool;
1921 struct size_class *class;
1922 int class_idx;
1923 enum fullness_group fullness;
1924 struct zspage *zspage;
1925 struct address_space *mapping;
1926
1927
1928
1929
1930
1931 VM_BUG_ON_PAGE(!PageMovable(page), page);
1932 VM_BUG_ON_PAGE(PageIsolated(page), page);
1933
1934 zspage = get_zspage(page);
1935
1936
1937
1938
1939
1940
1941 get_zspage_mapping(zspage, &class_idx, &fullness);
1942 mapping = page_mapping(page);
1943 pool = mapping->private_data;
1944 class = pool->size_class[class_idx];
1945
1946 spin_lock(&class->lock);
1947 if (get_zspage_inuse(zspage) == 0) {
1948 spin_unlock(&class->lock);
1949 return false;
1950 }
1951
1952
1953 if (list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1954 spin_unlock(&class->lock);
1955 return false;
1956 }
1957
1958
1959
1960
1961
1962 if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
1963 get_zspage_mapping(zspage, &class_idx, &fullness);
1964 atomic_long_inc(&pool->isolated_pages);
1965 remove_zspage(class, zspage, fullness);
1966 }
1967
1968 inc_zspage_isolation(zspage);
1969 spin_unlock(&class->lock);
1970
1971 return true;
1972}
1973
1974static int zs_page_migrate(struct address_space *mapping, struct page *newpage,
1975 struct page *page, enum migrate_mode mode)
1976{
1977 struct zs_pool *pool;
1978 struct size_class *class;
1979 int class_idx;
1980 enum fullness_group fullness;
1981 struct zspage *zspage;
1982 struct page *dummy;
1983 void *s_addr, *d_addr, *addr;
1984 int offset, pos;
1985 unsigned long handle, head;
1986 unsigned long old_obj, new_obj;
1987 unsigned int obj_idx;
1988 int ret = -EAGAIN;
1989
1990
1991
1992
1993
1994
1995 if (mode == MIGRATE_SYNC_NO_COPY)
1996 return -EINVAL;
1997
1998 VM_BUG_ON_PAGE(!PageMovable(page), page);
1999 VM_BUG_ON_PAGE(!PageIsolated(page), page);
2000
2001 zspage = get_zspage(page);
2002
2003
2004 migrate_write_lock(zspage);
2005 get_zspage_mapping(zspage, &class_idx, &fullness);
2006 pool = mapping->private_data;
2007 class = pool->size_class[class_idx];
2008 offset = get_first_obj_offset(page);
2009
2010 spin_lock(&class->lock);
2011 if (!get_zspage_inuse(zspage)) {
2012
2013
2014
2015
2016 offset = PAGE_SIZE;
2017 }
2018
2019 pos = offset;
2020 s_addr = kmap_atomic(page);
2021 while (pos < PAGE_SIZE) {
2022 head = obj_to_head(page, s_addr + pos);
2023 if (head & OBJ_ALLOCATED_TAG) {
2024 handle = head & ~OBJ_ALLOCATED_TAG;
2025 if (!trypin_tag(handle))
2026 goto unpin_objects;
2027 }
2028 pos += class->size;
2029 }
2030
2031
2032
2033
2034 d_addr = kmap_atomic(newpage);
2035 memcpy(d_addr, s_addr, PAGE_SIZE);
2036 kunmap_atomic(d_addr);
2037
2038 for (addr = s_addr + offset; addr < s_addr + pos;
2039 addr += class->size) {
2040 head = obj_to_head(page, addr);
2041 if (head & OBJ_ALLOCATED_TAG) {
2042 handle = head & ~OBJ_ALLOCATED_TAG;
2043 if (!testpin_tag(handle))
2044 BUG();
2045
2046 old_obj = handle_to_obj(handle);
2047 obj_to_location(old_obj, &dummy, &obj_idx);
2048 new_obj = (unsigned long)location_to_obj(newpage,
2049 obj_idx);
2050 new_obj |= BIT(HANDLE_PIN_BIT);
2051 record_obj(handle, new_obj);
2052 }
2053 }
2054
2055 replace_sub_page(class, zspage, newpage, page);
2056 get_page(newpage);
2057
2058 dec_zspage_isolation(zspage);
2059
2060
2061
2062
2063
2064 if (!is_zspage_isolated(zspage)) {
2065
2066
2067
2068
2069
2070
2071 putback_zspage_deferred(pool, class, zspage);
2072 zs_pool_dec_isolated(pool);
2073 }
2074
2075 reset_page(page);
2076 put_page(page);
2077 page = newpage;
2078
2079 ret = MIGRATEPAGE_SUCCESS;
2080unpin_objects:
2081 for (addr = s_addr + offset; addr < s_addr + pos;
2082 addr += class->size) {
2083 head = obj_to_head(page, addr);
2084 if (head & OBJ_ALLOCATED_TAG) {
2085 handle = head & ~OBJ_ALLOCATED_TAG;
2086 if (!testpin_tag(handle))
2087 BUG();
2088 unpin_tag(handle);
2089 }
2090 }
2091 kunmap_atomic(s_addr);
2092 spin_unlock(&class->lock);
2093 migrate_write_unlock(zspage);
2094
2095 return ret;
2096}
2097
2098static void zs_page_putback(struct page *page)
2099{
2100 struct zs_pool *pool;
2101 struct size_class *class;
2102 int class_idx;
2103 enum fullness_group fg;
2104 struct address_space *mapping;
2105 struct zspage *zspage;
2106
2107 VM_BUG_ON_PAGE(!PageMovable(page), page);
2108 VM_BUG_ON_PAGE(!PageIsolated(page), page);
2109
2110 zspage = get_zspage(page);
2111 get_zspage_mapping(zspage, &class_idx, &fg);
2112 mapping = page_mapping(page);
2113 pool = mapping->private_data;
2114 class = pool->size_class[class_idx];
2115
2116 spin_lock(&class->lock);
2117 dec_zspage_isolation(zspage);
2118 if (!is_zspage_isolated(zspage)) {
2119
2120
2121
2122
2123 putback_zspage_deferred(pool, class, zspage);
2124 zs_pool_dec_isolated(pool);
2125 }
2126 spin_unlock(&class->lock);
2127}
2128
2129static const struct address_space_operations zsmalloc_aops = {
2130 .isolate_page = zs_page_isolate,
2131 .migratepage = zs_page_migrate,
2132 .putback_page = zs_page_putback,
2133};
2134
2135static int zs_register_migration(struct zs_pool *pool)
2136{
2137 pool->inode = alloc_anon_inode(zsmalloc_mnt->mnt_sb);
2138 if (IS_ERR(pool->inode)) {
2139 pool->inode = NULL;
2140 return 1;
2141 }
2142
2143 pool->inode->i_mapping->private_data = pool;
2144 pool->inode->i_mapping->a_ops = &zsmalloc_aops;
2145 return 0;
2146}
2147
2148static bool pool_isolated_are_drained(struct zs_pool *pool)
2149{
2150 return atomic_long_read(&pool->isolated_pages) == 0;
2151}
2152
2153
2154static void wait_for_isolated_drain(struct zs_pool *pool)
2155{
2156
2157
2158
2159
2160
2161
2162
2163 wait_event(pool->migration_wait,
2164 pool_isolated_are_drained(pool));
2165}
2166
2167static void zs_unregister_migration(struct zs_pool *pool)
2168{
2169 pool->destroying = true;
2170
2171
2172
2173
2174
2175
2176 smp_mb();
2177 wait_for_isolated_drain(pool);
2178 flush_work(&pool->free_work);
2179 iput(pool->inode);
2180}
2181
2182
2183
2184
2185
2186static void async_free_zspage(struct work_struct *work)
2187{
2188 int i;
2189 struct size_class *class;
2190 unsigned int class_idx;
2191 enum fullness_group fullness;
2192 struct zspage *zspage, *tmp;
2193 LIST_HEAD(free_pages);
2194 struct zs_pool *pool = container_of(work, struct zs_pool,
2195 free_work);
2196
2197 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
2198 class = pool->size_class[i];
2199 if (class->index != i)
2200 continue;
2201
2202 spin_lock(&class->lock);
2203 list_splice_init(&class->fullness_list[ZS_EMPTY], &free_pages);
2204 spin_unlock(&class->lock);
2205 }
2206
2207
2208 list_for_each_entry_safe(zspage, tmp, &free_pages, list) {
2209 list_del(&zspage->list);
2210 lock_zspage(zspage);
2211
2212 get_zspage_mapping(zspage, &class_idx, &fullness);
2213 VM_BUG_ON(fullness != ZS_EMPTY);
2214 class = pool->size_class[class_idx];
2215 spin_lock(&class->lock);
2216 __free_zspage(pool, pool->size_class[class_idx], zspage);
2217 spin_unlock(&class->lock);
2218 }
2219};
2220
2221static void kick_deferred_free(struct zs_pool *pool)
2222{
2223 schedule_work(&pool->free_work);
2224}
2225
2226static void init_deferred_free(struct zs_pool *pool)
2227{
2228 INIT_WORK(&pool->free_work, async_free_zspage);
2229}
2230
2231static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)
2232{
2233 struct page *page = get_first_page(zspage);
2234
2235 do {
2236 WARN_ON(!trylock_page(page));
2237 __SetPageMovable(page, pool->inode->i_mapping);
2238 unlock_page(page);
2239 } while ((page = get_next_page(page)) != NULL);
2240}
2241#endif
2242
2243
2244
2245
2246
2247
2248static unsigned long zs_can_compact(struct size_class *class)
2249{
2250 unsigned long obj_wasted;
2251 unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
2252 unsigned long obj_used = zs_stat_get(class, OBJ_USED);
2253
2254 if (obj_allocated <= obj_used)
2255 return 0;
2256
2257 obj_wasted = obj_allocated - obj_used;
2258 obj_wasted /= class->objs_per_zspage;
2259
2260 return obj_wasted * class->pages_per_zspage;
2261}
2262
2263static void __zs_compact(struct zs_pool *pool, struct size_class *class)
2264{
2265 struct zs_compact_control cc;
2266 struct zspage *src_zspage;
2267 struct zspage *dst_zspage = NULL;
2268
2269 spin_lock(&class->lock);
2270 while ((src_zspage = isolate_zspage(class, true))) {
2271
2272 if (!zs_can_compact(class))
2273 break;
2274
2275 cc.obj_idx = 0;
2276 cc.s_page = get_first_page(src_zspage);
2277
2278 while ((dst_zspage = isolate_zspage(class, false))) {
2279 cc.d_page = get_first_page(dst_zspage);
2280
2281
2282
2283
2284 if (!migrate_zspage(pool, class, &cc))
2285 break;
2286
2287 putback_zspage(class, dst_zspage);
2288 }
2289
2290
2291 if (dst_zspage == NULL)
2292 break;
2293
2294 putback_zspage(class, dst_zspage);
2295 if (putback_zspage(class, src_zspage) == ZS_EMPTY) {
2296 free_zspage(pool, class, src_zspage);
2297 pool->stats.pages_compacted += class->pages_per_zspage;
2298 }
2299 spin_unlock(&class->lock);
2300 cond_resched();
2301 spin_lock(&class->lock);
2302 }
2303
2304 if (src_zspage)
2305 putback_zspage(class, src_zspage);
2306
2307 spin_unlock(&class->lock);
2308}
2309
2310unsigned long zs_compact(struct zs_pool *pool)
2311{
2312 int i;
2313 struct size_class *class;
2314
2315 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2316 class = pool->size_class[i];
2317 if (!class)
2318 continue;
2319 if (class->index != i)
2320 continue;
2321 __zs_compact(pool, class);
2322 }
2323
2324 return pool->stats.pages_compacted;
2325}
2326EXPORT_SYMBOL_GPL(zs_compact);
2327
2328void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats)
2329{
2330 memcpy(stats, &pool->stats, sizeof(struct zs_pool_stats));
2331}
2332EXPORT_SYMBOL_GPL(zs_pool_stats);
2333
2334static unsigned long zs_shrinker_scan(struct shrinker *shrinker,
2335 struct shrink_control *sc)
2336{
2337 unsigned long pages_freed;
2338 struct zs_pool *pool = container_of(shrinker, struct zs_pool,
2339 shrinker);
2340
2341 pages_freed = pool->stats.pages_compacted;
2342
2343
2344
2345
2346
2347 pages_freed = zs_compact(pool) - pages_freed;
2348
2349 return pages_freed ? pages_freed : SHRINK_STOP;
2350}
2351
2352static unsigned long zs_shrinker_count(struct shrinker *shrinker,
2353 struct shrink_control *sc)
2354{
2355 int i;
2356 struct size_class *class;
2357 unsigned long pages_to_free = 0;
2358 struct zs_pool *pool = container_of(shrinker, struct zs_pool,
2359 shrinker);
2360
2361 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2362 class = pool->size_class[i];
2363 if (!class)
2364 continue;
2365 if (class->index != i)
2366 continue;
2367
2368 pages_to_free += zs_can_compact(class);
2369 }
2370
2371 return pages_to_free;
2372}
2373
2374static void zs_unregister_shrinker(struct zs_pool *pool)
2375{
2376 unregister_shrinker(&pool->shrinker);
2377}
2378
2379static int zs_register_shrinker(struct zs_pool *pool)
2380{
2381 pool->shrinker.scan_objects = zs_shrinker_scan;
2382 pool->shrinker.count_objects = zs_shrinker_count;
2383 pool->shrinker.batch = 0;
2384 pool->shrinker.seeks = DEFAULT_SEEKS;
2385
2386 return register_shrinker(&pool->shrinker);
2387}
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399struct zs_pool *zs_create_pool(const char *name)
2400{
2401 int i;
2402 struct zs_pool *pool;
2403 struct size_class *prev_class = NULL;
2404
2405 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
2406 if (!pool)
2407 return NULL;
2408
2409 init_deferred_free(pool);
2410
2411 pool->name = kstrdup(name, GFP_KERNEL);
2412 if (!pool->name)
2413 goto err;
2414
2415#ifdef CONFIG_COMPACTION
2416 init_waitqueue_head(&pool->migration_wait);
2417#endif
2418
2419 if (create_cache(pool))
2420 goto err;
2421
2422
2423
2424
2425
2426 for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
2427 int size;
2428 int pages_per_zspage;
2429 int objs_per_zspage;
2430 struct size_class *class;
2431 int fullness = 0;
2432
2433 size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA;
2434 if (size > ZS_MAX_ALLOC_SIZE)
2435 size = ZS_MAX_ALLOC_SIZE;
2436 pages_per_zspage = get_pages_per_zspage(size);
2437 objs_per_zspage = pages_per_zspage * PAGE_SIZE / size;
2438
2439
2440
2441
2442
2443
2444
2445 if (pages_per_zspage != 1 && objs_per_zspage != 1 &&
2446 !huge_class_size) {
2447 huge_class_size = size;
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457 huge_class_size -= (ZS_HANDLE_SIZE - 1);
2458 }
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469 if (prev_class) {
2470 if (can_merge(prev_class, pages_per_zspage, objs_per_zspage)) {
2471 pool->size_class[i] = prev_class;
2472 continue;
2473 }
2474 }
2475
2476 class = kzalloc(sizeof(struct size_class), GFP_KERNEL);
2477 if (!class)
2478 goto err;
2479
2480 class->size = size;
2481 class->index = i;
2482 class->pages_per_zspage = pages_per_zspage;
2483 class->objs_per_zspage = objs_per_zspage;
2484 spin_lock_init(&class->lock);
2485 pool->size_class[i] = class;
2486 for (fullness = ZS_EMPTY; fullness < NR_ZS_FULLNESS;
2487 fullness++)
2488 INIT_LIST_HEAD(&class->fullness_list[fullness]);
2489
2490 prev_class = class;
2491 }
2492
2493
2494 zs_pool_stat_create(pool, name);
2495
2496 if (zs_register_migration(pool))
2497 goto err;
2498
2499
2500
2501
2502
2503
2504
2505 zs_register_shrinker(pool);
2506
2507 return pool;
2508
2509err:
2510 zs_destroy_pool(pool);
2511 return NULL;
2512}
2513EXPORT_SYMBOL_GPL(zs_create_pool);
2514
2515void zs_destroy_pool(struct zs_pool *pool)
2516{
2517 int i;
2518
2519 zs_unregister_shrinker(pool);
2520 zs_unregister_migration(pool);
2521 zs_pool_stat_destroy(pool);
2522
2523 for (i = 0; i < ZS_SIZE_CLASSES; i++) {
2524 int fg;
2525 struct size_class *class = pool->size_class[i];
2526
2527 if (!class)
2528 continue;
2529
2530 if (class->index != i)
2531 continue;
2532
2533 for (fg = ZS_EMPTY; fg < NR_ZS_FULLNESS; fg++) {
2534 if (!list_empty(&class->fullness_list[fg])) {
2535 pr_info("Freeing non-empty class with size %db, fullness group %d\n",
2536 class->size, fg);
2537 }
2538 }
2539 kfree(class);
2540 }
2541
2542 destroy_cache(pool);
2543 kfree(pool->name);
2544 kfree(pool);
2545}
2546EXPORT_SYMBOL_GPL(zs_destroy_pool);
2547
2548static int __init zs_init(void)
2549{
2550 int ret;
2551
2552 ret = zsmalloc_mount();
2553 if (ret)
2554 goto out;
2555
2556 ret = cpuhp_setup_state(CPUHP_MM_ZS_PREPARE, "mm/zsmalloc:prepare",
2557 zs_cpu_prepare, zs_cpu_dead);
2558 if (ret)
2559 goto hp_setup_fail;
2560
2561#ifdef CONFIG_ZPOOL
2562 zpool_register_driver(&zs_zpool_driver);
2563#endif
2564
2565 zs_stat_init();
2566
2567 return 0;
2568
2569hp_setup_fail:
2570 zsmalloc_unmount();
2571out:
2572 return ret;
2573}
2574
2575static void __exit zs_exit(void)
2576{
2577#ifdef CONFIG_ZPOOL
2578 zpool_unregister_driver(&zs_zpool_driver);
2579#endif
2580 zsmalloc_unmount();
2581 cpuhp_remove_state(CPUHP_MM_ZS_PREPARE);
2582
2583 zs_stat_exit();
2584}
2585
2586module_init(zs_init);
2587module_exit(zs_exit);
2588
2589MODULE_LICENSE("Dual BSD/GPL");
2590MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
2591