1
2
3
4
5
6
7#ifndef __EROFS_INTERNAL_H
8#define __EROFS_INTERNAL_H
9
10#include <linux/fs.h>
11#include <linux/dcache.h>
12#include <linux/mm.h>
13#include <linux/pagemap.h>
14#include <linux/bio.h>
15#include <linux/buffer_head.h>
16#include <linux/magic.h>
17#include <linux/slab.h>
18#include <linux/vmalloc.h>
19#include <linux/iomap.h>
20#include "erofs_fs.h"
21
22
23#undef pr_fmt
24#define pr_fmt(fmt) "erofs: " fmt
25
26__printf(3, 4) void _erofs_err(struct super_block *sb,
27 const char *function, const char *fmt, ...);
28#define erofs_err(sb, fmt, ...) \
29 _erofs_err(sb, __func__, fmt "\n", ##__VA_ARGS__)
30__printf(3, 4) void _erofs_info(struct super_block *sb,
31 const char *function, const char *fmt, ...);
32#define erofs_info(sb, fmt, ...) \
33 _erofs_info(sb, __func__, fmt "\n", ##__VA_ARGS__)
34#ifdef CONFIG_EROFS_FS_DEBUG
35#define erofs_dbg(x, ...) pr_debug(x "\n", ##__VA_ARGS__)
36#define DBG_BUGON BUG_ON
37#else
38#define erofs_dbg(x, ...) ((void)0)
39#define DBG_BUGON(x) ((void)(x))
40#endif
41
42
43#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1
44
45typedef u64 erofs_nid_t;
46typedef u64 erofs_off_t;
47
48typedef u32 erofs_blk_t;
49
50struct erofs_device_info {
51 char *path;
52 struct block_device *bdev;
53 struct dax_device *dax_dev;
54 u64 dax_part_off;
55
56 u32 blocks;
57 u32 mapped_blkaddr;
58};
59
60enum {
61 EROFS_SYNC_DECOMPRESS_AUTO,
62 EROFS_SYNC_DECOMPRESS_FORCE_ON,
63 EROFS_SYNC_DECOMPRESS_FORCE_OFF
64};
65
66struct erofs_mount_opts {
67#ifdef CONFIG_EROFS_FS_ZIP
68
69 unsigned char cache_strategy;
70
71 unsigned int sync_decompress;
72
73
74 unsigned int max_sync_decompress_pages;
75#endif
76 unsigned int mount_opt;
77};
78
79struct erofs_dev_context {
80 struct idr tree;
81 struct rw_semaphore rwsem;
82
83 unsigned int extra_devices;
84};
85
86struct erofs_fs_context {
87 struct erofs_mount_opts opt;
88 struct erofs_dev_context *devs;
89};
90
91
92struct erofs_sb_lz4_info {
93
94 u16 max_distance_pages;
95
96 u16 max_pclusterblks;
97};
98
99struct erofs_sb_info {
100 struct erofs_mount_opts opt;
101#ifdef CONFIG_EROFS_FS_ZIP
102
103 struct list_head list;
104 struct mutex umount_mutex;
105
106
107 struct xarray managed_pslots;
108
109 unsigned int shrinker_run_no;
110 u16 available_compr_algs;
111
112
113 struct inode *managed_cache;
114
115 struct erofs_sb_lz4_info lz4;
116#endif
117 struct erofs_dev_context *devs;
118 struct dax_device *dax_dev;
119 u64 dax_part_off;
120 u64 total_blocks;
121 u32 primarydevice_blocks;
122
123 u32 meta_blkaddr;
124#ifdef CONFIG_EROFS_FS_XATTR
125 u32 xattr_blkaddr;
126#endif
127 u16 device_id_mask;
128
129
130 unsigned char islotbits;
131
132 u32 sb_size;
133 u32 build_time_nsec;
134 u64 build_time;
135
136
137 erofs_nid_t root_nid;
138
139 u64 inos;
140
141 u8 uuid[16];
142 u8 volume_name[16];
143 u32 feature_compat;
144 u32 feature_incompat;
145
146
147 struct kobject s_kobj;
148 struct completion s_kobj_unregister;
149};
150
151#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
152#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)
153
154
155#define EROFS_MOUNT_XATTR_USER 0x00000010
156#define EROFS_MOUNT_POSIX_ACL 0x00000020
157#define EROFS_MOUNT_DAX_ALWAYS 0x00000040
158#define EROFS_MOUNT_DAX_NEVER 0x00000080
159
160#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option)
161#define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option)
162#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option)
163
164enum {
165 EROFS_ZIP_CACHE_DISABLED,
166 EROFS_ZIP_CACHE_READAHEAD,
167 EROFS_ZIP_CACHE_READAROUND
168};
169
170#ifdef CONFIG_EROFS_FS_ZIP
171#define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL)
172
173
174struct erofs_workgroup {
175
176 pgoff_t index;
177
178
179 atomic_t refcount;
180};
181
182#if defined(CONFIG_SMP)
183static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp,
184 int val)
185{
186 preempt_disable();
187 if (val != atomic_cmpxchg(&grp->refcount, val, EROFS_LOCKED_MAGIC)) {
188 preempt_enable();
189 return false;
190 }
191 return true;
192}
193
194static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp,
195 int orig_val)
196{
197
198
199
200
201 smp_mb();
202 atomic_set(&grp->refcount, orig_val);
203 preempt_enable();
204}
205
206static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
207{
208 return atomic_cond_read_relaxed(&grp->refcount,
209 VAL != EROFS_LOCKED_MAGIC);
210}
211#else
212static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp,
213 int val)
214{
215 preempt_disable();
216
217 if (val != atomic_read(&grp->refcount)) {
218 preempt_enable();
219 return false;
220 }
221 return true;
222}
223
224static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp,
225 int orig_val)
226{
227 preempt_enable();
228}
229
230static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
231{
232 int v = atomic_read(&grp->refcount);
233
234
235 DBG_BUGON(v == EROFS_LOCKED_MAGIC);
236 return v;
237}
238#endif
239#endif
240
241
242#define LOG_BLOCK_SIZE PAGE_SHIFT
243
244#undef LOG_SECTORS_PER_BLOCK
245#define LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9)
246
247#undef SECTORS_PER_BLOCK
248#define SECTORS_PER_BLOCK (1 << SECTORS_PER_BLOCK)
249
250#define EROFS_BLKSIZ (1 << LOG_BLOCK_SIZE)
251
252#if (EROFS_BLKSIZ % 4096 || !EROFS_BLKSIZ)
253#error erofs cannot be used in this platform
254#endif
255
256enum erofs_kmap_type {
257 EROFS_NO_KMAP,
258 EROFS_KMAP,
259 EROFS_KMAP_ATOMIC,
260};
261
262struct erofs_buf {
263 struct page *page;
264 void *base;
265 enum erofs_kmap_type kmap_type;
266};
267#define __EROFS_BUF_INITIALIZER ((struct erofs_buf){ .page = NULL })
268
269#define ROOT_NID(sb) ((sb)->root_nid)
270
271#define erofs_blknr(addr) ((addr) / EROFS_BLKSIZ)
272#define erofs_blkoff(addr) ((addr) % EROFS_BLKSIZ)
273#define blknr_to_addr(nr) ((erofs_off_t)(nr) * EROFS_BLKSIZ)
274
275static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid)
276{
277 return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits);
278}
279
280#define EROFS_FEATURE_FUNCS(name, compat, feature) \
281static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
282{ \
283 return sbi->feature_##compat & EROFS_FEATURE_##feature; \
284}
285
286EROFS_FEATURE_FUNCS(zero_padding, incompat, INCOMPAT_ZERO_PADDING)
287EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
288EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
289EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
290EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
291EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)
292EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)
293EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
294
295
296#define EROFS_I_EA_INITED_BIT 0
297#define EROFS_I_Z_INITED_BIT 1
298
299
300#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1)
301#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2)
302
303struct erofs_inode {
304 erofs_nid_t nid;
305
306
307 unsigned long flags;
308
309 unsigned char datalayout;
310 unsigned char inode_isize;
311 unsigned short xattr_isize;
312
313 unsigned int xattr_shared_count;
314 unsigned int *xattr_shared_xattrs;
315
316 union {
317 erofs_blk_t raw_blkaddr;
318 struct {
319 unsigned short chunkformat;
320 unsigned char chunkbits;
321 };
322#ifdef CONFIG_EROFS_FS_ZIP
323 struct {
324 unsigned short z_advise;
325 unsigned char z_algorithmtype[2];
326 unsigned char z_logical_clusterbits;
327 unsigned long z_tailextent_headlcn;
328 erofs_off_t z_idataoff;
329 unsigned short z_idata_size;
330 };
331#endif
332 };
333
334 struct inode vfs_inode;
335};
336
337#define EROFS_I(ptr) \
338 container_of(ptr, struct erofs_inode, vfs_inode)
339
340static inline unsigned long erofs_inode_datablocks(struct inode *inode)
341{
342
343 return DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
344}
345
346static inline unsigned int erofs_bitrange(unsigned int value, unsigned int bit,
347 unsigned int bits)
348{
349
350 return (value >> bit) & ((1 << bits) - 1);
351}
352
353
354static inline unsigned int erofs_inode_version(unsigned int value)
355{
356 return erofs_bitrange(value, EROFS_I_VERSION_BIT,
357 EROFS_I_VERSION_BITS);
358}
359
360static inline unsigned int erofs_inode_datalayout(unsigned int value)
361{
362 return erofs_bitrange(value, EROFS_I_DATALAYOUT_BIT,
363 EROFS_I_DATALAYOUT_BITS);
364}
365
366
367
368
369
370static inline
371struct page *erofs_grab_cache_page_nowait(struct address_space *mapping,
372 pgoff_t index)
373{
374 return pagecache_get_page(mapping, index,
375 FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,
376 readahead_gfp_mask(mapping) & ~__GFP_RECLAIM);
377}
378
379extern const struct super_operations erofs_sops;
380
381extern const struct address_space_operations erofs_raw_access_aops;
382extern const struct address_space_operations z_erofs_aops;
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409enum {
410 BH_Encoded = BH_PrivateStart,
411 BH_FullMapped,
412};
413
414
415#define EROFS_MAP_MAPPED (1 << BH_Mapped)
416
417#define EROFS_MAP_META (1 << BH_Meta)
418
419#define EROFS_MAP_ENCODED (1 << BH_Encoded)
420
421#define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped)
422
423struct erofs_map_blocks {
424 struct erofs_buf buf;
425
426 erofs_off_t m_pa, m_la;
427 u64 m_plen, m_llen;
428
429 unsigned short m_deviceid;
430 char m_algorithmformat;
431 unsigned int m_flags;
432};
433
434
435#define EROFS_GET_BLOCKS_RAW 0x0001
436
437
438
439
440#define EROFS_GET_BLOCKS_FIEMAP 0x0002
441
442#define EROFS_GET_BLOCKS_READMORE 0x0004
443
444#define EROFS_GET_BLOCKS_FINDTAIL 0x0008
445
446enum {
447 Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
448 Z_EROFS_COMPRESSION_RUNTIME_MAX
449};
450
451
452extern const struct iomap_ops z_erofs_iomap_report_ops;
453
454#ifdef CONFIG_EROFS_FS_ZIP
455int z_erofs_fill_inode(struct inode *inode);
456int z_erofs_map_blocks_iter(struct inode *inode,
457 struct erofs_map_blocks *map,
458 int flags);
459#else
460static inline int z_erofs_fill_inode(struct inode *inode) { return -EOPNOTSUPP; }
461static inline int z_erofs_map_blocks_iter(struct inode *inode,
462 struct erofs_map_blocks *map,
463 int flags)
464{
465 return -EOPNOTSUPP;
466}
467#endif
468
469struct erofs_map_dev {
470 struct block_device *m_bdev;
471 struct dax_device *m_daxdev;
472 u64 m_dax_part_off;
473
474 erofs_off_t m_pa;
475 unsigned int m_deviceid;
476};
477
478
479extern const struct file_operations erofs_file_fops;
480void erofs_unmap_metabuf(struct erofs_buf *buf);
481void erofs_put_metabuf(struct erofs_buf *buf);
482void *erofs_bread(struct erofs_buf *buf, struct inode *inode,
483 erofs_blk_t blkaddr, enum erofs_kmap_type type);
484void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb,
485 erofs_blk_t blkaddr, enum erofs_kmap_type type);
486int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *dev);
487int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
488 u64 start, u64 len);
489
490
491static inline unsigned long erofs_inode_hash(erofs_nid_t nid)
492{
493#if BITS_PER_LONG == 32
494 return (nid >> 32) ^ (nid & 0xffffffff);
495#else
496 return nid;
497#endif
498}
499
500extern const struct inode_operations erofs_generic_iops;
501extern const struct inode_operations erofs_symlink_iops;
502extern const struct inode_operations erofs_fast_symlink_iops;
503
504struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid, bool dir);
505int erofs_getattr(struct user_namespace *mnt_userns, const struct path *path,
506 struct kstat *stat, u32 request_mask,
507 unsigned int query_flags);
508
509
510extern const struct inode_operations erofs_dir_iops;
511
512int erofs_namei(struct inode *dir, struct qstr *name,
513 erofs_nid_t *nid, unsigned int *d_type);
514
515
516extern const struct file_operations erofs_dir_fops;
517
518static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count)
519{
520 int retried = 0;
521
522 while (1) {
523 void *p = vm_map_ram(pages, count, -1);
524
525
526 if (p || ++retried >= 3)
527 return p;
528 vm_unmap_aliases();
529 }
530 return NULL;
531}
532
533
534void *erofs_get_pcpubuf(unsigned int requiredpages);
535void erofs_put_pcpubuf(void *ptr);
536int erofs_pcpubuf_growsize(unsigned int nrpages);
537void erofs_pcpubuf_init(void);
538void erofs_pcpubuf_exit(void);
539
540
541int erofs_register_sysfs(struct super_block *sb);
542void erofs_unregister_sysfs(struct super_block *sb);
543int __init erofs_init_sysfs(void);
544void erofs_exit_sysfs(void);
545
546
547struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp);
548static inline void erofs_pagepool_add(struct page **pagepool,
549 struct page *page)
550{
551 set_page_private(page, (unsigned long)*pagepool);
552 *pagepool = page;
553}
554void erofs_release_pages(struct page **pagepool);
555
556#ifdef CONFIG_EROFS_FS_ZIP
557int erofs_workgroup_put(struct erofs_workgroup *grp);
558struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,
559 pgoff_t index);
560struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,
561 struct erofs_workgroup *grp);
562void erofs_workgroup_free_rcu(struct erofs_workgroup *grp);
563void erofs_shrinker_register(struct super_block *sb);
564void erofs_shrinker_unregister(struct super_block *sb);
565int __init erofs_init_shrinker(void);
566void erofs_exit_shrinker(void);
567int __init z_erofs_init_zip_subsystem(void);
568void z_erofs_exit_zip_subsystem(void);
569int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
570 struct erofs_workgroup *egrp);
571int erofs_try_to_free_cached_page(struct page *page);
572int z_erofs_load_lz4_config(struct super_block *sb,
573 struct erofs_super_block *dsb,
574 struct z_erofs_lz4_cfgs *lz4, int len);
575#else
576static inline void erofs_shrinker_register(struct super_block *sb) {}
577static inline void erofs_shrinker_unregister(struct super_block *sb) {}
578static inline int erofs_init_shrinker(void) { return 0; }
579static inline void erofs_exit_shrinker(void) {}
580static inline int z_erofs_init_zip_subsystem(void) { return 0; }
581static inline void z_erofs_exit_zip_subsystem(void) {}
582static inline int z_erofs_load_lz4_config(struct super_block *sb,
583 struct erofs_super_block *dsb,
584 struct z_erofs_lz4_cfgs *lz4, int len)
585{
586 if (lz4 || dsb->u1.lz4_max_distance) {
587 erofs_err(sb, "lz4 algorithm isn't enabled");
588 return -EINVAL;
589 }
590 return 0;
591}
592#endif
593
594#ifdef CONFIG_EROFS_FS_ZIP_LZMA
595int z_erofs_lzma_init(void);
596void z_erofs_lzma_exit(void);
597int z_erofs_load_lzma_config(struct super_block *sb,
598 struct erofs_super_block *dsb,
599 struct z_erofs_lzma_cfgs *lzma, int size);
600#else
601static inline int z_erofs_lzma_init(void) { return 0; }
602static inline int z_erofs_lzma_exit(void) { return 0; }
603static inline int z_erofs_load_lzma_config(struct super_block *sb,
604 struct erofs_super_block *dsb,
605 struct z_erofs_lzma_cfgs *lzma, int size) {
606 if (lzma) {
607 erofs_err(sb, "lzma algorithm isn't enabled");
608 return -EINVAL;
609 }
610 return 0;
611}
612#endif
613
614#define EFSCORRUPTED EUCLEAN
615
616#endif
617