1
2
3
4
5
6
7#ifndef __EROFS_INTERNAL_H
8#define __EROFS_INTERNAL_H
9
10#include <linux/fs.h>
11#include <linux/dcache.h>
12#include <linux/mm.h>
13#include <linux/pagemap.h>
14#include <linux/bio.h>
15#include <linux/buffer_head.h>
16#include <linux/magic.h>
17#include <linux/slab.h>
18#include <linux/vmalloc.h>
19#include "erofs_fs.h"
20
21
22#undef pr_fmt
23#define pr_fmt(fmt) "erofs: " fmt
24
25__printf(3, 4) void _erofs_err(struct super_block *sb,
26 const char *function, const char *fmt, ...);
27#define erofs_err(sb, fmt, ...) \
28 _erofs_err(sb, __func__, fmt "\n", ##__VA_ARGS__)
29__printf(3, 4) void _erofs_info(struct super_block *sb,
30 const char *function, const char *fmt, ...);
31#define erofs_info(sb, fmt, ...) \
32 _erofs_info(sb, __func__, fmt "\n", ##__VA_ARGS__)
33#ifdef CONFIG_EROFS_FS_DEBUG
34#define erofs_dbg(x, ...) pr_debug(x "\n", ##__VA_ARGS__)
35#define DBG_BUGON BUG_ON
36#else
37#define erofs_dbg(x, ...) ((void)0)
38#define DBG_BUGON(x) ((void)(x))
39#endif
40
41
42#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1
43
44typedef u64 erofs_nid_t;
45typedef u64 erofs_off_t;
46
47typedef u32 erofs_blk_t;
48
49struct erofs_fs_context {
50#ifdef CONFIG_EROFS_FS_ZIP
51
52 unsigned char cache_strategy;
53
54 bool readahead_sync_decompress;
55
56
57 unsigned int max_sync_decompress_pages;
58#endif
59 unsigned int mount_opt;
60};
61
62
63struct erofs_sb_lz4_info {
64
65 u16 max_distance_pages;
66
67 u16 max_pclusterblks;
68};
69
70struct erofs_sb_info {
71#ifdef CONFIG_EROFS_FS_ZIP
72
73 struct list_head list;
74 struct mutex umount_mutex;
75
76
77 struct xarray managed_pslots;
78
79 unsigned int shrinker_run_no;
80 u16 available_compr_algs;
81
82
83 struct inode *managed_cache;
84
85 struct erofs_sb_lz4_info lz4;
86#endif
87 u32 blocks;
88 u32 meta_blkaddr;
89#ifdef CONFIG_EROFS_FS_XATTR
90 u32 xattr_blkaddr;
91#endif
92
93
94 unsigned char islotbits;
95
96 u32 sb_size;
97 u32 build_time_nsec;
98 u64 build_time;
99
100
101 erofs_nid_t root_nid;
102
103 u64 inos;
104
105 u8 uuid[16];
106 u8 volume_name[16];
107 u32 feature_compat;
108 u32 feature_incompat;
109
110 struct erofs_fs_context ctx;
111};
112
113#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
114#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)
115
116
117#define EROFS_MOUNT_XATTR_USER 0x00000010
118#define EROFS_MOUNT_POSIX_ACL 0x00000020
119
120#define clear_opt(ctx, option) ((ctx)->mount_opt &= ~EROFS_MOUNT_##option)
121#define set_opt(ctx, option) ((ctx)->mount_opt |= EROFS_MOUNT_##option)
122#define test_opt(ctx, option) ((ctx)->mount_opt & EROFS_MOUNT_##option)
123
124enum {
125 EROFS_ZIP_CACHE_DISABLED,
126 EROFS_ZIP_CACHE_READAHEAD,
127 EROFS_ZIP_CACHE_READAROUND
128};
129
130#ifdef CONFIG_EROFS_FS_ZIP
131#define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL)
132
133
134struct erofs_workgroup {
135
136 pgoff_t index;
137
138
139 atomic_t refcount;
140};
141
142#if defined(CONFIG_SMP)
143static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp,
144 int val)
145{
146 preempt_disable();
147 if (val != atomic_cmpxchg(&grp->refcount, val, EROFS_LOCKED_MAGIC)) {
148 preempt_enable();
149 return false;
150 }
151 return true;
152}
153
154static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp,
155 int orig_val)
156{
157
158
159
160
161 smp_mb();
162 atomic_set(&grp->refcount, orig_val);
163 preempt_enable();
164}
165
166static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
167{
168 return atomic_cond_read_relaxed(&grp->refcount,
169 VAL != EROFS_LOCKED_MAGIC);
170}
171#else
172static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp,
173 int val)
174{
175 preempt_disable();
176
177 if (val != atomic_read(&grp->refcount)) {
178 preempt_enable();
179 return false;
180 }
181 return true;
182}
183
184static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp,
185 int orig_val)
186{
187 preempt_enable();
188}
189
190static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
191{
192 int v = atomic_read(&grp->refcount);
193
194
195 DBG_BUGON(v == EROFS_LOCKED_MAGIC);
196 return v;
197}
198#endif
199#endif
200
201
202#define LOG_BLOCK_SIZE PAGE_SHIFT
203
204#undef LOG_SECTORS_PER_BLOCK
205#define LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9)
206
207#undef SECTORS_PER_BLOCK
208#define SECTORS_PER_BLOCK (1 << SECTORS_PER_BLOCK)
209
210#define EROFS_BLKSIZ (1 << LOG_BLOCK_SIZE)
211
212#if (EROFS_BLKSIZ % 4096 || !EROFS_BLKSIZ)
213#error erofs cannot be used in this platform
214#endif
215
216#define ROOT_NID(sb) ((sb)->root_nid)
217
218#define erofs_blknr(addr) ((addr) / EROFS_BLKSIZ)
219#define erofs_blkoff(addr) ((addr) % EROFS_BLKSIZ)
220#define blknr_to_addr(nr) ((erofs_off_t)(nr) * EROFS_BLKSIZ)
221
222static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid)
223{
224 return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits);
225}
226
227#define EROFS_FEATURE_FUNCS(name, compat, feature) \
228static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
229{ \
230 return sbi->feature_##compat & EROFS_FEATURE_##feature; \
231}
232
233EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
234EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
235EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
236EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
237
238
239#define EROFS_I_EA_INITED_BIT 0
240#define EROFS_I_Z_INITED_BIT 1
241
242
243#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1)
244#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2)
245
246struct erofs_inode {
247 erofs_nid_t nid;
248
249
250 unsigned long flags;
251
252 unsigned char datalayout;
253 unsigned char inode_isize;
254 unsigned short xattr_isize;
255
256 unsigned int xattr_shared_count;
257 unsigned int *xattr_shared_xattrs;
258
259 union {
260 erofs_blk_t raw_blkaddr;
261#ifdef CONFIG_EROFS_FS_ZIP
262 struct {
263 unsigned short z_advise;
264 unsigned char z_algorithmtype[2];
265 unsigned char z_logical_clusterbits;
266 };
267#endif
268 };
269
270 struct inode vfs_inode;
271};
272
273#define EROFS_I(ptr) \
274 container_of(ptr, struct erofs_inode, vfs_inode)
275
276static inline unsigned long erofs_inode_datablocks(struct inode *inode)
277{
278
279 return DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
280}
281
282static inline unsigned int erofs_bitrange(unsigned int value, unsigned int bit,
283 unsigned int bits)
284{
285
286 return (value >> bit) & ((1 << bits) - 1);
287}
288
289
290static inline unsigned int erofs_inode_version(unsigned int value)
291{
292 return erofs_bitrange(value, EROFS_I_VERSION_BIT,
293 EROFS_I_VERSION_BITS);
294}
295
296static inline unsigned int erofs_inode_datalayout(unsigned int value)
297{
298 return erofs_bitrange(value, EROFS_I_DATALAYOUT_BIT,
299 EROFS_I_DATALAYOUT_BITS);
300}
301
302extern const struct super_operations erofs_sops;
303
304extern const struct address_space_operations erofs_raw_access_aops;
305extern const struct address_space_operations z_erofs_aops;
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332enum {
333 BH_Zipped = BH_PrivateStart,
334 BH_FullMapped,
335};
336
337
338#define EROFS_MAP_MAPPED (1 << BH_Mapped)
339
340#define EROFS_MAP_META (1 << BH_Meta)
341
342#define EROFS_MAP_ZIPPED (1 << BH_Zipped)
343
344#define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped)
345
346struct erofs_map_blocks {
347 erofs_off_t m_pa, m_la;
348 u64 m_plen, m_llen;
349
350 unsigned int m_flags;
351
352 struct page *mpage;
353};
354
355
356#define EROFS_GET_BLOCKS_RAW 0x0001
357
358
359#ifdef CONFIG_EROFS_FS_ZIP
360int z_erofs_fill_inode(struct inode *inode);
361int z_erofs_map_blocks_iter(struct inode *inode,
362 struct erofs_map_blocks *map,
363 int flags);
364#else
365static inline int z_erofs_fill_inode(struct inode *inode) { return -EOPNOTSUPP; }
366static inline int z_erofs_map_blocks_iter(struct inode *inode,
367 struct erofs_map_blocks *map,
368 int flags)
369{
370 return -EOPNOTSUPP;
371}
372#endif
373
374
375struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr);
376
377
378static inline unsigned long erofs_inode_hash(erofs_nid_t nid)
379{
380#if BITS_PER_LONG == 32
381 return (nid >> 32) ^ (nid & 0xffffffff);
382#else
383 return nid;
384#endif
385}
386
387extern const struct inode_operations erofs_generic_iops;
388extern const struct inode_operations erofs_symlink_iops;
389extern const struct inode_operations erofs_fast_symlink_iops;
390
391struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid, bool dir);
392int erofs_getattr(struct user_namespace *mnt_userns, const struct path *path,
393 struct kstat *stat, u32 request_mask,
394 unsigned int query_flags);
395
396
397extern const struct inode_operations erofs_dir_iops;
398
399int erofs_namei(struct inode *dir, struct qstr *name,
400 erofs_nid_t *nid, unsigned int *d_type);
401
402
403extern const struct file_operations erofs_dir_fops;
404
405static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count)
406{
407 int retried = 0;
408
409 while (1) {
410 void *p = vm_map_ram(pages, count, -1);
411
412
413 if (p || ++retried >= 3)
414 return p;
415 vm_unmap_aliases();
416 }
417 return NULL;
418}
419
420
421void *erofs_get_pcpubuf(unsigned int requiredpages);
422void erofs_put_pcpubuf(void *ptr);
423int erofs_pcpubuf_growsize(unsigned int nrpages);
424void erofs_pcpubuf_init(void);
425void erofs_pcpubuf_exit(void);
426
427
428struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp);
429
430#ifdef CONFIG_EROFS_FS_ZIP
431int erofs_workgroup_put(struct erofs_workgroup *grp);
432struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,
433 pgoff_t index);
434struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,
435 struct erofs_workgroup *grp);
436void erofs_workgroup_free_rcu(struct erofs_workgroup *grp);
437void erofs_shrinker_register(struct super_block *sb);
438void erofs_shrinker_unregister(struct super_block *sb);
439int __init erofs_init_shrinker(void);
440void erofs_exit_shrinker(void);
441int __init z_erofs_init_zip_subsystem(void);
442void z_erofs_exit_zip_subsystem(void);
443int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
444 struct erofs_workgroup *egrp);
445int erofs_try_to_free_cached_page(struct address_space *mapping,
446 struct page *page);
447int z_erofs_load_lz4_config(struct super_block *sb,
448 struct erofs_super_block *dsb,
449 struct z_erofs_lz4_cfgs *lz4, int len);
450#else
451static inline void erofs_shrinker_register(struct super_block *sb) {}
452static inline void erofs_shrinker_unregister(struct super_block *sb) {}
453static inline int erofs_init_shrinker(void) { return 0; }
454static inline void erofs_exit_shrinker(void) {}
455static inline int z_erofs_init_zip_subsystem(void) { return 0; }
456static inline void z_erofs_exit_zip_subsystem(void) {}
457static inline int z_erofs_load_lz4_config(struct super_block *sb,
458 struct erofs_super_block *dsb,
459 struct z_erofs_lz4_cfgs *lz4, int len)
460{
461 if (lz4 || dsb->u1.lz4_max_distance) {
462 erofs_err(sb, "lz4 algorithm isn't enabled");
463 return -EINVAL;
464 }
465 return 0;
466}
467#endif
468
469#define EFSCORRUPTED EUCLEAN
470
471#endif
472
473