1
2
3
4
5
6
7#ifndef __EROFS_INTERNAL_H
8#define __EROFS_INTERNAL_H
9
10#include <linux/fs.h>
11#include <linux/dcache.h>
12#include <linux/mm.h>
13#include <linux/pagemap.h>
14#include <linux/bio.h>
15#include <linux/buffer_head.h>
16#include <linux/magic.h>
17#include <linux/slab.h>
18#include <linux/vmalloc.h>
19#include <linux/iomap.h>
20#include "erofs_fs.h"
21
22
23#undef pr_fmt
24#define pr_fmt(fmt) "erofs: " fmt
25
26__printf(3, 4) void _erofs_err(struct super_block *sb,
27 const char *function, const char *fmt, ...);
28#define erofs_err(sb, fmt, ...) \
29 _erofs_err(sb, __func__, fmt "\n", ##__VA_ARGS__)
30__printf(3, 4) void _erofs_info(struct super_block *sb,
31 const char *function, const char *fmt, ...);
32#define erofs_info(sb, fmt, ...) \
33 _erofs_info(sb, __func__, fmt "\n", ##__VA_ARGS__)
34#ifdef CONFIG_EROFS_FS_DEBUG
35#define erofs_dbg(x, ...) pr_debug(x "\n", ##__VA_ARGS__)
36#define DBG_BUGON BUG_ON
37#else
38#define erofs_dbg(x, ...) ((void)0)
39#define DBG_BUGON(x) ((void)(x))
40#endif
41
42
43#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1
44
45typedef u64 erofs_nid_t;
46typedef u64 erofs_off_t;
47
48typedef u32 erofs_blk_t;
49
50struct erofs_fs_context {
51#ifdef CONFIG_EROFS_FS_ZIP
52
53 unsigned char cache_strategy;
54
55 bool readahead_sync_decompress;
56
57
58 unsigned int max_sync_decompress_pages;
59#endif
60 unsigned int mount_opt;
61};
62
63
64struct erofs_sb_lz4_info {
65
66 u16 max_distance_pages;
67
68 u16 max_pclusterblks;
69};
70
71struct erofs_sb_info {
72#ifdef CONFIG_EROFS_FS_ZIP
73
74 struct list_head list;
75 struct mutex umount_mutex;
76
77
78 struct xarray managed_pslots;
79
80 unsigned int shrinker_run_no;
81 u16 available_compr_algs;
82
83
84 struct inode *managed_cache;
85
86 struct erofs_sb_lz4_info lz4;
87#endif
88 struct dax_device *dax_dev;
89 u32 blocks;
90 u32 meta_blkaddr;
91#ifdef CONFIG_EROFS_FS_XATTR
92 u32 xattr_blkaddr;
93#endif
94
95
96 unsigned char islotbits;
97
98 u32 sb_size;
99 u32 build_time_nsec;
100 u64 build_time;
101
102
103 erofs_nid_t root_nid;
104
105 u64 inos;
106
107 u8 uuid[16];
108 u8 volume_name[16];
109 u32 feature_compat;
110 u32 feature_incompat;
111
112 struct erofs_fs_context ctx;
113};
114
115#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
116#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)
117
118
119#define EROFS_MOUNT_XATTR_USER 0x00000010
120#define EROFS_MOUNT_POSIX_ACL 0x00000020
121#define EROFS_MOUNT_DAX_ALWAYS 0x00000040
122#define EROFS_MOUNT_DAX_NEVER 0x00000080
123
124#define clear_opt(ctx, option) ((ctx)->mount_opt &= ~EROFS_MOUNT_##option)
125#define set_opt(ctx, option) ((ctx)->mount_opt |= EROFS_MOUNT_##option)
126#define test_opt(ctx, option) ((ctx)->mount_opt & EROFS_MOUNT_##option)
127
128enum {
129 EROFS_ZIP_CACHE_DISABLED,
130 EROFS_ZIP_CACHE_READAHEAD,
131 EROFS_ZIP_CACHE_READAROUND
132};
133
134#ifdef CONFIG_EROFS_FS_ZIP
135#define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL)
136
137
138struct erofs_workgroup {
139
140 pgoff_t index;
141
142
143 atomic_t refcount;
144};
145
146#if defined(CONFIG_SMP)
147static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp,
148 int val)
149{
150 preempt_disable();
151 if (val != atomic_cmpxchg(&grp->refcount, val, EROFS_LOCKED_MAGIC)) {
152 preempt_enable();
153 return false;
154 }
155 return true;
156}
157
158static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp,
159 int orig_val)
160{
161
162
163
164
165 smp_mb();
166 atomic_set(&grp->refcount, orig_val);
167 preempt_enable();
168}
169
170static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
171{
172 return atomic_cond_read_relaxed(&grp->refcount,
173 VAL != EROFS_LOCKED_MAGIC);
174}
175#else
176static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp,
177 int val)
178{
179 preempt_disable();
180
181 if (val != atomic_read(&grp->refcount)) {
182 preempt_enable();
183 return false;
184 }
185 return true;
186}
187
188static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp,
189 int orig_val)
190{
191 preempt_enable();
192}
193
194static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
195{
196 int v = atomic_read(&grp->refcount);
197
198
199 DBG_BUGON(v == EROFS_LOCKED_MAGIC);
200 return v;
201}
202#endif
203#endif
204
205
206#define LOG_BLOCK_SIZE PAGE_SHIFT
207
208#undef LOG_SECTORS_PER_BLOCK
209#define LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9)
210
211#undef SECTORS_PER_BLOCK
212#define SECTORS_PER_BLOCK (1 << SECTORS_PER_BLOCK)
213
214#define EROFS_BLKSIZ (1 << LOG_BLOCK_SIZE)
215
216#if (EROFS_BLKSIZ % 4096 || !EROFS_BLKSIZ)
217#error erofs cannot be used in this platform
218#endif
219
220#define ROOT_NID(sb) ((sb)->root_nid)
221
222#define erofs_blknr(addr) ((addr) / EROFS_BLKSIZ)
223#define erofs_blkoff(addr) ((addr) % EROFS_BLKSIZ)
224#define blknr_to_addr(nr) ((erofs_off_t)(nr) * EROFS_BLKSIZ)
225
226static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid)
227{
228 return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits);
229}
230
231#define EROFS_FEATURE_FUNCS(name, compat, feature) \
232static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
233{ \
234 return sbi->feature_##compat & EROFS_FEATURE_##feature; \
235}
236
237EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
238EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
239EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
240EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
241
242
243#define EROFS_I_EA_INITED_BIT 0
244#define EROFS_I_Z_INITED_BIT 1
245
246
247#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1)
248#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2)
249
250struct erofs_inode {
251 erofs_nid_t nid;
252
253
254 unsigned long flags;
255
256 unsigned char datalayout;
257 unsigned char inode_isize;
258 unsigned short xattr_isize;
259
260 unsigned int xattr_shared_count;
261 unsigned int *xattr_shared_xattrs;
262
263 union {
264 erofs_blk_t raw_blkaddr;
265 struct {
266 unsigned short chunkformat;
267 unsigned char chunkbits;
268 };
269#ifdef CONFIG_EROFS_FS_ZIP
270 struct {
271 unsigned short z_advise;
272 unsigned char z_algorithmtype[2];
273 unsigned char z_logical_clusterbits;
274 };
275#endif
276 };
277
278 struct inode vfs_inode;
279};
280
281#define EROFS_I(ptr) \
282 container_of(ptr, struct erofs_inode, vfs_inode)
283
284static inline unsigned long erofs_inode_datablocks(struct inode *inode)
285{
286
287 return DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
288}
289
290static inline unsigned int erofs_bitrange(unsigned int value, unsigned int bit,
291 unsigned int bits)
292{
293
294 return (value >> bit) & ((1 << bits) - 1);
295}
296
297
298static inline unsigned int erofs_inode_version(unsigned int value)
299{
300 return erofs_bitrange(value, EROFS_I_VERSION_BIT,
301 EROFS_I_VERSION_BITS);
302}
303
304static inline unsigned int erofs_inode_datalayout(unsigned int value)
305{
306 return erofs_bitrange(value, EROFS_I_DATALAYOUT_BIT,
307 EROFS_I_DATALAYOUT_BITS);
308}
309
310extern const struct super_operations erofs_sops;
311
312extern const struct address_space_operations erofs_raw_access_aops;
313extern const struct address_space_operations z_erofs_aops;
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340enum {
341 BH_Zipped = BH_PrivateStart,
342 BH_FullMapped,
343};
344
345
346#define EROFS_MAP_MAPPED (1 << BH_Mapped)
347
348#define EROFS_MAP_META (1 << BH_Meta)
349
350#define EROFS_MAP_ZIPPED (1 << BH_Zipped)
351
352#define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped)
353
354struct erofs_map_blocks {
355 erofs_off_t m_pa, m_la;
356 u64 m_plen, m_llen;
357
358 unsigned int m_flags;
359
360 struct page *mpage;
361};
362
363
364#define EROFS_GET_BLOCKS_RAW 0x0001
365
366
367
368
369#define EROFS_GET_BLOCKS_FIEMAP 0x0002
370
371
372extern const struct iomap_ops z_erofs_iomap_report_ops;
373
374#ifdef CONFIG_EROFS_FS_ZIP
375int z_erofs_fill_inode(struct inode *inode);
376int z_erofs_map_blocks_iter(struct inode *inode,
377 struct erofs_map_blocks *map,
378 int flags);
379#else
380static inline int z_erofs_fill_inode(struct inode *inode) { return -EOPNOTSUPP; }
381static inline int z_erofs_map_blocks_iter(struct inode *inode,
382 struct erofs_map_blocks *map,
383 int flags)
384{
385 return -EOPNOTSUPP;
386}
387#endif
388
389
390extern const struct file_operations erofs_file_fops;
391struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr);
392int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
393 u64 start, u64 len);
394
395
396static inline unsigned long erofs_inode_hash(erofs_nid_t nid)
397{
398#if BITS_PER_LONG == 32
399 return (nid >> 32) ^ (nid & 0xffffffff);
400#else
401 return nid;
402#endif
403}
404
405extern const struct inode_operations erofs_generic_iops;
406extern const struct inode_operations erofs_symlink_iops;
407extern const struct inode_operations erofs_fast_symlink_iops;
408
409struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid, bool dir);
410int erofs_getattr(struct user_namespace *mnt_userns, const struct path *path,
411 struct kstat *stat, u32 request_mask,
412 unsigned int query_flags);
413
414
415extern const struct inode_operations erofs_dir_iops;
416
417int erofs_namei(struct inode *dir, struct qstr *name,
418 erofs_nid_t *nid, unsigned int *d_type);
419
420
421extern const struct file_operations erofs_dir_fops;
422
423static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count)
424{
425 int retried = 0;
426
427 while (1) {
428 void *p = vm_map_ram(pages, count, -1);
429
430
431 if (p || ++retried >= 3)
432 return p;
433 vm_unmap_aliases();
434 }
435 return NULL;
436}
437
438
439void *erofs_get_pcpubuf(unsigned int requiredpages);
440void erofs_put_pcpubuf(void *ptr);
441int erofs_pcpubuf_growsize(unsigned int nrpages);
442void erofs_pcpubuf_init(void);
443void erofs_pcpubuf_exit(void);
444
445
446struct page *erofs_allocpage(struct list_head *pool, gfp_t gfp);
447
448#ifdef CONFIG_EROFS_FS_ZIP
449int erofs_workgroup_put(struct erofs_workgroup *grp);
450struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,
451 pgoff_t index);
452struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,
453 struct erofs_workgroup *grp);
454void erofs_workgroup_free_rcu(struct erofs_workgroup *grp);
455void erofs_shrinker_register(struct super_block *sb);
456void erofs_shrinker_unregister(struct super_block *sb);
457int __init erofs_init_shrinker(void);
458void erofs_exit_shrinker(void);
459int __init z_erofs_init_zip_subsystem(void);
460void z_erofs_exit_zip_subsystem(void);
461int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
462 struct erofs_workgroup *egrp);
463int erofs_try_to_free_cached_page(struct page *page);
464int z_erofs_load_lz4_config(struct super_block *sb,
465 struct erofs_super_block *dsb,
466 struct z_erofs_lz4_cfgs *lz4, int len);
467#else
468static inline void erofs_shrinker_register(struct super_block *sb) {}
469static inline void erofs_shrinker_unregister(struct super_block *sb) {}
470static inline int erofs_init_shrinker(void) { return 0; }
471static inline void erofs_exit_shrinker(void) {}
472static inline int z_erofs_init_zip_subsystem(void) { return 0; }
473static inline void z_erofs_exit_zip_subsystem(void) {}
474static inline int z_erofs_load_lz4_config(struct super_block *sb,
475 struct erofs_super_block *dsb,
476 struct z_erofs_lz4_cfgs *lz4, int len)
477{
478 if (lz4 || dsb->u1.lz4_max_distance) {
479 erofs_err(sb, "lz4 algorithm isn't enabled");
480 return -EINVAL;
481 }
482 return 0;
483}
484#endif
485
486#define EFSCORRUPTED EUCLEAN
487
488#endif
489