1
2#ifndef __MM_KASAN_KASAN_H
3#define __MM_KASAN_KASAN_H
4
5#include <linux/kasan.h>
6#include <linux/kasan-tags.h>
7#include <linux/kfence.h>
8#include <linux/stackdepot.h>
9
10#ifdef CONFIG_KASAN_HW_TAGS
11
12#include <linux/static_key.h>
13#include "../slab.h"
14
15DECLARE_STATIC_KEY_FALSE(kasan_flag_stacktrace);
16extern bool kasan_flag_async __ro_after_init;
17
18static inline bool kasan_stack_collection_enabled(void)
19{
20 return static_branch_unlikely(&kasan_flag_stacktrace);
21}
22
23static inline bool kasan_async_mode_enabled(void)
24{
25 return kasan_flag_async;
26}
27#else
28
29static inline bool kasan_stack_collection_enabled(void)
30{
31 return true;
32}
33
34static inline bool kasan_async_mode_enabled(void)
35{
36 return false;
37}
38
39#endif
40
41extern bool kasan_flag_async __ro_after_init;
42
43#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
44#define KASAN_GRANULE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT)
45#else
46#include <asm/mte-kasan.h>
47#define KASAN_GRANULE_SIZE MTE_GRANULE_SIZE
48#endif
49
50#define KASAN_GRANULE_MASK (KASAN_GRANULE_SIZE - 1)
51
52#define KASAN_MEMORY_PER_SHADOW_PAGE (KASAN_GRANULE_SIZE << PAGE_SHIFT)
53
54#ifdef CONFIG_KASAN_GENERIC
55#define KASAN_FREE_PAGE 0xFF
56#define KASAN_PAGE_REDZONE 0xFE
57#define KASAN_KMALLOC_REDZONE 0xFC
58#define KASAN_KMALLOC_FREE 0xFB
59#define KASAN_KMALLOC_FREETRACK 0xFA
60#else
61#define KASAN_FREE_PAGE KASAN_TAG_INVALID
62#define KASAN_PAGE_REDZONE KASAN_TAG_INVALID
63#define KASAN_KMALLOC_REDZONE KASAN_TAG_INVALID
64#define KASAN_KMALLOC_FREE KASAN_TAG_INVALID
65#define KASAN_KMALLOC_FREETRACK KASAN_TAG_INVALID
66#endif
67
68#define KASAN_GLOBAL_REDZONE 0xF9
69#define KASAN_VMALLOC_INVALID 0xF8
70
71
72
73
74
75#define KASAN_STACK_LEFT 0xF1
76#define KASAN_STACK_MID 0xF2
77#define KASAN_STACK_RIGHT 0xF3
78#define KASAN_STACK_PARTIAL 0xF4
79
80
81
82
83#define KASAN_ALLOCA_LEFT 0xCA
84#define KASAN_ALLOCA_RIGHT 0xCB
85
86#define KASAN_ALLOCA_REDZONE_SIZE 32
87
88
89
90
91#define KASAN_CURRENT_STACK_FRAME_MAGIC 0x41B58AB3
92
93
94#ifndef KASAN_ABI_VERSION
95#define KASAN_ABI_VERSION 1
96#endif
97
98
99#define META_BYTES_PER_BLOCK 1
100#define META_BLOCKS_PER_ROW 16
101#define META_BYTES_PER_ROW (META_BLOCKS_PER_ROW * META_BYTES_PER_BLOCK)
102#define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE)
103#define META_ROWS_AROUND_ADDR 2
104
105struct kasan_access_info {
106 const void *access_addr;
107 const void *first_bad_addr;
108 size_t access_size;
109 bool is_write;
110 unsigned long ip;
111};
112
113
114struct kasan_source_location {
115 const char *filename;
116 int line_no;
117 int column_no;
118};
119
120
121struct kasan_global {
122 const void *beg;
123 size_t size;
124 size_t size_with_redzone;
125 const void *name;
126 const void *module_name;
127 unsigned long has_dynamic_init;
128#if KASAN_ABI_VERSION >= 4
129 struct kasan_source_location *location;
130#endif
131#if KASAN_ABI_VERSION >= 5
132 char *odr_indicator;
133#endif
134};
135
136
137
138
139
140#define KASAN_STACK_DEPTH 64
141
142struct kasan_track {
143 u32 pid;
144 depot_stack_handle_t stack;
145};
146
147#if defined(CONFIG_KASAN_TAGS_IDENTIFY) && defined(CONFIG_KASAN_SW_TAGS)
148#define KASAN_NR_FREE_STACKS 5
149#else
150#define KASAN_NR_FREE_STACKS 1
151#endif
152
153struct kasan_alloc_meta {
154 struct kasan_track alloc_track;
155#ifdef CONFIG_KASAN_GENERIC
156
157
158
159
160 depot_stack_handle_t aux_stack[2];
161#else
162 struct kasan_track free_track[KASAN_NR_FREE_STACKS];
163#endif
164#ifdef CONFIG_KASAN_TAGS_IDENTIFY
165 u8 free_pointer_tag[KASAN_NR_FREE_STACKS];
166 u8 free_track_idx;
167#endif
168};
169
170struct qlist_node {
171 struct qlist_node *next;
172};
173
174
175
176
177
178
179
180#define KASAN_NO_FREE_META INT_MAX
181
182struct kasan_free_meta {
183#ifdef CONFIG_KASAN_GENERIC
184
185
186
187 struct qlist_node quarantine_link;
188 struct kasan_track free_track;
189#endif
190};
191
192struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
193 const void *object);
194#ifdef CONFIG_KASAN_GENERIC
195struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
196 const void *object);
197#endif
198
199#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
200
201static inline const void *kasan_shadow_to_mem(const void *shadow_addr)
202{
203 return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET)
204 << KASAN_SHADOW_SCALE_SHIFT);
205}
206
207static inline bool addr_has_metadata(const void *addr)
208{
209 return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START));
210}
211
212
213
214
215
216
217
218
219
220bool kasan_check_range(unsigned long addr, size_t size, bool write,
221 unsigned long ret_ip);
222
223#else
224
225static inline bool addr_has_metadata(const void *addr)
226{
227 return (is_vmalloc_addr(addr) || virt_addr_valid(addr));
228}
229
230#endif
231
232#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
233void kasan_print_tags(u8 addr_tag, const void *addr);
234#else
235static inline void kasan_print_tags(u8 addr_tag, const void *addr) { }
236#endif
237
238void *kasan_find_first_bad_addr(void *addr, size_t size);
239const char *kasan_get_bug_type(struct kasan_access_info *info);
240void kasan_metadata_fetch_row(char *buffer, void *row);
241
242#if defined(CONFIG_KASAN_GENERIC) && defined(CONFIG_KASAN_STACK)
243void kasan_print_address_stack_frame(const void *addr);
244#else
245static inline void kasan_print_address_stack_frame(const void *addr) { }
246#endif
247
248bool kasan_report(unsigned long addr, size_t size,
249 bool is_write, unsigned long ip);
250void kasan_report_invalid_free(void *object, unsigned long ip);
251
252struct page *kasan_addr_to_page(const void *addr);
253
254depot_stack_handle_t kasan_save_stack(gfp_t flags);
255void kasan_set_track(struct kasan_track *track, gfp_t flags);
256void kasan_set_free_info(struct kmem_cache *cache, void *object, u8 tag);
257struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
258 void *object, u8 tag);
259
260#if defined(CONFIG_KASAN_GENERIC) && \
261 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB))
262bool kasan_quarantine_put(struct kmem_cache *cache, void *object);
263void kasan_quarantine_reduce(void);
264void kasan_quarantine_remove_cache(struct kmem_cache *cache);
265#else
266static inline bool kasan_quarantine_put(struct kmem_cache *cache, void *object) { return false; }
267static inline void kasan_quarantine_reduce(void) { }
268static inline void kasan_quarantine_remove_cache(struct kmem_cache *cache) { }
269#endif
270
271#ifndef arch_kasan_set_tag
272static inline const void *arch_kasan_set_tag(const void *addr, u8 tag)
273{
274 return addr;
275}
276#endif
277#ifndef arch_kasan_get_tag
278#define arch_kasan_get_tag(addr) 0
279#endif
280
281#define set_tag(addr, tag) ((void *)arch_kasan_set_tag((addr), (tag)))
282#define get_tag(addr) arch_kasan_get_tag(addr)
283
284#ifdef CONFIG_KASAN_HW_TAGS
285
286#ifndef arch_enable_tagging_sync
287#define arch_enable_tagging_sync()
288#endif
289#ifndef arch_enable_tagging_async
290#define arch_enable_tagging_async()
291#endif
292#ifndef arch_force_async_tag_fault
293#define arch_force_async_tag_fault()
294#endif
295#ifndef arch_get_random_tag
296#define arch_get_random_tag() (0xFF)
297#endif
298#ifndef arch_get_mem_tag
299#define arch_get_mem_tag(addr) (0xFF)
300#endif
301#ifndef arch_set_mem_tag_range
302#define arch_set_mem_tag_range(addr, size, tag, init) ((void *)(addr))
303#endif
304
305#define hw_enable_tagging_sync() arch_enable_tagging_sync()
306#define hw_enable_tagging_async() arch_enable_tagging_async()
307#define hw_force_async_tag_fault() arch_force_async_tag_fault()
308#define hw_get_random_tag() arch_get_random_tag()
309#define hw_get_mem_tag(addr) arch_get_mem_tag(addr)
310#define hw_set_mem_tag_range(addr, size, tag, init) \
311 arch_set_mem_tag_range((addr), (size), (tag), (init))
312
313#else
314
315#define hw_enable_tagging_sync()
316#define hw_enable_tagging_async()
317
318#endif
319
320#if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
321
322void kasan_enable_tagging_sync(void);
323void kasan_force_async_fault(void);
324
325#else
326
327static inline void kasan_enable_tagging_sync(void) { }
328static inline void kasan_force_async_fault(void) { }
329
330#endif
331
332#ifdef CONFIG_KASAN_SW_TAGS
333u8 kasan_random_tag(void);
334#elif defined(CONFIG_KASAN_HW_TAGS)
335static inline u8 kasan_random_tag(void) { return hw_get_random_tag(); }
336#else
337static inline u8 kasan_random_tag(void) { return 0; }
338#endif
339
340#ifdef CONFIG_KASAN_HW_TAGS
341
342static inline void kasan_poison(const void *addr, size_t size, u8 value, bool init)
343{
344 addr = kasan_reset_tag(addr);
345
346
347 if (is_kfence_address(addr))
348 return;
349
350 if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
351 return;
352 if (WARN_ON(size & KASAN_GRANULE_MASK))
353 return;
354
355 hw_set_mem_tag_range((void *)addr, size, value, init);
356}
357
358static inline void kasan_unpoison(const void *addr, size_t size, bool init)
359{
360 u8 tag = get_tag(addr);
361
362 addr = kasan_reset_tag(addr);
363
364
365 if (is_kfence_address(addr))
366 return;
367
368 if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
369 return;
370
371
372
373
374
375
376 if (__slub_debug_enabled() &&
377 init && ((unsigned long)size & KASAN_GRANULE_MASK)) {
378 init = false;
379 memzero_explicit((void *)addr, size);
380 }
381 size = round_up(size, KASAN_GRANULE_SIZE);
382
383 hw_set_mem_tag_range((void *)addr, size, tag, init);
384}
385
386static inline bool kasan_byte_accessible(const void *addr)
387{
388 u8 ptr_tag = get_tag(addr);
389 u8 mem_tag = hw_get_mem_tag((void *)addr);
390
391 return ptr_tag == KASAN_TAG_KERNEL || ptr_tag == mem_tag;
392}
393
394#else
395
396
397
398
399
400
401
402
403
404
405void kasan_poison(const void *addr, size_t size, u8 value, bool init);
406
407
408
409
410
411
412
413
414
415
416
417
418void kasan_unpoison(const void *addr, size_t size, bool init);
419
420bool kasan_byte_accessible(const void *addr);
421
422#endif
423
424#ifdef CONFIG_KASAN_GENERIC
425
426
427
428
429
430
431
432
433
434
435void kasan_poison_last_granule(const void *address, size_t size);
436
437#else
438
439static inline void kasan_poison_last_granule(const void *address, size_t size) { }
440
441#endif
442
443#ifndef kasan_arch_is_ready
444static inline bool kasan_arch_is_ready(void) { return true; }
445#elif !defined(CONFIG_KASAN_GENERIC) || !defined(CONFIG_KASAN_OUTLINE)
446#error kasan_arch_is_ready only works in KASAN generic outline mode!
447#endif
448
449
450
451
452
453asmlinkage void kasan_unpoison_task_stack_below(const void *watermark);
454void __asan_register_globals(struct kasan_global *globals, size_t size);
455void __asan_unregister_globals(struct kasan_global *globals, size_t size);
456void __asan_handle_no_return(void);
457void __asan_alloca_poison(unsigned long addr, size_t size);
458void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom);
459
460void __asan_load1(unsigned long addr);
461void __asan_store1(unsigned long addr);
462void __asan_load2(unsigned long addr);
463void __asan_store2(unsigned long addr);
464void __asan_load4(unsigned long addr);
465void __asan_store4(unsigned long addr);
466void __asan_load8(unsigned long addr);
467void __asan_store8(unsigned long addr);
468void __asan_load16(unsigned long addr);
469void __asan_store16(unsigned long addr);
470void __asan_loadN(unsigned long addr, size_t size);
471void __asan_storeN(unsigned long addr, size_t size);
472
473void __asan_load1_noabort(unsigned long addr);
474void __asan_store1_noabort(unsigned long addr);
475void __asan_load2_noabort(unsigned long addr);
476void __asan_store2_noabort(unsigned long addr);
477void __asan_load4_noabort(unsigned long addr);
478void __asan_store4_noabort(unsigned long addr);
479void __asan_load8_noabort(unsigned long addr);
480void __asan_store8_noabort(unsigned long addr);
481void __asan_load16_noabort(unsigned long addr);
482void __asan_store16_noabort(unsigned long addr);
483void __asan_loadN_noabort(unsigned long addr, size_t size);
484void __asan_storeN_noabort(unsigned long addr, size_t size);
485
486void __asan_report_load1_noabort(unsigned long addr);
487void __asan_report_store1_noabort(unsigned long addr);
488void __asan_report_load2_noabort(unsigned long addr);
489void __asan_report_store2_noabort(unsigned long addr);
490void __asan_report_load4_noabort(unsigned long addr);
491void __asan_report_store4_noabort(unsigned long addr);
492void __asan_report_load8_noabort(unsigned long addr);
493void __asan_report_store8_noabort(unsigned long addr);
494void __asan_report_load16_noabort(unsigned long addr);
495void __asan_report_store16_noabort(unsigned long addr);
496void __asan_report_load_n_noabort(unsigned long addr, size_t size);
497void __asan_report_store_n_noabort(unsigned long addr, size_t size);
498
499void __asan_set_shadow_00(const void *addr, size_t size);
500void __asan_set_shadow_f1(const void *addr, size_t size);
501void __asan_set_shadow_f2(const void *addr, size_t size);
502void __asan_set_shadow_f3(const void *addr, size_t size);
503void __asan_set_shadow_f5(const void *addr, size_t size);
504void __asan_set_shadow_f8(const void *addr, size_t size);
505
506void __hwasan_load1_noabort(unsigned long addr);
507void __hwasan_store1_noabort(unsigned long addr);
508void __hwasan_load2_noabort(unsigned long addr);
509void __hwasan_store2_noabort(unsigned long addr);
510void __hwasan_load4_noabort(unsigned long addr);
511void __hwasan_store4_noabort(unsigned long addr);
512void __hwasan_load8_noabort(unsigned long addr);
513void __hwasan_store8_noabort(unsigned long addr);
514void __hwasan_load16_noabort(unsigned long addr);
515void __hwasan_store16_noabort(unsigned long addr);
516void __hwasan_loadN_noabort(unsigned long addr, size_t size);
517void __hwasan_storeN_noabort(unsigned long addr, size_t size);
518
519void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size);
520
521#endif
522