LXR linux/mm/kasan/kasan.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef __MM_KASAN_KASAN_H
   3#define __MM_KASAN_KASAN_H
   4
   5#include <linux/kasan.h>
   6#include <linux/kfence.h>
   7#include <linux/stackdepot.h>
   8
   9#ifdef CONFIG_KASAN_HW_TAGS
  10
  11#include <linux/static_key.h>
  12#include "../slab.h"
  13
  14DECLARE_STATIC_KEY_FALSE(kasan_flag_stacktrace);
  15extern bool kasan_flag_async __ro_after_init;
  16
  17static inline bool kasan_stack_collection_enabled(void)
  18{
  19        return static_branch_unlikely(&kasan_flag_stacktrace);
  20}
  21
  22static inline bool kasan_async_mode_enabled(void)
  23{
  24        return kasan_flag_async;
  25}
  26#else
  27
  28static inline bool kasan_stack_collection_enabled(void)
  29{
  30        return true;
  31}
  32
  33static inline bool kasan_async_mode_enabled(void)
  34{
  35        return false;
  36}
  37
  38#endif
  39
  40extern bool kasan_flag_panic __ro_after_init;
  41extern bool kasan_flag_async __ro_after_init;
  42
  43#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
  44#define KASAN_GRANULE_SIZE      (1UL << KASAN_SHADOW_SCALE_SHIFT)
  45#else
  46#include <asm/mte-kasan.h>
  47#define KASAN_GRANULE_SIZE      MTE_GRANULE_SIZE
  48#endif
  49
  50#define KASAN_GRANULE_MASK      (KASAN_GRANULE_SIZE - 1)
  51
  52#define KASAN_MEMORY_PER_SHADOW_PAGE    (KASAN_GRANULE_SIZE << PAGE_SHIFT)
  53
  54#define KASAN_TAG_KERNEL        0xFF /* native kernel pointers tag */
  55#define KASAN_TAG_INVALID       0xFE /* inaccessible memory tag */
  56#define KASAN_TAG_MAX           0xFD /* maximum value for random tags */
  57
  58#ifdef CONFIG_KASAN_HW_TAGS
  59#define KASAN_TAG_MIN           0xF0 /* minimum value for random tags */
  60#else
  61#define KASAN_TAG_MIN           0x00 /* minimum value for random tags */
  62#endif
  63
  64#ifdef CONFIG_KASAN_GENERIC
  65#define KASAN_FREE_PAGE         0xFF  /* page was freed */
  66#define KASAN_PAGE_REDZONE      0xFE  /* redzone for kmalloc_large allocations */
  67#define KASAN_KMALLOC_REDZONE   0xFC  /* redzone inside slub object */
  68#define KASAN_KMALLOC_FREE      0xFB  /* object was freed (kmem_cache_free/kfree) */
  69#define KASAN_KMALLOC_FREETRACK 0xFA  /* object was freed and has free track set */
  70#else
  71#define KASAN_FREE_PAGE         KASAN_TAG_INVALID
  72#define KASAN_PAGE_REDZONE      KASAN_TAG_INVALID
  73#define KASAN_KMALLOC_REDZONE   KASAN_TAG_INVALID
  74#define KASAN_KMALLOC_FREE      KASAN_TAG_INVALID
  75#define KASAN_KMALLOC_FREETRACK KASAN_TAG_INVALID
  76#endif
  77
  78#define KASAN_GLOBAL_REDZONE    0xF9  /* redzone for global variable */
  79#define KASAN_VMALLOC_INVALID   0xF8  /* unallocated space in vmapped page */
  80
  81/*
  82 * Stack redzone shadow values
  83 * (Those are compiler's ABI, don't change them)
  84 */
  85#define KASAN_STACK_LEFT        0xF1
  86#define KASAN_STACK_MID         0xF2
  87#define KASAN_STACK_RIGHT       0xF3
  88#define KASAN_STACK_PARTIAL     0xF4
  89
  90/*
  91 * alloca redzone shadow values
  92 */
  93#define KASAN_ALLOCA_LEFT       0xCA
  94#define KASAN_ALLOCA_RIGHT      0xCB
  95
  96#define KASAN_ALLOCA_REDZONE_SIZE       32
  97
  98/*
  99 * Stack frame marker (compiler ABI).
 100 */
 101#define KASAN_CURRENT_STACK_FRAME_MAGIC 0x41B58AB3
 102
 103/* Don't break randconfig/all*config builds */
 104#ifndef KASAN_ABI_VERSION
 105#define KASAN_ABI_VERSION 1
 106#endif
 107
 108/* Metadata layout customization. */
 109#define META_BYTES_PER_BLOCK 1
 110#define META_BLOCKS_PER_ROW 16
 111#define META_BYTES_PER_ROW (META_BLOCKS_PER_ROW * META_BYTES_PER_BLOCK)
 112#define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE)
 113#define META_ROWS_AROUND_ADDR 2
 114
 115struct kasan_access_info {
 116        const void *access_addr;
 117        const void *first_bad_addr;
 118        size_t access_size;
 119        bool is_write;
 120        unsigned long ip;
 121};
 122
 123/* The layout of struct dictated by compiler */
 124struct kasan_source_location {
 125        const char *filename;
 126        int line_no;
 127        int column_no;
 128};
 129
 130/* The layout of struct dictated by compiler */
 131struct kasan_global {
 132        const void *beg;                /* Address of the beginning of the global variable. */
 133        size_t size;                    /* Size of the global variable. */
 134        size_t size_with_redzone;       /* Size of the variable + size of the red zone. 32 bytes aligned */
 135        const void *name;
 136        const void *module_name;        /* Name of the module where the global variable is declared. */
 137        unsigned long has_dynamic_init; /* This needed for C++ */
 138#if KASAN_ABI_VERSION >= 4
 139        struct kasan_source_location *location;
 140#endif
 141#if KASAN_ABI_VERSION >= 5
 142        char *odr_indicator;
 143#endif
 144};
 145
 146/**
 147 * Structures to keep alloc and free tracks *
 148 */
 149
 150#define KASAN_STACK_DEPTH 64
 151
 152struct kasan_track {
 153        u32 pid;
 154        depot_stack_handle_t stack;
 155};
 156
 157#if defined(CONFIG_KASAN_TAGS_IDENTIFY) && defined(CONFIG_KASAN_SW_TAGS)
 158#define KASAN_NR_FREE_STACKS 5
 159#else
 160#define KASAN_NR_FREE_STACKS 1
 161#endif
 162
 163struct kasan_alloc_meta {
 164        struct kasan_track alloc_track;
 165#ifdef CONFIG_KASAN_GENERIC
 166        /*
 167         * The auxiliary stack is stored into struct kasan_alloc_meta.
 168         * The free stack is stored into struct kasan_free_meta.
 169         */
 170        depot_stack_handle_t aux_stack[2];
 171#else
 172        struct kasan_track free_track[KASAN_NR_FREE_STACKS];
 173#endif
 174#ifdef CONFIG_KASAN_TAGS_IDENTIFY
 175        u8 free_pointer_tag[KASAN_NR_FREE_STACKS];
 176        u8 free_track_idx;
 177#endif
 178};
 179
 180struct qlist_node {
 181        struct qlist_node *next;
 182};
 183
 184/*
 185 * Generic mode either stores free meta in the object itself or in the redzone
 186 * after the object. In the former case free meta offset is 0, in the latter
 187 * case it has some sane value smaller than INT_MAX. Use INT_MAX as free meta
 188 * offset when free meta isn't present.
 189 */
 190#define KASAN_NO_FREE_META INT_MAX
 191
 192struct kasan_free_meta {
 193#ifdef CONFIG_KASAN_GENERIC
 194        /* This field is used while the object is in the quarantine.
 195         * Otherwise it might be used for the allocator freelist.
 196         */
 197        struct qlist_node quarantine_link;
 198        struct kasan_track free_track;
 199#endif
 200};
 201
 202struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
 203                                                const void *object);
 204#ifdef CONFIG_KASAN_GENERIC
 205struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
 206                                                const void *object);
 207#endif
 208
 209#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
 210
 211static inline const void *kasan_shadow_to_mem(const void *shadow_addr)
 212{
 213        return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET)
 214                << KASAN_SHADOW_SCALE_SHIFT);
 215}
 216
 217static inline bool addr_has_metadata(const void *addr)
 218{
 219        return (addr >= kasan_shadow_to_mem((void *)KASAN_SHADOW_START));
 220}
 221
 222/**
 223 * kasan_check_range - Check memory region, and report if invalid access.
 224 * @addr: the accessed address
 225 * @size: the accessed size
 226 * @write: true if access is a write access
 227 * @ret_ip: return address
 228 * @return: true if access was valid, false if invalid
 229 */
 230bool kasan_check_range(unsigned long addr, size_t size, bool write,
 231                                unsigned long ret_ip);
 232
 233#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
 234
 235static inline bool addr_has_metadata(const void *addr)
 236{
 237        return (is_vmalloc_addr(addr) || virt_addr_valid(addr));
 238}
 239
 240#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
 241
 242#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
 243void kasan_print_tags(u8 addr_tag, const void *addr);
 244#else
 245static inline void kasan_print_tags(u8 addr_tag, const void *addr) { }
 246#endif
 247
 248void *kasan_find_first_bad_addr(void *addr, size_t size);
 249const char *kasan_get_bug_type(struct kasan_access_info *info);
 250void kasan_metadata_fetch_row(char *buffer, void *row);
 251
 252#if defined(CONFIG_KASAN_GENERIC) && defined(CONFIG_KASAN_STACK)
 253void kasan_print_address_stack_frame(const void *addr);
 254#else
 255static inline void kasan_print_address_stack_frame(const void *addr) { }
 256#endif
 257
 258bool kasan_report(unsigned long addr, size_t size,
 259                bool is_write, unsigned long ip);
 260void kasan_report_invalid_free(void *object, unsigned long ip);
 261
 262struct page *kasan_addr_to_page(const void *addr);
 263
 264depot_stack_handle_t kasan_save_stack(gfp_t flags);
 265void kasan_set_track(struct kasan_track *track, gfp_t flags);
 266void kasan_set_free_info(struct kmem_cache *cache, void *object, u8 tag);
 267struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
 268                                void *object, u8 tag);
 269
 270#if defined(CONFIG_KASAN_GENERIC) && \
 271        (defined(CONFIG_SLAB) || defined(CONFIG_SLUB))
 272bool kasan_quarantine_put(struct kmem_cache *cache, void *object);
 273void kasan_quarantine_reduce(void);
 274void kasan_quarantine_remove_cache(struct kmem_cache *cache);
 275#else
 276static inline bool kasan_quarantine_put(struct kmem_cache *cache, void *object) { return false; }
 277static inline void kasan_quarantine_reduce(void) { }
 278static inline void kasan_quarantine_remove_cache(struct kmem_cache *cache) { }
 279#endif
 280
 281#ifndef arch_kasan_set_tag
 282static inline const void *arch_kasan_set_tag(const void *addr, u8 tag)
 283{
 284        return addr;
 285}
 286#endif
 287#ifndef arch_kasan_get_tag
 288#define arch_kasan_get_tag(addr)        0
 289#endif
 290
 291#define set_tag(addr, tag)      ((void *)arch_kasan_set_tag((addr), (tag)))
 292#define get_tag(addr)           arch_kasan_get_tag(addr)
 293
 294#ifdef CONFIG_KASAN_HW_TAGS
 295
 296#ifndef arch_enable_tagging_sync
 297#define arch_enable_tagging_sync()
 298#endif
 299#ifndef arch_enable_tagging_async
 300#define arch_enable_tagging_async()
 301#endif
 302#ifndef arch_init_tags
 303#define arch_init_tags(max_tag)
 304#endif
 305#ifndef arch_set_tagging_report_once
 306#define arch_set_tagging_report_once(state)
 307#endif
 308#ifndef arch_force_async_tag_fault
 309#define arch_force_async_tag_fault()
 310#endif
 311#ifndef arch_get_random_tag
 312#define arch_get_random_tag()   (0xFF)
 313#endif
 314#ifndef arch_get_mem_tag
 315#define arch_get_mem_tag(addr)  (0xFF)
 316#endif
 317#ifndef arch_set_mem_tag_range
 318#define arch_set_mem_tag_range(addr, size, tag, init) ((void *)(addr))
 319#endif
 320
 321#define hw_enable_tagging_sync()                arch_enable_tagging_sync()
 322#define hw_enable_tagging_async()               arch_enable_tagging_async()
 323#define hw_init_tags(max_tag)                   arch_init_tags(max_tag)
 324#define hw_set_tagging_report_once(state)       arch_set_tagging_report_once(state)
 325#define hw_force_async_tag_fault()              arch_force_async_tag_fault()
 326#define hw_get_random_tag()                     arch_get_random_tag()
 327#define hw_get_mem_tag(addr)                    arch_get_mem_tag(addr)
 328#define hw_set_mem_tag_range(addr, size, tag, init) \
 329                        arch_set_mem_tag_range((addr), (size), (tag), (init))
 330
 331#else /* CONFIG_KASAN_HW_TAGS */
 332
 333#define hw_enable_tagging_sync()
 334#define hw_enable_tagging_async()
 335#define hw_set_tagging_report_once(state)
 336
 337#endif /* CONFIG_KASAN_HW_TAGS */
 338
 339#if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
 340
 341void kasan_set_tagging_report_once(bool state);
 342void kasan_enable_tagging_sync(void);
 343void kasan_force_async_fault(void);
 344
 345#else /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */
 346
 347static inline void kasan_set_tagging_report_once(bool state) { }
 348static inline void kasan_enable_tagging_sync(void) { }
 349static inline void kasan_force_async_fault(void) { }
 350
 351#endif /* CONFIG_KASAN_HW_TAGS || CONFIG_KASAN_KUNIT_TEST */
 352
 353#ifdef CONFIG_KASAN_SW_TAGS
 354u8 kasan_random_tag(void);
 355#elif defined(CONFIG_KASAN_HW_TAGS)
 356static inline u8 kasan_random_tag(void) { return hw_get_random_tag(); }
 357#else
 358static inline u8 kasan_random_tag(void) { return 0; }
 359#endif
 360
 361#ifdef CONFIG_KASAN_HW_TAGS
 362
 363static inline void kasan_poison(const void *addr, size_t size, u8 value, bool init)
 364{
 365        addr = kasan_reset_tag(addr);
 366
 367        /* Skip KFENCE memory if called explicitly outside of sl*b. */
 368        if (is_kfence_address(addr))
 369                return;
 370
 371        if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
 372                return;
 373        if (WARN_ON(size & KASAN_GRANULE_MASK))
 374                return;
 375
 376        hw_set_mem_tag_range((void *)addr, size, value, init);
 377}
 378
 379static inline void kasan_unpoison(const void *addr, size_t size, bool init)
 380{
 381        u8 tag = get_tag(addr);
 382
 383        addr = kasan_reset_tag(addr);
 384
 385        /* Skip KFENCE memory if called explicitly outside of sl*b. */
 386        if (is_kfence_address(addr))
 387                return;
 388
 389        if (WARN_ON((unsigned long)addr & KASAN_GRANULE_MASK))
 390                return;
 391        /*
 392         * Explicitly initialize the memory with the precise object size to
 393         * avoid overwriting the SLAB redzone. This disables initialization in
 394         * the arch code and may thus lead to performance penalty. The penalty
 395         * is accepted since SLAB redzones aren't enabled in production builds.
 396         */
 397        if (__slub_debug_enabled() &&
 398            init && ((unsigned long)size & KASAN_GRANULE_MASK)) {
 399                init = false;
 400                memzero_explicit((void *)addr, size);
 401        }
 402        size = round_up(size, KASAN_GRANULE_SIZE);
 403
 404        hw_set_mem_tag_range((void *)addr, size, tag, init);
 405}
 406
 407static inline bool kasan_byte_accessible(const void *addr)
 408{
 409        u8 ptr_tag = get_tag(addr);
 410        u8 mem_tag = hw_get_mem_tag((void *)addr);
 411
 412        return ptr_tag == KASAN_TAG_KERNEL || ptr_tag == mem_tag;
 413}
 414
 415#else /* CONFIG_KASAN_HW_TAGS */
 416
 417/**
 418 * kasan_poison - mark the memory range as inaccessible
 419 * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
 420 * @size - range size, must be aligned to KASAN_GRANULE_SIZE
 421 * @value - value that's written to metadata for the range
 422 * @init - whether to initialize the memory range (only for hardware tag-based)
 423 *
 424 * The size gets aligned to KASAN_GRANULE_SIZE before marking the range.
 425 */
 426void kasan_poison(const void *addr, size_t size, u8 value, bool init);
 427
 428/**
 429 * kasan_unpoison - mark the memory range as accessible
 430 * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
 431 * @size - range size, can be unaligned
 432 * @init - whether to initialize the memory range (only for hardware tag-based)
 433 *
 434 * For the tag-based modes, the @size gets aligned to KASAN_GRANULE_SIZE before
 435 * marking the range.
 436 * For the generic mode, the last granule of the memory range gets partially
 437 * unpoisoned based on the @size.
 438 */
 439void kasan_unpoison(const void *addr, size_t size, bool init);
 440
 441bool kasan_byte_accessible(const void *addr);
 442
 443#endif /* CONFIG_KASAN_HW_TAGS */
 444
 445#ifdef CONFIG_KASAN_GENERIC
 446
 447/**
 448 * kasan_poison_last_granule - mark the last granule of the memory range as
 449 * inaccessible
 450 * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE
 451 * @size - range size
 452 *
 453 * This function is only available for the generic mode, as it's the only mode
 454 * that has partially poisoned memory granules.
 455 */
 456void kasan_poison_last_granule(const void *address, size_t size);
 457
 458#else /* CONFIG_KASAN_GENERIC */
 459
 460static inline void kasan_poison_last_granule(const void *address, size_t size) { }
 461
 462#endif /* CONFIG_KASAN_GENERIC */
 463
 464#ifndef kasan_arch_is_ready
 465static inline bool kasan_arch_is_ready(void)    { return true; }
 466#elif !defined(CONFIG_KASAN_GENERIC) || !defined(CONFIG_KASAN_OUTLINE)
 467#error kasan_arch_is_ready only works in KASAN generic outline mode!
 468#endif
 469
 470/*
 471 * Exported functions for interfaces called from assembly or from generated
 472 * code. Declarations here to avoid warning about missing declarations.
 473 */
 474asmlinkage void kasan_unpoison_task_stack_below(const void *watermark);
 475void __asan_register_globals(struct kasan_global *globals, size_t size);
 476void __asan_unregister_globals(struct kasan_global *globals, size_t size);
 477void __asan_handle_no_return(void);
 478void __asan_alloca_poison(unsigned long addr, size_t size);
 479void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom);
 480
 481void __asan_load1(unsigned long addr);
 482void __asan_store1(unsigned long addr);
 483void __asan_load2(unsigned long addr);
 484void __asan_store2(unsigned long addr);
 485void __asan_load4(unsigned long addr);
 486void __asan_store4(unsigned long addr);
 487void __asan_load8(unsigned long addr);
 488void __asan_store8(unsigned long addr);
 489void __asan_load16(unsigned long addr);
 490void __asan_store16(unsigned long addr);
 491void __asan_loadN(unsigned long addr, size_t size);
 492void __asan_storeN(unsigned long addr, size_t size);
 493
 494void __asan_load1_noabort(unsigned long addr);
 495void __asan_store1_noabort(unsigned long addr);
 496void __asan_load2_noabort(unsigned long addr);
 497void __asan_store2_noabort(unsigned long addr);
 498void __asan_load4_noabort(unsigned long addr);
 499void __asan_store4_noabort(unsigned long addr);
 500void __asan_load8_noabort(unsigned long addr);
 501void __asan_store8_noabort(unsigned long addr);
 502void __asan_load16_noabort(unsigned long addr);
 503void __asan_store16_noabort(unsigned long addr);
 504void __asan_loadN_noabort(unsigned long addr, size_t size);
 505void __asan_storeN_noabort(unsigned long addr, size_t size);
 506
 507void __asan_report_load1_noabort(unsigned long addr);
 508void __asan_report_store1_noabort(unsigned long addr);
 509void __asan_report_load2_noabort(unsigned long addr);
 510void __asan_report_store2_noabort(unsigned long addr);
 511void __asan_report_load4_noabort(unsigned long addr);
 512void __asan_report_store4_noabort(unsigned long addr);
 513void __asan_report_load8_noabort(unsigned long addr);
 514void __asan_report_store8_noabort(unsigned long addr);
 515void __asan_report_load16_noabort(unsigned long addr);
 516void __asan_report_store16_noabort(unsigned long addr);
 517void __asan_report_load_n_noabort(unsigned long addr, size_t size);
 518void __asan_report_store_n_noabort(unsigned long addr, size_t size);
 519
 520void __asan_set_shadow_00(const void *addr, size_t size);
 521void __asan_set_shadow_f1(const void *addr, size_t size);
 522void __asan_set_shadow_f2(const void *addr, size_t size);
 523void __asan_set_shadow_f3(const void *addr, size_t size);
 524void __asan_set_shadow_f5(const void *addr, size_t size);
 525void __asan_set_shadow_f8(const void *addr, size_t size);
 526
 527void __hwasan_load1_noabort(unsigned long addr);
 528void __hwasan_store1_noabort(unsigned long addr);
 529void __hwasan_load2_noabort(unsigned long addr);
 530void __hwasan_store2_noabort(unsigned long addr);
 531void __hwasan_load4_noabort(unsigned long addr);
 532void __hwasan_store4_noabort(unsigned long addr);
 533void __hwasan_load8_noabort(unsigned long addr);
 534void __hwasan_store8_noabort(unsigned long addr);
 535void __hwasan_load16_noabort(unsigned long addr);
 536void __hwasan_store16_noabort(unsigned long addr);
 537void __hwasan_loadN_noabort(unsigned long addr, size_t size);
 538void __hwasan_storeN_noabort(unsigned long addr, size_t size);
 539
 540void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size);
 541
 542#endif
 543