LXR linux/mm/kasan/hw

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * This file contains core hardware tag-based KASAN code.
   4 *
   5 * Copyright (c) 2020 Google, Inc.
   6 * Author: Andrey Konovalov <andreyknvl@google.com>
   7 */
   8
   9#define pr_fmt(fmt) "kasan: " fmt
  10
  11#include <linux/init.h>
  12#include <linux/kasan.h>
  13#include <linux/kernel.h>
  14#include <linux/memory.h>
  15#include <linux/mm.h>
  16#include <linux/static_key.h>
  17#include <linux/string.h>
  18#include <linux/types.h>
  19
  20#include "kasan.h"
  21
  22enum kasan_arg {
  23        KASAN_ARG_DEFAULT,
  24        KASAN_ARG_OFF,
  25        KASAN_ARG_ON,
  26};
  27
  28enum kasan_arg_mode {
  29        KASAN_ARG_MODE_DEFAULT,
  30        KASAN_ARG_MODE_SYNC,
  31        KASAN_ARG_MODE_ASYNC,
  32};
  33
  34enum kasan_arg_stacktrace {
  35        KASAN_ARG_STACKTRACE_DEFAULT,
  36        KASAN_ARG_STACKTRACE_OFF,
  37        KASAN_ARG_STACKTRACE_ON,
  38};
  39
  40static enum kasan_arg kasan_arg __ro_after_init;
  41static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
  42static enum kasan_arg_stacktrace kasan_arg_stacktrace __ro_after_init;
  43
  44/* Whether KASAN is enabled at all. */
  45DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
  46EXPORT_SYMBOL(kasan_flag_enabled);
  47
  48/* Whether the asynchronous mode is enabled. */
  49bool kasan_flag_async __ro_after_init;
  50EXPORT_SYMBOL_GPL(kasan_flag_async);
  51
  52/* Whether to collect alloc/free stack traces. */
  53DEFINE_STATIC_KEY_FALSE(kasan_flag_stacktrace);
  54
  55/* kasan=off/on */
  56static int __init early_kasan_flag(char *arg)
  57{
  58        if (!arg)
  59                return -EINVAL;
  60
  61        if (!strcmp(arg, "off"))
  62                kasan_arg = KASAN_ARG_OFF;
  63        else if (!strcmp(arg, "on"))
  64                kasan_arg = KASAN_ARG_ON;
  65        else
  66                return -EINVAL;
  67
  68        return 0;
  69}
  70early_param("kasan", early_kasan_flag);
  71
  72/* kasan.mode=sync/async */
  73static int __init early_kasan_mode(char *arg)
  74{
  75        if (!arg)
  76                return -EINVAL;
  77
  78        if (!strcmp(arg, "sync"))
  79                kasan_arg_mode = KASAN_ARG_MODE_SYNC;
  80        else if (!strcmp(arg, "async"))
  81                kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
  82        else
  83                return -EINVAL;
  84
  85        return 0;
  86}
  87early_param("kasan.mode", early_kasan_mode);
  88
  89/* kasan.stacktrace=off/on */
  90static int __init early_kasan_flag_stacktrace(char *arg)
  91{
  92        if (!arg)
  93                return -EINVAL;
  94
  95        if (!strcmp(arg, "off"))
  96                kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
  97        else if (!strcmp(arg, "on"))
  98                kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
  99        else
 100                return -EINVAL;
 101
 102        return 0;
 103}
 104early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
 105
 106/* kasan_init_hw_tags_cpu() is called for each CPU. */
 107void kasan_init_hw_tags_cpu(void)
 108{
 109        /*
 110         * There's no need to check that the hardware is MTE-capable here,
 111         * as this function is only called for MTE-capable hardware.
 112         */
 113
 114        /* If KASAN is disabled via command line, don't initialize it. */
 115        if (kasan_arg == KASAN_ARG_OFF)
 116                return;
 117
 118        /*
 119         * Enable async mode only when explicitly requested through
 120         * the command line.
 121         */
 122        if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
 123                hw_enable_tagging_async();
 124        else
 125                hw_enable_tagging_sync();
 126}
 127
 128/* kasan_init_hw_tags() is called once on boot CPU. */
 129void __init kasan_init_hw_tags(void)
 130{
 131        /* If hardware doesn't support MTE, don't initialize KASAN. */
 132        if (!system_supports_mte())
 133                return;
 134
 135        /* If KASAN is disabled via command line, don't initialize it. */
 136        if (kasan_arg == KASAN_ARG_OFF)
 137                return;
 138
 139        /* Enable KASAN. */
 140        static_branch_enable(&kasan_flag_enabled);
 141
 142        switch (kasan_arg_mode) {
 143        case KASAN_ARG_MODE_DEFAULT:
 144                /*
 145                 * Default to sync mode.
 146                 * Do nothing, kasan_flag_async keeps its default value.
 147                 */
 148                break;
 149        case KASAN_ARG_MODE_SYNC:
 150                /* Do nothing, kasan_flag_async keeps its default value. */
 151                break;
 152        case KASAN_ARG_MODE_ASYNC:
 153                /* Async mode enabled. */
 154                kasan_flag_async = true;
 155                break;
 156        }
 157
 158        switch (kasan_arg_stacktrace) {
 159        case KASAN_ARG_STACKTRACE_DEFAULT:
 160                /* Default to enabling stack trace collection. */
 161                static_branch_enable(&kasan_flag_stacktrace);
 162                break;
 163        case KASAN_ARG_STACKTRACE_OFF:
 164                /* Do nothing, kasan_flag_stacktrace keeps its default value. */
 165                break;
 166        case KASAN_ARG_STACKTRACE_ON:
 167                static_branch_enable(&kasan_flag_stacktrace);
 168                break;
 169        }
 170
 171        pr_info("KernelAddressSanitizer initialized\n");
 172}
 173
 174void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags)
 175{
 176        /*
 177         * This condition should match the one in post_alloc_hook() in
 178         * page_alloc.c.
 179         */
 180        bool init = !want_init_on_free() && want_init_on_alloc(flags);
 181
 182        if (flags & __GFP_SKIP_KASAN_POISON)
 183                SetPageSkipKASanPoison(page);
 184
 185        if (flags & __GFP_ZEROTAGS) {
 186                int i;
 187
 188                for (i = 0; i != 1 << order; ++i)
 189                        tag_clear_highpage(page + i);
 190        } else {
 191                kasan_unpoison_pages(page, order, init);
 192        }
 193}
 194
 195void kasan_free_pages(struct page *page, unsigned int order)
 196{
 197        /*
 198         * This condition should match the one in free_pages_prepare() in
 199         * page_alloc.c.
 200         */
 201        bool init = want_init_on_free();
 202
 203        kasan_poison_pages(page, order, init);
 204}
 205
 206#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
 207
 208void kasan_enable_tagging_sync(void)
 209{
 210        hw_enable_tagging_sync();
 211}
 212EXPORT_SYMBOL_GPL(kasan_enable_tagging_sync);
 213
 214void kasan_force_async_fault(void)
 215{
 216        hw_force_async_tag_fault();
 217}
 218EXPORT_SYMBOL_GPL(kasan_force_async_fault);
 219
 220#endif
 221