LXR linux/arch/riscv/kernel/patch.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2020 SiFive
   4 */
   5
   6#include <linux/spinlock.h>
   7#include <linux/mm.h>
   8#include <linux/memory.h>
   9#include <linux/uaccess.h>
  10#include <linux/stop_machine.h>
  11#include <asm/kprobes.h>
  12#include <asm/cacheflush.h>
  13#include <asm/fixmap.h>
  14#include <asm/patch.h>
  15
  16struct patch_insn {
  17        void *addr;
  18        u32 insn;
  19        atomic_t cpu_count;
  20};
  21
  22#ifdef CONFIG_MMU
  23static void *patch_map(void *addr, int fixmap)
  24{
  25        uintptr_t uintaddr = (uintptr_t) addr;
  26        struct page *page;
  27
  28        if (core_kernel_text(uintaddr))
  29                page = phys_to_page(__pa_symbol(addr));
  30        else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
  31                page = vmalloc_to_page(addr);
  32        else
  33                return addr;
  34
  35        BUG_ON(!page);
  36
  37        return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
  38                                         (uintaddr & ~PAGE_MASK));
  39}
  40NOKPROBE_SYMBOL(patch_map);
  41
  42static void patch_unmap(int fixmap)
  43{
  44        clear_fixmap(fixmap);
  45}
  46NOKPROBE_SYMBOL(patch_unmap);
  47
  48static int patch_insn_write(void *addr, const void *insn, size_t len)
  49{
  50        void *waddr = addr;
  51        bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
  52        int ret;
  53
  54        /*
  55         * Before reaching here, it was expected to lock the text_mutex
  56         * already, so we don't need to give another lock here and could
  57         * ensure that it was safe between each cores.
  58         */
  59        lockdep_assert_held(&text_mutex);
  60
  61        if (across_pages)
  62                patch_map(addr + len, FIX_TEXT_POKE1);
  63
  64        waddr = patch_map(addr, FIX_TEXT_POKE0);
  65
  66        ret = copy_to_kernel_nofault(waddr, insn, len);
  67
  68        patch_unmap(FIX_TEXT_POKE0);
  69
  70        if (across_pages)
  71                patch_unmap(FIX_TEXT_POKE1);
  72
  73        return ret;
  74}
  75NOKPROBE_SYMBOL(patch_insn_write);
  76#else
  77static int patch_insn_write(void *addr, const void *insn, size_t len)
  78{
  79        return copy_to_kernel_nofault(addr, insn, len);
  80}
  81NOKPROBE_SYMBOL(patch_insn_write);
  82#endif /* CONFIG_MMU */
  83
  84int patch_text_nosync(void *addr, const void *insns, size_t len)
  85{
  86        u32 *tp = addr;
  87        int ret;
  88
  89        ret = patch_insn_write(tp, insns, len);
  90
  91        if (!ret)
  92                flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
  93
  94        return ret;
  95}
  96NOKPROBE_SYMBOL(patch_text_nosync);
  97
  98static int patch_text_cb(void *data)
  99{
 100        struct patch_insn *patch = data;
 101        int ret = 0;
 102
 103        if (atomic_inc_return(&patch->cpu_count) == 1) {
 104                ret =
 105                    patch_text_nosync(patch->addr, &patch->insn,
 106                                            GET_INSN_LENGTH(patch->insn));
 107                atomic_inc(&patch->cpu_count);
 108        } else {
 109                while (atomic_read(&patch->cpu_count) <= num_online_cpus())
 110                        cpu_relax();
 111                smp_mb();
 112        }
 113
 114        return ret;
 115}
 116NOKPROBE_SYMBOL(patch_text_cb);
 117
 118int patch_text(void *addr, u32 insn)
 119{
 120        struct patch_insn patch = {
 121                .addr = addr,
 122                .insn = insn,
 123                .cpu_count = ATOMIC_INIT(0),
 124        };
 125
 126        return stop_machine_cpuslocked(patch_text_cb,
 127                                       &patch, cpu_online_mask);
 128}
 129NOKPROBE_SYMBOL(patch_text);
 130