LXR linux/arch/arm64/include/asm/uaccess.h

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Based on arch/arm/include/asm/uaccess.h
   4 *
   5 * Copyright (C) 2012 ARM Ltd.
   6 */
   7#ifndef __ASM_UACCESS_H
   8#define __ASM_UACCESS_H
   9
  10#include <asm/alternative.h>
  11#include <asm/kernel-pgtable.h>
  12#include <asm/sysreg.h>
  13
  14/*
  15 * User space memory access functions
  16 */
  17#include <linux/bitops.h>
  18#include <linux/kasan-checks.h>
  19#include <linux/string.h>
  20
  21#include <asm/cpufeature.h>
  22#include <asm/mmu.h>
  23#include <asm/ptrace.h>
  24#include <asm/memory.h>
  25#include <asm/extable.h>
  26
  27#define get_fs()        (current_thread_info()->addr_limit)
  28
  29static inline void set_fs(mm_segment_t fs)
  30{
  31        current_thread_info()->addr_limit = fs;
  32
  33        /*
  34         * Prevent a mispredicted conditional call to set_fs from forwarding
  35         * the wrong address limit to access_ok under speculation.
  36         */
  37        spec_bar();
  38
  39        /* On user-mode return, check fs is correct */
  40        set_thread_flag(TIF_FSCHECK);
  41
  42        /*
  43         * Enable/disable UAO so that copy_to_user() etc can access
  44         * kernel memory with the unprivileged instructions.
  45         */
  46        if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS)
  47                asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
  48        else
  49                asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO,
  50                                CONFIG_ARM64_UAO));
  51}
  52
  53#define uaccess_kernel()        (get_fs() == KERNEL_DS)
  54
  55/*
  56 * Test whether a block of memory is a valid user space address.
  57 * Returns 1 if the range is valid, 0 otherwise.
  58 *
  59 * This is equivalent to the following test:
  60 * (u65)addr + (u65)size <= (u65)current->addr_limit + 1
  61 */
  62static inline unsigned long __range_ok(const void __user *addr, unsigned long size)
  63{
  64        unsigned long ret, limit = current_thread_info()->addr_limit;
  65
  66        /*
  67         * Asynchronous I/O running in a kernel thread does not have the
  68         * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag
  69         * the user address before checking.
  70         */
  71        if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) &&
  72            (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR)))
  73                addr = untagged_addr(addr);
  74
  75        __chk_user_ptr(addr);
  76        asm volatile(
  77        // A + B <= C + 1 for all A,B,C, in four easy steps:
  78        // 1: X = A + B; X' = X % 2^64
  79        "       adds    %0, %3, %2\n"
  80        // 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4
  81        "       csel    %1, xzr, %1, hi\n"
  82        // 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X'
  83        //    to compensate for the carry flag being set in step 4. For
  84        //    X > 2^64, X' merely has to remain nonzero, which it does.
  85        "       csinv   %0, %0, xzr, cc\n"
  86        // 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1
  87        //    comes from the carry in being clear. Otherwise, we are
  88        //    testing X' - C == 0, subject to the previous adjustments.
  89        "       sbcs    xzr, %0, %1\n"
  90        "       cset    %0, ls\n"
  91        : "=&r" (ret), "+r" (limit) : "Ir" (size), "0" (addr) : "cc");
  92
  93        return ret;
  94}
  95
  96#define access_ok(addr, size)   __range_ok(addr, size)
  97#define user_addr_max                   get_fs
  98
  99#define _ASM_EXTABLE(from, to)                                          \
 100        "       .pushsection    __ex_table, \"a\"\n"                    \
 101        "       .align          3\n"                                    \
 102        "       .long           (" #from " - .), (" #to " - .)\n"       \
 103        "       .popsection\n"
 104
 105/*
 106 * User access enabling/disabling.
 107 */
 108#ifdef CONFIG_ARM64_SW_TTBR0_PAN
 109static inline void __uaccess_ttbr0_disable(void)
 110{
 111        unsigned long flags, ttbr;
 112
 113        local_irq_save(flags);
 114        ttbr = read_sysreg(ttbr1_el1);
 115        ttbr &= ~TTBR_ASID_MASK;
 116        /* reserved_ttbr0 placed before swapper_pg_dir */
 117        write_sysreg(ttbr - RESERVED_TTBR0_SIZE, ttbr0_el1);
 118        isb();
 119        /* Set reserved ASID */
 120        write_sysreg(ttbr, ttbr1_el1);
 121        isb();
 122        local_irq_restore(flags);
 123}
 124
 125static inline void __uaccess_ttbr0_enable(void)
 126{
 127        unsigned long flags, ttbr0, ttbr1;
 128
 129        /*
 130         * Disable interrupts to avoid preemption between reading the 'ttbr0'
 131         * variable and the MSR. A context switch could trigger an ASID
 132         * roll-over and an update of 'ttbr0'.
 133         */
 134        local_irq_save(flags);
 135        ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
 136
 137        /* Restore active ASID */
 138        ttbr1 = read_sysreg(ttbr1_el1);
 139        ttbr1 &= ~TTBR_ASID_MASK;               /* safety measure */
 140        ttbr1 |= ttbr0 & TTBR_ASID_MASK;
 141        write_sysreg(ttbr1, ttbr1_el1);
 142        isb();
 143
 144        /* Restore user page table */
 145        write_sysreg(ttbr0, ttbr0_el1);
 146        isb();
 147        local_irq_restore(flags);
 148}
 149
 150static inline bool uaccess_ttbr0_disable(void)
 151{
 152        if (!system_uses_ttbr0_pan())
 153                return false;
 154        __uaccess_ttbr0_disable();
 155        return true;
 156}
 157
 158static inline bool uaccess_ttbr0_enable(void)
 159{
 160        if (!system_uses_ttbr0_pan())
 161                return false;
 162        __uaccess_ttbr0_enable();
 163        return true;
 164}
 165#else
 166static inline bool uaccess_ttbr0_disable(void)
 167{
 168        return false;
 169}
 170
 171static inline bool uaccess_ttbr0_enable(void)
 172{
 173        return false;
 174}
 175#endif
 176
 177static inline void __uaccess_disable_hw_pan(void)
 178{
 179        asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,
 180                        CONFIG_ARM64_PAN));
 181}
 182
 183static inline void __uaccess_enable_hw_pan(void)
 184{
 185        asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,
 186                        CONFIG_ARM64_PAN));
 187}
 188
 189#define __uaccess_disable(alt)                                          \
 190do {                                                                    \
 191        if (!uaccess_ttbr0_disable())                                   \
 192                asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt,          \
 193                                CONFIG_ARM64_PAN));                     \
 194} while (0)
 195
 196#define __uaccess_enable(alt)                                           \
 197do {                                                                    \
 198        if (!uaccess_ttbr0_enable())                                    \
 199                asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt,          \
 200                                CONFIG_ARM64_PAN));                     \
 201} while (0)
 202
 203static inline void uaccess_disable(void)
 204{
 205        __uaccess_disable(ARM64_HAS_PAN);
 206}
 207
 208static inline void uaccess_enable(void)
 209{
 210        __uaccess_enable(ARM64_HAS_PAN);
 211}
 212
 213/*
 214 * These functions are no-ops when UAO is present.
 215 */
 216static inline void uaccess_disable_not_uao(void)
 217{
 218        __uaccess_disable(ARM64_ALT_PAN_NOT_UAO);
 219}
 220
 221static inline void uaccess_enable_not_uao(void)
 222{
 223        __uaccess_enable(ARM64_ALT_PAN_NOT_UAO);
 224}
 225
 226/*
 227 * Sanitise a uaccess pointer such that it becomes NULL if above the
 228 * current addr_limit. In case the pointer is tagged (has the top byte set),
 229 * untag the pointer before checking.
 230 */
 231#define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
 232static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
 233{
 234        void __user *safe_ptr;
 235
 236        asm volatile(
 237        "       bics    xzr, %3, %2\n"
 238        "       csel    %0, %1, xzr, eq\n"
 239        : "=&r" (safe_ptr)
 240        : "r" (ptr), "r" (current_thread_info()->addr_limit),
 241          "r" (untagged_addr(ptr))
 242        : "cc");
 243
 244        csdb();
 245        return safe_ptr;
 246}
 247
 248/*
 249 * The "__xxx" versions of the user access functions do not verify the address
 250 * space - it must have been done previously with a separate "access_ok()"
 251 * call.
 252 *
 253 * The "__xxx_error" versions set the third argument to -EFAULT if an error
 254 * occurs, and leave it unchanged on success.
 255 */
 256#define __get_user_asm(instr, alt_instr, reg, x, addr, err, feature)    \
 257        asm volatile(                                                   \
 258        "1:"ALTERNATIVE(instr "     " reg "1, [%2]\n",                  \
 259                        alt_instr " " reg "1, [%2]\n", feature)         \
 260        "2:\n"                                                          \
 261        "       .section .fixup, \"ax\"\n"                              \
 262        "       .align  2\n"                                            \
 263        "3:     mov     %w0, %3\n"                                      \
 264        "       mov     %1, #0\n"                                       \
 265        "       b       2b\n"                                           \
 266        "       .previous\n"                                            \
 267        _ASM_EXTABLE(1b, 3b)                                            \
 268        : "+r" (err), "=&r" (x)                                         \
 269        : "r" (addr), "i" (-EFAULT))
 270
 271#define __raw_get_user(x, ptr, err)                                     \
 272do {                                                                    \
 273        unsigned long __gu_val;                                         \
 274        __chk_user_ptr(ptr);                                            \
 275        uaccess_enable_not_uao();                                       \
 276        switch (sizeof(*(ptr))) {                                       \
 277        case 1:                                                         \
 278                __get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr),  \
 279                               (err), ARM64_HAS_UAO);                   \
 280                break;                                                  \
 281        case 2:                                                         \
 282                __get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr),  \
 283                               (err), ARM64_HAS_UAO);                   \
 284                break;                                                  \
 285        case 4:                                                         \
 286                __get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr),    \
 287                               (err), ARM64_HAS_UAO);                   \
 288                break;                                                  \
 289        case 8:                                                         \
 290                __get_user_asm("ldr", "ldtr", "%x",  __gu_val, (ptr),   \
 291                               (err), ARM64_HAS_UAO);                   \
 292                break;                                                  \
 293        default:                                                        \
 294                BUILD_BUG();                                            \
 295        }                                                               \
 296        uaccess_disable_not_uao();                                      \
 297        (x) = (__force __typeof__(*(ptr)))__gu_val;                     \
 298} while (0)
 299
 300#define __get_user_error(x, ptr, err)                                   \
 301do {                                                                    \
 302        __typeof__(*(ptr)) __user *__p = (ptr);                         \
 303        might_fault();                                                  \
 304        if (access_ok(__p, sizeof(*__p))) {                             \
 305                __p = uaccess_mask_ptr(__p);                            \
 306                __raw_get_user((x), __p, (err));                        \
 307        } else {                                                        \
 308                (x) = (__force __typeof__(x))0; (err) = -EFAULT;        \
 309        }                                                               \
 310} while (0)
 311
 312#define __get_user(x, ptr)                                              \
 313({                                                                      \
 314        int __gu_err = 0;                                               \
 315        __get_user_error((x), (ptr), __gu_err);                         \
 316        __gu_err;                                                       \
 317})
 318
 319#define get_user        __get_user
 320
 321#define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature)    \
 322        asm volatile(                                                   \
 323        "1:"ALTERNATIVE(instr "     " reg "1, [%2]\n",                  \
 324                        alt_instr " " reg "1, [%2]\n", feature)         \
 325        "2:\n"                                                          \
 326        "       .section .fixup,\"ax\"\n"                               \
 327        "       .align  2\n"                                            \
 328        "3:     mov     %w0, %3\n"                                      \
 329        "       b       2b\n"                                           \
 330        "       .previous\n"                                            \
 331        _ASM_EXTABLE(1b, 3b)                                            \
 332        : "+r" (err)                                                    \
 333        : "r" (x), "r" (addr), "i" (-EFAULT))
 334
 335#define __raw_put_user(x, ptr, err)                                     \
 336do {                                                                    \
 337        __typeof__(*(ptr)) __pu_val = (x);                              \
 338        __chk_user_ptr(ptr);                                            \
 339        uaccess_enable_not_uao();                                       \
 340        switch (sizeof(*(ptr))) {                                       \
 341        case 1:                                                         \
 342                __put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr),  \
 343                               (err), ARM64_HAS_UAO);                   \
 344                break;                                                  \
 345        case 2:                                                         \
 346                __put_user_asm("strh", "sttrh", "%w", __pu_val, (ptr),  \
 347                               (err), ARM64_HAS_UAO);                   \
 348                break;                                                  \
 349        case 4:                                                         \
 350                __put_user_asm("str", "sttr", "%w", __pu_val, (ptr),    \
 351                               (err), ARM64_HAS_UAO);                   \
 352                break;                                                  \
 353        case 8:                                                         \
 354                __put_user_asm("str", "sttr", "%x", __pu_val, (ptr),    \
 355                               (err), ARM64_HAS_UAO);                   \
 356                break;                                                  \
 357        default:                                                        \
 358                BUILD_BUG();                                            \
 359        }                                                               \
 360        uaccess_disable_not_uao();                                      \
 361} while (0)
 362
 363#define __put_user_error(x, ptr, err)                                   \
 364do {                                                                    \
 365        __typeof__(*(ptr)) __user *__p = (ptr);                         \
 366        might_fault();                                                  \
 367        if (access_ok(__p, sizeof(*__p))) {                             \
 368                __p = uaccess_mask_ptr(__p);                            \
 369                __raw_put_user((x), __p, (err));                        \
 370        } else  {                                                       \
 371                (err) = -EFAULT;                                        \
 372        }                                                               \
 373} while (0)
 374
 375#define __put_user(x, ptr)                                              \
 376({                                                                      \
 377        int __pu_err = 0;                                               \
 378        __put_user_error((x), (ptr), __pu_err);                         \
 379        __pu_err;                                                       \
 380})
 381
 382#define put_user        __put_user
 383
 384extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
 385#define raw_copy_from_user(to, from, n)                                 \
 386({                                                                      \
 387        unsigned long __acfu_ret;                                       \
 388        uaccess_enable_not_uao();                                       \
 389        __acfu_ret = __arch_copy_from_user((to),                        \
 390                                      __uaccess_mask_ptr(from), (n));   \
 391        uaccess_disable_not_uao();                                      \
 392        __acfu_ret;                                                     \
 393})
 394
 395extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
 396#define raw_copy_to_user(to, from, n)                                   \
 397({                                                                      \
 398        unsigned long __actu_ret;                                       \
 399        uaccess_enable_not_uao();                                       \
 400        __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),        \
 401                                    (from), (n));                       \
 402        uaccess_disable_not_uao();                                      \
 403        __actu_ret;                                                     \
 404})
 405
 406extern unsigned long __must_check __arch_copy_in_user(void __user *to, const void __user *from, unsigned long n);
 407#define raw_copy_in_user(to, from, n)                                   \
 408({                                                                      \
 409        unsigned long __aciu_ret;                                       \
 410        uaccess_enable_not_uao();                                       \
 411        __aciu_ret = __arch_copy_in_user(__uaccess_mask_ptr(to),        \
 412                                    __uaccess_mask_ptr(from), (n));     \
 413        uaccess_disable_not_uao();                                      \
 414        __aciu_ret;                                                     \
 415})
 416
 417#define INLINE_COPY_TO_USER
 418#define INLINE_COPY_FROM_USER
 419
 420extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n);
 421static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
 422{
 423        if (access_ok(to, n)) {
 424                uaccess_enable_not_uao();
 425                n = __arch_clear_user(__uaccess_mask_ptr(to), n);
 426                uaccess_disable_not_uao();
 427        }
 428        return n;
 429}
 430#define clear_user      __clear_user
 431
 432extern long strncpy_from_user(char *dest, const char __user *src, long count);
 433
 434extern __must_check long strnlen_user(const char __user *str, long n);
 435
 436#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 437struct page;
 438void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len);
 439extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
 440
 441static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
 442{
 443        kasan_check_write(dst, size);
 444        return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
 445}
 446#endif
 447
 448#endif /* __ASM_UACCESS_H */
 449