linux/arch/arm64/lib/mte.S
<<
>>
Prefs
   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Copyright (C) 2020 ARM Ltd.
   4 */
   5#include <linux/linkage.h>
   6
   7#include <asm/asm-uaccess.h>
   8#include <asm/assembler.h>
   9#include <asm/mte.h>
  10#include <asm/page.h>
  11#include <asm/sysreg.h>
  12
  13        .arch   armv8.5-a+memtag
  14
  15/*
  16 * multitag_transfer_size - set \reg to the block size that is accessed by the
  17 * LDGM/STGM instructions.
  18 */
  19        .macro  multitag_transfer_size, reg, tmp
  20        mrs_s   \reg, SYS_GMID_EL1
  21        ubfx    \reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE
  22        mov     \tmp, #4
  23        lsl     \reg, \tmp, \reg
  24        .endm
  25
  26/*
  27 * Clear the tags in a page
  28 *   x0 - address of the page to be cleared
  29 */
  30SYM_FUNC_START(mte_clear_page_tags)
  31        multitag_transfer_size x1, x2
  321:      stgm    xzr, [x0]
  33        add     x0, x0, x1
  34        tst     x0, #(PAGE_SIZE - 1)
  35        b.ne    1b
  36        ret
  37SYM_FUNC_END(mte_clear_page_tags)
  38
  39/*
  40 * Zero the page and tags at the same time
  41 *
  42 * Parameters:
  43 *      x0 - address to the beginning of the page
  44 */
  45SYM_FUNC_START(mte_zero_clear_page_tags)
  46        mrs     x1, dczid_el0
  47        and     w1, w1, #0xf
  48        mov     x2, #4
  49        lsl     x1, x2, x1
  50        and     x0, x0, #(1 << MTE_TAG_SHIFT) - 1       // clear the tag
  51
  521:      dc      gzva, x0
  53        add     x0, x0, x1
  54        tst     x0, #(PAGE_SIZE - 1)
  55        b.ne    1b
  56        ret
  57SYM_FUNC_END(mte_zero_clear_page_tags)
  58
  59/*
  60 * Copy the tags from the source page to the destination one
  61 *   x0 - address of the destination page
  62 *   x1 - address of the source page
  63 */
  64SYM_FUNC_START(mte_copy_page_tags)
  65        mov     x2, x0
  66        mov     x3, x1
  67        multitag_transfer_size x5, x6
  681:      ldgm    x4, [x3]
  69        stgm    x4, [x2]
  70        add     x2, x2, x5
  71        add     x3, x3, x5
  72        tst     x2, #(PAGE_SIZE - 1)
  73        b.ne    1b
  74        ret
  75SYM_FUNC_END(mte_copy_page_tags)
  76
  77/*
  78 * Read tags from a user buffer (one tag per byte) and set the corresponding
  79 * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
  80 *   x0 - kernel address (to)
  81 *   x1 - user buffer (from)
  82 *   x2 - number of tags/bytes (n)
  83 * Returns:
  84 *   x0 - number of tags read/set
  85 */
  86SYM_FUNC_START(mte_copy_tags_from_user)
  87        mov     x3, x1
  88        cbz     x2, 2f
  891:
  90        user_ldst 2f, ldtrb, w4, x1, 0
  91        lsl     x4, x4, #MTE_TAG_SHIFT
  92        stg     x4, [x0], #MTE_GRANULE_SIZE
  93        add     x1, x1, #1
  94        subs    x2, x2, #1
  95        b.ne    1b
  96
  97        // exception handling and function return
  982:      sub     x0, x1, x3              // update the number of tags set
  99        ret
 100SYM_FUNC_END(mte_copy_tags_from_user)
 101
 102/*
 103 * Get the tags from a kernel address range and write the tag values to the
 104 * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
 105 *   x0 - user buffer (to)
 106 *   x1 - kernel address (from)
 107 *   x2 - number of tags/bytes (n)
 108 * Returns:
 109 *   x0 - number of tags read/set
 110 */
 111SYM_FUNC_START(mte_copy_tags_to_user)
 112        mov     x3, x0
 113        cbz     x2, 2f
 1141:
 115        ldg     x4, [x1]
 116        ubfx    x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
 117        user_ldst 2f, sttrb, w4, x0, 0
 118        add     x0, x0, #1
 119        add     x1, x1, #MTE_GRANULE_SIZE
 120        subs    x2, x2, #1
 121        b.ne    1b
 122
 123        // exception handling and function return
 1242:      sub     x0, x0, x3              // update the number of tags copied
 125        ret
 126SYM_FUNC_END(mte_copy_tags_to_user)
 127
 128/*
 129 * Save the tags in a page
 130 *   x0 - page address
 131 *   x1 - tag storage
 132 */
 133SYM_FUNC_START(mte_save_page_tags)
 134        multitag_transfer_size x7, x5
 1351:
 136        mov     x2, #0
 1372:
 138        ldgm    x5, [x0]
 139        orr     x2, x2, x5
 140        add     x0, x0, x7
 141        tst     x0, #0xFF               // 16 tag values fit in a register,
 142        b.ne    2b                      // which is 16*16=256 bytes
 143
 144        str     x2, [x1], #8
 145
 146        tst     x0, #(PAGE_SIZE - 1)
 147        b.ne    1b
 148
 149        ret
 150SYM_FUNC_END(mte_save_page_tags)
 151
 152/*
 153 * Restore the tags in a page
 154 *   x0 - page address
 155 *   x1 - tag storage
 156 */
 157SYM_FUNC_START(mte_restore_page_tags)
 158        multitag_transfer_size x7, x5
 1591:
 160        ldr     x2, [x1], #8
 1612:
 162        stgm    x2, [x0]
 163        add     x0, x0, x7
 164        tst     x0, #0xFF
 165        b.ne    2b
 166
 167        tst     x0, #(PAGE_SIZE - 1)
 168        b.ne    1b
 169
 170        ret
 171SYM_FUNC_END(mte_restore_page_tags)
 172