LXR linux/arch/s390/lib/mem.S

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * String handling functions.
   4 *
   5 * Copyright IBM Corp. 2012
   6 */
   7
   8#include <linux/linkage.h>
   9#include <asm/export.h>
  10#include <asm/nospec-insn.h>
  11
  12        GEN_BR_THUNK %r14
  13
  14/*
  15 * void *memmove(void *dest, const void *src, size_t n)
  16 */
  17WEAK(memmove)
  18ENTRY(__memmove)
  19        ltgr    %r4,%r4
  20        lgr     %r1,%r2
  21        jz      .Lmemmove_exit
  22        aghi    %r4,-1
  23        clgr    %r2,%r3
  24        jnh     .Lmemmove_forward
  25        la      %r5,1(%r4,%r3)
  26        clgr    %r2,%r5
  27        jl      .Lmemmove_reverse
  28.Lmemmove_forward:
  29        srlg    %r0,%r4,8
  30        ltgr    %r0,%r0
  31        jz      .Lmemmove_forward_remainder
  32.Lmemmove_forward_loop:
  33        mvc     0(256,%r1),0(%r3)
  34        la      %r1,256(%r1)
  35        la      %r3,256(%r3)
  36        brctg   %r0,.Lmemmove_forward_loop
  37.Lmemmove_forward_remainder:
  38        larl    %r5,.Lmemmove_mvc
  39        ex      %r4,0(%r5)
  40.Lmemmove_exit:
  41        BR_EX   %r14
  42.Lmemmove_reverse:
  43        ic      %r0,0(%r4,%r3)
  44        stc     %r0,0(%r4,%r1)
  45        brctg   %r4,.Lmemmove_reverse
  46        ic      %r0,0(%r4,%r3)
  47        stc     %r0,0(%r4,%r1)
  48        BR_EX   %r14
  49.Lmemmove_mvc:
  50        mvc     0(1,%r1),0(%r3)
  51ENDPROC(__memmove)
  52EXPORT_SYMBOL(memmove)
  53
  54/*
  55 * memset implementation
  56 *
  57 * This code corresponds to the C construct below. We do distinguish
  58 * between clearing (c == 0) and setting a memory array (c != 0) simply
  59 * because nearly all memset invocations in the kernel clear memory and
  60 * the xc instruction is preferred in such cases.
  61 *
  62 * void *memset(void *s, int c, size_t n)
  63 * {
  64 *      if (likely(c == 0))
  65 *              return __builtin_memset(s, 0, n);
  66 *      return __builtin_memset(s, c, n);
  67 * }
  68 */
  69WEAK(memset)
  70ENTRY(__memset)
  71        ltgr    %r4,%r4
  72        jz      .Lmemset_exit
  73        ltgr    %r3,%r3
  74        jnz     .Lmemset_fill
  75        aghi    %r4,-1
  76        srlg    %r3,%r4,8
  77        ltgr    %r3,%r3
  78        lgr     %r1,%r2
  79        jz      .Lmemset_clear_remainder
  80.Lmemset_clear_loop:
  81        xc      0(256,%r1),0(%r1)
  82        la      %r1,256(%r1)
  83        brctg   %r3,.Lmemset_clear_loop
  84.Lmemset_clear_remainder:
  85        larl    %r3,.Lmemset_xc
  86        ex      %r4,0(%r3)
  87.Lmemset_exit:
  88        BR_EX   %r14
  89.Lmemset_fill:
  90        cghi    %r4,1
  91        lgr     %r1,%r2
  92        je      .Lmemset_fill_exit
  93        aghi    %r4,-2
  94        srlg    %r5,%r4,8
  95        ltgr    %r5,%r5
  96        jz      .Lmemset_fill_remainder
  97.Lmemset_fill_loop:
  98        stc     %r3,0(%r1)
  99        mvc     1(255,%r1),0(%r1)
 100        la      %r1,256(%r1)
 101        brctg   %r5,.Lmemset_fill_loop
 102.Lmemset_fill_remainder:
 103        stc     %r3,0(%r1)
 104        larl    %r5,.Lmemset_mvc
 105        ex      %r4,0(%r5)
 106        BR_EX   %r14
 107.Lmemset_fill_exit:
 108        stc     %r3,0(%r1)
 109        BR_EX   %r14
 110.Lmemset_xc:
 111        xc      0(1,%r1),0(%r1)
 112.Lmemset_mvc:
 113        mvc     1(1,%r1),0(%r1)
 114ENDPROC(__memset)
 115EXPORT_SYMBOL(memset)
 116
 117/*
 118 * memcpy implementation
 119 *
 120 * void *memcpy(void *dest, const void *src, size_t n)
 121 */
 122WEAK(memcpy)
 123ENTRY(__memcpy)
 124        ltgr    %r4,%r4
 125        jz      .Lmemcpy_exit
 126        aghi    %r4,-1
 127        srlg    %r5,%r4,8
 128        ltgr    %r5,%r5
 129        lgr     %r1,%r2
 130        jnz     .Lmemcpy_loop
 131.Lmemcpy_remainder:
 132        larl    %r5,.Lmemcpy_mvc
 133        ex      %r4,0(%r5)
 134.Lmemcpy_exit:
 135        BR_EX   %r14
 136.Lmemcpy_loop:
 137        mvc     0(256,%r1),0(%r3)
 138        la      %r1,256(%r1)
 139        la      %r3,256(%r3)
 140        brctg   %r5,.Lmemcpy_loop
 141        j       .Lmemcpy_remainder
 142.Lmemcpy_mvc:
 143        mvc     0(1,%r1),0(%r3)
 144ENDPROC(__memcpy)
 145EXPORT_SYMBOL(memcpy)
 146
 147/*
 148 * __memset16/32/64
 149 *
 150 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
 151 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
 152 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
 153 */
 154.macro __MEMSET bits,bytes,insn
 155ENTRY(__memset\bits)
 156        ltgr    %r4,%r4
 157        jz      .L__memset_exit\bits
 158        cghi    %r4,\bytes
 159        je      .L__memset_store\bits
 160        aghi    %r4,-(\bytes+1)
 161        srlg    %r5,%r4,8
 162        ltgr    %r5,%r5
 163        lgr     %r1,%r2
 164        jz      .L__memset_remainder\bits
 165.L__memset_loop\bits:
 166        \insn   %r3,0(%r1)
 167        mvc     \bytes(256-\bytes,%r1),0(%r1)
 168        la      %r1,256(%r1)
 169        brctg   %r5,.L__memset_loop\bits
 170.L__memset_remainder\bits:
 171        \insn   %r3,0(%r1)
 172        larl    %r5,.L__memset_mvc\bits
 173        ex      %r4,0(%r5)
 174        BR_EX   %r14
 175.L__memset_store\bits:
 176        \insn   %r3,0(%r2)
 177.L__memset_exit\bits:
 178        BR_EX   %r14
 179.L__memset_mvc\bits:
 180        mvc     \bytes(1,%r1),0(%r1)
 181ENDPROC(__memset\bits)
 182.endm
 183
 184__MEMSET 16,2,sth
 185EXPORT_SYMBOL(__memset16)
 186
 187__MEMSET 32,4,st
 188EXPORT_SYMBOL(__memset32)
 189
 190__MEMSET 64,8,stg
 191EXPORT_SYMBOL(__memset64)
 192