LXR linux/arch/powerpc/lib/string

   1/*
   2 * This program is free software; you can redistribute it and/or modify
   3 * it under the terms of the GNU General Public License as published by
   4 * the Free Software Foundation; either version 2 of the License, or
   5 * (at your option) any later version.
   6 *
   7 * This program is distributed in the hope that it will be useful,
   8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10 * GNU General Public License for more details.
  11 *
  12 * You should have received a copy of the GNU General Public License
  13 * along with this program; if not, write to the Free Software
  14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  15 *
  16 * Copyright (C) IBM Corporation, 2012
  17 *
  18 * Author: Anton Blanchard <anton@au.ibm.com>
  19 */
  20
  21#include <asm/ppc_asm.h>
  22#include <asm/linkage.h>
  23#include <asm/asm-offsets.h>
  24#include <asm/export.h>
  25
  26        .section        ".toc","aw"
  27PPC64_CACHES:
  28        .tc             ppc64_caches[TC],ppc64_caches
  29        .section        ".text"
  30
  31/**
  32 * __clear_user: - Zero a block of memory in user space, with less checking.
  33 * @to:   Destination address, in user space.
  34 * @n:    Number of bytes to zero.
  35 *
  36 * Zero a block of memory in user space.  Caller must check
  37 * the specified block with access_ok() before calling this function.
  38 *
  39 * Returns number of bytes that could not be cleared.
  40 * On success, this will be zero.
  41 */
  42
  43        .macro err1
  44100:
  45        EX_TABLE(100b,.Ldo_err1)
  46        .endm
  47
  48        .macro err2
  49200:
  50        EX_TABLE(200b,.Ldo_err2)
  51        .endm
  52
  53        .macro err3
  54300:
  55        EX_TABLE(300b,.Ldo_err3)
  56        .endm
  57
  58.Ldo_err1:
  59        mr      r3,r8
  60
  61.Ldo_err2:
  62        mtctr   r4
  631:
  64err3;   stb     r0,0(r3)
  65        addi    r3,r3,1
  66        addi    r4,r4,-1
  67        bdnz    1b
  68
  69.Ldo_err3:
  70        mr      r3,r4
  71        blr
  72
  73_GLOBAL_TOC(__clear_user)
  74        cmpdi   r4,32
  75        neg     r6,r3
  76        li      r0,0
  77        blt     .Lshort_clear
  78        mr      r8,r3
  79        mtocrf  0x01,r6
  80        clrldi  r6,r6,(64-3)
  81
  82        /* Get the destination 8 byte aligned */
  83        bf      cr7*4+3,1f
  84err1;   stb     r0,0(r3)
  85        addi    r3,r3,1
  86
  871:      bf      cr7*4+2,2f
  88err1;   sth     r0,0(r3)
  89        addi    r3,r3,2
  90
  912:      bf      cr7*4+1,3f
  92err1;   stw     r0,0(r3)
  93        addi    r3,r3,4
  94
  953:      sub     r4,r4,r6
  96
  97        cmpdi   r4,32
  98        cmpdi   cr1,r4,512
  99        blt     .Lshort_clear
 100        bgt     cr1,.Llong_clear
 101
 102.Lmedium_clear:
 103        srdi    r6,r4,5
 104        mtctr   r6
 105
 106        /* Do 32 byte chunks */
 1074:
 108err2;   std     r0,0(r3)
 109err2;   std     r0,8(r3)
 110err2;   std     r0,16(r3)
 111err2;   std     r0,24(r3)
 112        addi    r3,r3,32
 113        addi    r4,r4,-32
 114        bdnz    4b
 115
 116.Lshort_clear:
 117        /* up to 31 bytes to go */
 118        cmpdi   r4,16
 119        blt     6f
 120err2;   std     r0,0(r3)
 121err2;   std     r0,8(r3)
 122        addi    r3,r3,16
 123        addi    r4,r4,-16
 124
 125        /* Up to 15 bytes to go */
 1266:      mr      r8,r3
 127        clrldi  r4,r4,(64-4)
 128        mtocrf  0x01,r4
 129        bf      cr7*4+0,7f
 130err1;   std     r0,0(r3)
 131        addi    r3,r3,8
 132
 1337:      bf      cr7*4+1,8f
 134err1;   stw     r0,0(r3)
 135        addi    r3,r3,4
 136
 1378:      bf      cr7*4+2,9f
 138err1;   sth     r0,0(r3)
 139        addi    r3,r3,2
 140
 1419:      bf      cr7*4+3,10f
 142err1;   stb     r0,0(r3)
 143
 14410:     li      r3,0
 145        blr
 146
 147.Llong_clear:
 148        ld      r5,PPC64_CACHES@toc(r2)
 149
 150        bf      cr7*4+0,11f
 151err2;   std     r0,0(r3)
 152        addi    r3,r3,8
 153        addi    r4,r4,-8
 154
 155        /* Destination is 16 byte aligned, need to get it cache block aligned */
 15611:     lwz     r7,DCACHEL1LOGBLOCKSIZE(r5)
 157        lwz     r9,DCACHEL1BLOCKSIZE(r5)
 158
 159        /*
 160         * With worst case alignment the long clear loop takes a minimum
 161         * of 1 byte less than 2 cachelines.
 162         */
 163        sldi    r10,r9,2
 164        cmpd    r4,r10
 165        blt     .Lmedium_clear
 166
 167        neg     r6,r3
 168        addi    r10,r9,-1
 169        and.    r5,r6,r10
 170        beq     13f
 171
 172        srdi    r6,r5,4
 173        mtctr   r6
 174        mr      r8,r3
 17512:
 176err1;   std     r0,0(r3)
 177err1;   std     r0,8(r3)
 178        addi    r3,r3,16
 179        bdnz    12b
 180
 181        sub     r4,r4,r5
 182
 18313:     srd     r6,r4,r7
 184        mtctr   r6
 185        mr      r8,r3
 18614:
 187err1;   dcbz    0,r3
 188        add     r3,r3,r9
 189        bdnz    14b
 190
 191        and     r4,r4,r10
 192
 193        cmpdi   r4,32
 194        blt     .Lshort_clear
 195        b       .Lmedium_clear
 196EXPORT_SYMBOL(__clear_user)
 197