linux/arch/x86/lib/memcpy_32.c
<<
>>
Prefs
   1#include <linux/string.h>
   2#include <linux/module.h>
   3
   4#undef memcpy
   5#undef memset
   6
   7void *memcpy(void *to, const void *from, size_t n)
   8{
   9#ifdef CONFIG_X86_USE_3DNOW
  10        return __memcpy3d(to, from, n);
  11#else
  12        return __memcpy(to, from, n);
  13#endif
  14}
  15EXPORT_SYMBOL(memcpy);
  16
  17void *memset(void *s, int c, size_t count)
  18{
  19        return __memset(s, c, count);
  20}
  21EXPORT_SYMBOL(memset);
  22
  23void *memmove(void *dest, const void *src, size_t n)
  24{
  25        int d0,d1,d2,d3,d4,d5;
  26        char *ret = dest;
  27
  28        __asm__ __volatile__(
  29                /* Handle more 16 bytes in loop */
  30                "cmp $0x10, %0\n\t"
  31                "jb     1f\n\t"
  32
  33                /* Decide forward/backward copy mode */
  34                "cmp %2, %1\n\t"
  35                "jb     2f\n\t"
  36
  37                /*
  38                 * movs instruction have many startup latency
  39                 * so we handle small size by general register.
  40                 */
  41                "cmp  $680, %0\n\t"
  42                "jb 3f\n\t"
  43                /*
  44                 * movs instruction is only good for aligned case.
  45                 */
  46                "mov %1, %3\n\t"
  47                "xor %2, %3\n\t"
  48                "and $0xff, %3\n\t"
  49                "jz 4f\n\t"
  50                "3:\n\t"
  51                "sub $0x10, %0\n\t"
  52
  53                /*
  54                 * We gobble 16 bytes forward in each loop.
  55                 */
  56                "3:\n\t"
  57                "sub $0x10, %0\n\t"
  58                "mov 0*4(%1), %3\n\t"
  59                "mov 1*4(%1), %4\n\t"
  60                "mov  %3, 0*4(%2)\n\t"
  61                "mov  %4, 1*4(%2)\n\t"
  62                "mov 2*4(%1), %3\n\t"
  63                "mov 3*4(%1), %4\n\t"
  64                "mov  %3, 2*4(%2)\n\t"
  65                "mov  %4, 3*4(%2)\n\t"
  66                "lea  0x10(%1), %1\n\t"
  67                "lea  0x10(%2), %2\n\t"
  68                "jae 3b\n\t"
  69                "add $0x10, %0\n\t"
  70                "jmp 1f\n\t"
  71
  72                /*
  73                 * Handle data forward by movs.
  74                 */
  75                ".p2align 4\n\t"
  76                "4:\n\t"
  77                "mov -4(%1, %0), %3\n\t"
  78                "lea -4(%2, %0), %4\n\t"
  79                "shr $2, %0\n\t"
  80                "rep movsl\n\t"
  81                "mov %3, (%4)\n\t"
  82                "jmp 11f\n\t"
  83                /*
  84                 * Handle data backward by movs.
  85                 */
  86                ".p2align 4\n\t"
  87                "6:\n\t"
  88                "mov (%1), %3\n\t"
  89                "mov %2, %4\n\t"
  90                "lea -4(%1, %0), %1\n\t"
  91                "lea -4(%2, %0), %2\n\t"
  92                "shr $2, %0\n\t"
  93                "std\n\t"
  94                "rep movsl\n\t"
  95                "mov %3,(%4)\n\t"
  96                "cld\n\t"
  97                "jmp 11f\n\t"
  98
  99                /*
 100                 * Start to prepare for backward copy.
 101                 */
 102                ".p2align 4\n\t"
 103                "2:\n\t"
 104                "cmp  $680, %0\n\t"
 105                "jb 5f\n\t"
 106                "mov %1, %3\n\t"
 107                "xor %2, %3\n\t"
 108                "and $0xff, %3\n\t"
 109                "jz 6b\n\t"
 110
 111                /*
 112                 * Calculate copy position to tail.
 113                 */
 114                "5:\n\t"
 115                "add %0, %1\n\t"
 116                "add %0, %2\n\t"
 117                "sub $0x10, %0\n\t"
 118
 119                /*
 120                 * We gobble 16 bytes backward in each loop.
 121                 */
 122                "7:\n\t"
 123                "sub $0x10, %0\n\t"
 124
 125                "mov -1*4(%1), %3\n\t"
 126                "mov -2*4(%1), %4\n\t"
 127                "mov  %3, -1*4(%2)\n\t"
 128                "mov  %4, -2*4(%2)\n\t"
 129                "mov -3*4(%1), %3\n\t"
 130                "mov -4*4(%1), %4\n\t"
 131                "mov  %3, -3*4(%2)\n\t"
 132                "mov  %4, -4*4(%2)\n\t"
 133                "lea  -0x10(%1), %1\n\t"
 134                "lea  -0x10(%2), %2\n\t"
 135                "jae 7b\n\t"
 136                /*
 137                 * Calculate copy position to head.
 138                 */
 139                "add $0x10, %0\n\t"
 140                "sub %0, %1\n\t"
 141                "sub %0, %2\n\t"
 142
 143                /*
 144                 * Move data from 8 bytes to 15 bytes.
 145                 */
 146                ".p2align 4\n\t"
 147                "1:\n\t"
 148                "cmp $8, %0\n\t"
 149                "jb 8f\n\t"
 150                "mov 0*4(%1), %3\n\t"
 151                "mov 1*4(%1), %4\n\t"
 152                "mov -2*4(%1, %0), %5\n\t"
 153                "mov -1*4(%1, %0), %1\n\t"
 154
 155                "mov  %3, 0*4(%2)\n\t"
 156                "mov  %4, 1*4(%2)\n\t"
 157                "mov  %5, -2*4(%2, %0)\n\t"
 158                "mov  %1, -1*4(%2, %0)\n\t"
 159                "jmp 11f\n\t"
 160
 161                /*
 162                 * Move data from 4 bytes to 7 bytes.
 163                 */
 164                ".p2align 4\n\t"
 165                "8:\n\t"
 166                "cmp $4, %0\n\t"
 167                "jb 9f\n\t"
 168                "mov 0*4(%1), %3\n\t"
 169                "mov -1*4(%1, %0), %4\n\t"
 170                "mov  %3, 0*4(%2)\n\t"
 171                "mov  %4, -1*4(%2, %0)\n\t"
 172                "jmp 11f\n\t"
 173
 174                /*
 175                 * Move data from 2 bytes to 3 bytes.
 176                 */
 177                ".p2align 4\n\t"
 178                "9:\n\t"
 179                "cmp $2, %0\n\t"
 180                "jb 10f\n\t"
 181                "movw 0*2(%1), %%dx\n\t"
 182                "movw -1*2(%1, %0), %%bx\n\t"
 183                "movw %%dx, 0*2(%2)\n\t"
 184                "movw %%bx, -1*2(%2, %0)\n\t"
 185                "jmp 11f\n\t"
 186
 187                /*
 188                 * Move data for 1 byte.
 189                 */
 190                ".p2align 4\n\t"
 191                "10:\n\t"
 192                "cmp $1, %0\n\t"
 193                "jb 11f\n\t"
 194                "movb (%1), %%cl\n\t"
 195                "movb %%cl, (%2)\n\t"
 196                ".p2align 4\n\t"
 197                "11:"
 198                : "=&c" (d0), "=&S" (d1), "=&D" (d2),
 199                  "=r" (d3),"=r" (d4), "=r"(d5)
 200                :"0" (n),
 201                 "1" (src),
 202                 "2" (dest)
 203                :"memory");
 204
 205        return ret;
 206
 207}
 208EXPORT_SYMBOL(memmove);
 209