linux/arch/arm64/lib/copy_page.S
<<
>>
Prefs
   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Copyright (C) 2012 ARM Ltd.
   4 */
   5
   6#include <linux/linkage.h>
   7#include <linux/const.h>
   8#include <asm/assembler.h>
   9#include <asm/page.h>
  10#include <asm/cpufeature.h>
  11#include <asm/alternative.h>
  12
  13/*
  14 * Copy a page from src to dest (both are page aligned)
  15 *
  16 * Parameters:
  17 *      x0 - dest
  18 *      x1 - src
  19 */
  20SYM_FUNC_START_PI(copy_page)
  21alternative_if ARM64_HAS_NO_HW_PREFETCH
  22        // Prefetch three cache lines ahead.
  23        prfm    pldl1strm, [x1, #128]
  24        prfm    pldl1strm, [x1, #256]
  25        prfm    pldl1strm, [x1, #384]
  26alternative_else_nop_endif
  27
  28        ldp     x2, x3, [x1]
  29        ldp     x4, x5, [x1, #16]
  30        ldp     x6, x7, [x1, #32]
  31        ldp     x8, x9, [x1, #48]
  32        ldp     x10, x11, [x1, #64]
  33        ldp     x12, x13, [x1, #80]
  34        ldp     x14, x15, [x1, #96]
  35        ldp     x16, x17, [x1, #112]
  36
  37        add     x0, x0, #256
  38        add     x1, x1, #128
  391:
  40        tst     x0, #(PAGE_SIZE - 1)
  41
  42alternative_if ARM64_HAS_NO_HW_PREFETCH
  43        prfm    pldl1strm, [x1, #384]
  44alternative_else_nop_endif
  45
  46        stnp    x2, x3, [x0, #-256]
  47        ldp     x2, x3, [x1]
  48        stnp    x4, x5, [x0, #16 - 256]
  49        ldp     x4, x5, [x1, #16]
  50        stnp    x6, x7, [x0, #32 - 256]
  51        ldp     x6, x7, [x1, #32]
  52        stnp    x8, x9, [x0, #48 - 256]
  53        ldp     x8, x9, [x1, #48]
  54        stnp    x10, x11, [x0, #64 - 256]
  55        ldp     x10, x11, [x1, #64]
  56        stnp    x12, x13, [x0, #80 - 256]
  57        ldp     x12, x13, [x1, #80]
  58        stnp    x14, x15, [x0, #96 - 256]
  59        ldp     x14, x15, [x1, #96]
  60        stnp    x16, x17, [x0, #112 - 256]
  61        ldp     x16, x17, [x1, #112]
  62
  63        add     x0, x0, #128
  64        add     x1, x1, #128
  65
  66        b.ne    1b
  67
  68        stnp    x2, x3, [x0, #-256]
  69        stnp    x4, x5, [x0, #16 - 256]
  70        stnp    x6, x7, [x0, #32 - 256]
  71        stnp    x8, x9, [x0, #48 - 256]
  72        stnp    x10, x11, [x0, #64 - 256]
  73        stnp    x12, x13, [x0, #80 - 256]
  74        stnp    x14, x15, [x0, #96 - 256]
  75        stnp    x16, x17, [x0, #112 - 256]
  76
  77        ret
  78SYM_FUNC_END_PI(copy_page)
  79EXPORT_SYMBOL(copy_page)
  80