LXR linux/arch/x86/lib/rwsem.S

   1/*
   2 * x86 semaphore implementation.
   3 *
   4 * (C) Copyright 1999 Linus Torvalds
   5 *
   6 * Portions Copyright 1999 Red Hat, Inc.
   7 *
   8 *      This program is free software; you can redistribute it and/or
   9 *      modify it under the terms of the GNU General Public License
  10 *      as published by the Free Software Foundation; either version
  11 *      2 of the License, or (at your option) any later version.
  12 *
  13 * rw semaphores implemented November 1999 by Benjamin LaHaise <bcrl@kvack.org>
  14 */
  15
  16#include <linux/linkage.h>
  17#include <asm/alternative-asm.h>
  18#include <asm/dwarf2.h>
  19
  20#define __ASM_HALF_REG(reg)     __ASM_SEL(reg, e##reg)
  21#define __ASM_HALF_SIZE(inst)   __ASM_SEL(inst##w, inst##l)
  22
  23#ifdef CONFIG_X86_32
  24
  25/*
  26 * The semaphore operations have a special calling sequence that
  27 * allow us to do a simpler in-line version of them. These routines
  28 * need to convert that sequence back into the C sequence when
  29 * there is contention on the semaphore.
  30 *
  31 * %eax contains the semaphore pointer on entry. Save the C-clobbered
  32 * registers (%eax, %edx and %ecx) except %eax whish is either a return
  33 * value or just clobbered..
  34 */
  35
  36#define save_common_regs \
  37        pushl_cfi %ecx; CFI_REL_OFFSET ecx, 0
  38
  39#define restore_common_regs \
  40        popl_cfi %ecx; CFI_RESTORE ecx
  41
  42        /* Avoid uglifying the argument copying x86-64 needs to do. */
  43        .macro movq src, dst
  44        .endm
  45
  46#else
  47
  48/*
  49 * x86-64 rwsem wrappers
  50 *
  51 * This interfaces the inline asm code to the slow-path
  52 * C routines. We need to save the call-clobbered regs
  53 * that the asm does not mark as clobbered, and move the
  54 * argument from %rax to %rdi.
  55 *
  56 * NOTE! We don't need to save %rax, because the functions
  57 * will always return the semaphore pointer in %rax (which
  58 * is also the input argument to these helpers)
  59 *
  60 * The following can clobber %rdx because the asm clobbers it:
  61 *   call_rwsem_down_write_failed
  62 *   call_rwsem_wake
  63 * but %rdi, %rsi, %rcx, %r8-r11 always need saving.
  64 */
  65
  66#define save_common_regs \
  67        pushq_cfi %rdi; CFI_REL_OFFSET rdi, 0; \
  68        pushq_cfi %rsi; CFI_REL_OFFSET rsi, 0; \
  69        pushq_cfi %rcx; CFI_REL_OFFSET rcx, 0; \
  70        pushq_cfi %r8;  CFI_REL_OFFSET r8,  0; \
  71        pushq_cfi %r9;  CFI_REL_OFFSET r9,  0; \
  72        pushq_cfi %r10; CFI_REL_OFFSET r10, 0; \
  73        pushq_cfi %r11; CFI_REL_OFFSET r11, 0
  74
  75#define restore_common_regs \
  76        popq_cfi %r11; CFI_RESTORE r11; \
  77        popq_cfi %r10; CFI_RESTORE r10; \
  78        popq_cfi %r9;  CFI_RESTORE r9; \
  79        popq_cfi %r8;  CFI_RESTORE r8; \
  80        popq_cfi %rcx; CFI_RESTORE rcx; \
  81        popq_cfi %rsi; CFI_RESTORE rsi; \
  82        popq_cfi %rdi; CFI_RESTORE rdi
  83
  84#endif
  85
  86/* Fix up special calling conventions */
  87ENTRY(call_rwsem_down_read_failed)
  88        CFI_STARTPROC
  89        save_common_regs
  90        __ASM_SIZE(push,_cfi) %__ASM_REG(dx)
  91        CFI_REL_OFFSET __ASM_REG(dx), 0
  92        movq %rax,%rdi
  93        call rwsem_down_read_failed
  94        __ASM_SIZE(pop,_cfi) %__ASM_REG(dx)
  95        CFI_RESTORE __ASM_REG(dx)
  96        restore_common_regs
  97        ret
  98        CFI_ENDPROC
  99ENDPROC(call_rwsem_down_read_failed)
 100
 101ENTRY(call_rwsem_down_write_failed)
 102        CFI_STARTPROC
 103        save_common_regs
 104        movq %rax,%rdi
 105        call rwsem_down_write_failed
 106        restore_common_regs
 107        ret
 108        CFI_ENDPROC
 109ENDPROC(call_rwsem_down_write_failed)
 110
 111ENTRY(call_rwsem_wake)
 112        CFI_STARTPROC
 113        /* do nothing if still outstanding active readers */
 114        __ASM_HALF_SIZE(dec) %__ASM_HALF_REG(dx)
 115        jnz 1f
 116        save_common_regs
 117        movq %rax,%rdi
 118        call rwsem_wake
 119        restore_common_regs
 1201:      ret
 121        CFI_ENDPROC
 122ENDPROC(call_rwsem_wake)
 123
 124ENTRY(call_rwsem_downgrade_wake)
 125        CFI_STARTPROC
 126        save_common_regs
 127        __ASM_SIZE(push,_cfi) %__ASM_REG(dx)
 128        CFI_REL_OFFSET __ASM_REG(dx), 0
 129        movq %rax,%rdi
 130        call rwsem_downgrade_wake
 131        __ASM_SIZE(pop,_cfi) %__ASM_REG(dx)
 132        CFI_RESTORE __ASM_REG(dx)
 133        restore_common_regs
 134        ret
 135        CFI_ENDPROC
 136ENDPROC(call_rwsem_downgrade_wake)
 137