LXR linux/arch/x86/kernel/mcount

   1/*
   2 *  linux/arch/x86_64/mcount_64.S
   3 *
   4 *  Copyright (C) 2014  Steven Rostedt, Red Hat Inc
   5 */
   6
   7#include <linux/linkage.h>
   8#include <asm/ptrace.h>
   9#include <asm/ftrace.h>
  10
  11
  12        .code64
  13        .section .entry.text, "ax"
  14
  15
  16#ifdef CONFIG_FUNCTION_TRACER
  17
  18#ifdef CC_USING_FENTRY
  19# define function_hook  __fentry__
  20#else
  21# define function_hook  mcount
  22#endif
  23
  24/* All cases save the original rbp (8 bytes) */
  25#ifdef CONFIG_FRAME_POINTER
  26# ifdef CC_USING_FENTRY
  27/* Save parent and function stack frames (rip and rbp) */
  28#  define MCOUNT_FRAME_SIZE     (8+16*2)
  29# else
  30/* Save just function stack frame (rip and rbp) */
  31#  define MCOUNT_FRAME_SIZE     (8+16)
  32# endif
  33#else
  34/* No need to save a stack frame */
  35# define MCOUNT_FRAME_SIZE      8
  36#endif /* CONFIG_FRAME_POINTER */
  37
  38/* Size of stack used to save mcount regs in save_mcount_regs */
  39#define MCOUNT_REG_SIZE         (SS+8 + MCOUNT_FRAME_SIZE)
  40
  41/*
  42 * gcc -pg option adds a call to 'mcount' in most functions.
  43 * When -mfentry is used, the call is to 'fentry' and not 'mcount'
  44 * and is done before the function's stack frame is set up.
  45 * They both require a set of regs to be saved before calling
  46 * any C code and restored before returning back to the function.
  47 *
  48 * On boot up, all these calls are converted into nops. When tracing
  49 * is enabled, the call can jump to either ftrace_caller or
  50 * ftrace_regs_caller. Callbacks (tracing functions) that require
  51 * ftrace_regs_caller (like kprobes) need to have pt_regs passed to
  52 * it. For this reason, the size of the pt_regs structure will be
  53 * allocated on the stack and the required mcount registers will
  54 * be saved in the locations that pt_regs has them in.
  55 */
  56
  57/*
  58 * @added: the amount of stack added before calling this
  59 *
  60 * After this is called, the following registers contain:
  61 *
  62 *  %rdi - holds the address that called the trampoline
  63 *  %rsi - holds the parent function (traced function's return address)
  64 *  %rdx - holds the original %rbp
  65 */
  66.macro save_mcount_regs added=0
  67
  68        /* Always save the original rbp */
  69        pushq %rbp
  70
  71#ifdef CONFIG_FRAME_POINTER
  72        /*
  73         * Stack traces will stop at the ftrace trampoline if the frame pointer
  74         * is not set up properly. If fentry is used, we need to save a frame
  75         * pointer for the parent as well as the function traced, because the
  76         * fentry is called before the stack frame is set up, where as mcount
  77         * is called afterward.
  78         */
  79#ifdef CC_USING_FENTRY
  80        /* Save the parent pointer (skip orig rbp and our return address) */
  81        pushq \added+8*2(%rsp)
  82        pushq %rbp
  83        movq %rsp, %rbp
  84        /* Save the return address (now skip orig rbp, rbp and parent) */
  85        pushq \added+8*3(%rsp)
  86#else
  87        /* Can't assume that rip is before this (unless added was zero) */
  88        pushq \added+8(%rsp)
  89#endif
  90        pushq %rbp
  91        movq %rsp, %rbp
  92#endif /* CONFIG_FRAME_POINTER */
  93
  94        /*
  95         * We add enough stack to save all regs.
  96         */
  97        subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp
  98        movq %rax, RAX(%rsp)
  99        movq %rcx, RCX(%rsp)
 100        movq %rdx, RDX(%rsp)
 101        movq %rsi, RSI(%rsp)
 102        movq %rdi, RDI(%rsp)
 103        movq %r8, R8(%rsp)
 104        movq %r9, R9(%rsp)
 105        /*
 106         * Save the original RBP. Even though the mcount ABI does not
 107         * require this, it helps out callers.
 108         */
 109        movq MCOUNT_REG_SIZE-8(%rsp), %rdx
 110        movq %rdx, RBP(%rsp)
 111
 112        /* Copy the parent address into %rsi (second parameter) */
 113#ifdef CC_USING_FENTRY
 114        movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi
 115#else
 116        /* %rdx contains original %rbp */
 117        movq 8(%rdx), %rsi
 118#endif
 119
 120         /* Move RIP to its proper location */
 121        movq MCOUNT_REG_SIZE+\added(%rsp), %rdi
 122        movq %rdi, RIP(%rsp)
 123
 124        /*
 125         * Now %rdi (the first parameter) has the return address of
 126         * where ftrace_call returns. But the callbacks expect the
 127         * address of the call itself.
 128         */
 129        subq $MCOUNT_INSN_SIZE, %rdi
 130        .endm
 131
 132.macro restore_mcount_regs
 133        movq R9(%rsp), %r9
 134        movq R8(%rsp), %r8
 135        movq RDI(%rsp), %rdi
 136        movq RSI(%rsp), %rsi
 137        movq RDX(%rsp), %rdx
 138        movq RCX(%rsp), %rcx
 139        movq RAX(%rsp), %rax
 140
 141        /* ftrace_regs_caller can modify %rbp */
 142        movq RBP(%rsp), %rbp
 143
 144        addq $MCOUNT_REG_SIZE, %rsp
 145
 146        .endm
 147
 148#ifdef CONFIG_DYNAMIC_FTRACE
 149
 150ENTRY(function_hook)
 151        retq
 152END(function_hook)
 153
 154ENTRY(ftrace_caller)
 155        /* save_mcount_regs fills in first two parameters */
 156        save_mcount_regs
 157
 158GLOBAL(ftrace_caller_op_ptr)
 159        /* Load the ftrace_ops into the 3rd parameter */
 160        movq function_trace_op(%rip), %rdx
 161
 162        /* regs go into 4th parameter (but make it NULL) */
 163        movq $0, %rcx
 164
 165GLOBAL(ftrace_call)
 166        call ftrace_stub
 167
 168        restore_mcount_regs
 169
 170        /*
 171         * The copied trampoline must call ftrace_return as it
 172         * still may need to call the function graph tracer.
 173         */
 174GLOBAL(ftrace_caller_end)
 175
 176GLOBAL(ftrace_return)
 177
 178#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 179GLOBAL(ftrace_graph_call)
 180        jmp ftrace_stub
 181#endif
 182
 183GLOBAL(ftrace_stub)
 184        retq
 185END(ftrace_caller)
 186
 187ENTRY(ftrace_regs_caller)
 188        /* Save the current flags before any operations that can change them */
 189        pushfq
 190
 191        /* added 8 bytes to save flags */
 192        save_mcount_regs 8
 193        /* save_mcount_regs fills in first two parameters */
 194
 195GLOBAL(ftrace_regs_caller_op_ptr)
 196        /* Load the ftrace_ops into the 3rd parameter */
 197        movq function_trace_op(%rip), %rdx
 198
 199        /* Save the rest of pt_regs */
 200        movq %r15, R15(%rsp)
 201        movq %r14, R14(%rsp)
 202        movq %r13, R13(%rsp)
 203        movq %r12, R12(%rsp)
 204        movq %r11, R11(%rsp)
 205        movq %r10, R10(%rsp)
 206        movq %rbx, RBX(%rsp)
 207        /* Copy saved flags */
 208        movq MCOUNT_REG_SIZE(%rsp), %rcx
 209        movq %rcx, EFLAGS(%rsp)
 210        /* Kernel segments */
 211        movq $__KERNEL_DS, %rcx
 212        movq %rcx, SS(%rsp)
 213        movq $__KERNEL_CS, %rcx
 214        movq %rcx, CS(%rsp)
 215        /* Stack - skipping return address and flags */
 216        leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
 217        movq %rcx, RSP(%rsp)
 218
 219        /* regs go into 4th parameter */
 220        leaq (%rsp), %rcx
 221
 222GLOBAL(ftrace_regs_call)
 223        call ftrace_stub
 224
 225        /* Copy flags back to SS, to restore them */
 226        movq EFLAGS(%rsp), %rax
 227        movq %rax, MCOUNT_REG_SIZE(%rsp)
 228
 229        /* Handlers can change the RIP */
 230        movq RIP(%rsp), %rax
 231        movq %rax, MCOUNT_REG_SIZE+8(%rsp)
 232
 233        /* restore the rest of pt_regs */
 234        movq R15(%rsp), %r15
 235        movq R14(%rsp), %r14
 236        movq R13(%rsp), %r13
 237        movq R12(%rsp), %r12
 238        movq R10(%rsp), %r10
 239        movq RBX(%rsp), %rbx
 240
 241        restore_mcount_regs
 242
 243        /* Restore flags */
 244        popfq
 245
 246        /*
 247         * As this jmp to ftrace_return can be a short jump
 248         * it must not be copied into the trampoline.
 249         * The trampoline will add the code to jump
 250         * to the return.
 251         */
 252GLOBAL(ftrace_regs_caller_end)
 253
 254        jmp ftrace_return
 255
 256END(ftrace_regs_caller)
 257
 258
 259#else /* ! CONFIG_DYNAMIC_FTRACE */
 260
 261ENTRY(function_hook)
 262        cmpq $ftrace_stub, ftrace_trace_function
 263        jnz trace
 264
 265fgraph_trace:
 266#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 267        cmpq $ftrace_stub, ftrace_graph_return
 268        jnz ftrace_graph_caller
 269
 270        cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
 271        jnz ftrace_graph_caller
 272#endif
 273
 274GLOBAL(ftrace_stub)
 275        retq
 276
 277trace:
 278        /* save_mcount_regs fills in first two parameters */
 279        save_mcount_regs
 280
 281        call   *ftrace_trace_function
 282
 283        restore_mcount_regs
 284
 285        jmp fgraph_trace
 286END(function_hook)
 287#endif /* CONFIG_DYNAMIC_FTRACE */
 288#endif /* CONFIG_FUNCTION_TRACER */
 289
 290#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 291ENTRY(ftrace_graph_caller)
 292        /* Saves rbp into %rdx and fills first parameter  */
 293        save_mcount_regs
 294
 295#ifdef CC_USING_FENTRY
 296        leaq MCOUNT_REG_SIZE+8(%rsp), %rsi
 297        movq $0, %rdx   /* No framepointers needed */
 298#else
 299        /* Save address of the return address of traced function */
 300        leaq 8(%rdx), %rsi
 301        /* ftrace does sanity checks against frame pointers */
 302        movq (%rdx), %rdx
 303#endif
 304        call    prepare_ftrace_return
 305
 306        restore_mcount_regs
 307
 308        retq
 309END(ftrace_graph_caller)
 310
 311GLOBAL(return_to_handler)
 312        subq  $24, %rsp
 313
 314        /* Save the return values */
 315        movq %rax, (%rsp)
 316        movq %rdx, 8(%rsp)
 317        movq %rbp, %rdi
 318
 319        call ftrace_return_to_handler
 320
 321        movq %rax, %rdi
 322        movq 8(%rsp), %rdx
 323        movq (%rsp), %rax
 324        addq $24, %rsp
 325        jmp *%rdi
 326#endif
 327