LXR linux/arch/x86/kernel/ftrace

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
   4 */
   5
   6#include <linux/linkage.h>
   7#include <asm/page_types.h>
   8#include <asm/segment.h>
   9#include <asm/export.h>
  10#include <asm/ftrace.h>
  11#include <asm/nospec-branch.h>
  12#include <asm/frame.h>
  13#include <asm/asm-offsets.h>
  14
  15#ifdef CONFIG_FRAME_POINTER
  16# define MCOUNT_FRAME                   1       /* using frame = true  */
  17#else
  18# define MCOUNT_FRAME                   0       /* using frame = false */
  19#endif
  20
  21SYM_FUNC_START(__fentry__)
  22        ret
  23SYM_FUNC_END(__fentry__)
  24EXPORT_SYMBOL(__fentry__)
  25
  26SYM_CODE_START(ftrace_caller)
  27
  28#ifdef CONFIG_FRAME_POINTER
  29        /*
  30         * Frame pointers are of ip followed by bp.
  31         * Since fentry is an immediate jump, we are left with
  32         * parent-ip, function-ip. We need to add a frame with
  33         * parent-ip followed by ebp.
  34         */
  35        pushl   4(%esp)                         /* parent ip */
  36        pushl   %ebp
  37        movl    %esp, %ebp
  38        pushl   2*4(%esp)                       /* function ip */
  39
  40        /* For mcount, the function ip is directly above */
  41        pushl   %ebp
  42        movl    %esp, %ebp
  43#endif
  44        pushl   %eax
  45        pushl   %ecx
  46        pushl   %edx
  47        pushl   $0                              /* Pass NULL as regs pointer */
  48
  49#ifdef CONFIG_FRAME_POINTER
  50        /* Load parent ebp into edx */
  51        movl    4*4(%esp), %edx
  52#else
  53        /* There's no frame pointer, load the appropriate stack addr instead */
  54        lea     4*4(%esp), %edx
  55#endif
  56
  57        movl    (MCOUNT_FRAME+4)*4(%esp), %eax  /* load the rip */
  58        /* Get the parent ip */
  59        movl    4(%edx), %edx                   /* edx has ebp */
  60
  61        movl    function_trace_op, %ecx
  62        subl    $MCOUNT_INSN_SIZE, %eax
  63
  64.globl ftrace_call
  65ftrace_call:
  66        call    ftrace_stub
  67
  68        addl    $4, %esp                        /* skip NULL pointer */
  69        popl    %edx
  70        popl    %ecx
  71        popl    %eax
  72#ifdef CONFIG_FRAME_POINTER
  73        popl    %ebp
  74        addl    $4,%esp                         /* skip function ip */
  75        popl    %ebp                            /* this is the orig bp */
  76        addl    $4, %esp                        /* skip parent ip */
  77#endif
  78.Lftrace_ret:
  79#ifdef CONFIG_FUNCTION_GRAPH_TRACER
  80.globl ftrace_graph_call
  81ftrace_graph_call:
  82        jmp     ftrace_stub
  83#endif
  84
  85/* This is weak to keep gas from relaxing the jumps */
  86SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
  87        ret
  88SYM_CODE_END(ftrace_caller)
  89
  90SYM_CODE_START(ftrace_regs_caller)
  91        /*
  92         * We're here from an mcount/fentry CALL, and the stack frame looks like:
  93         *
  94         *  <previous context>
  95         *  RET-IP
  96         *
  97         * The purpose of this function is to call out in an emulated INT3
  98         * environment with a stack frame like:
  99         *
 100         *  <previous context>
 101         *  gap / RET-IP
 102         *  gap
 103         *  gap
 104         *  gap
 105         *  pt_regs
 106         *
 107         * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
 108         */
 109        subl    $3*4, %esp      # RET-IP + 3 gaps
 110        pushl   %ss             # ss
 111        pushl   %esp            # points at ss
 112        addl    $5*4, (%esp)    #   make it point at <previous context>
 113        pushfl                  # flags
 114        pushl   $__KERNEL_CS    # cs
 115        pushl   7*4(%esp)       # ip <- RET-IP
 116        pushl   $0              # orig_eax
 117
 118        pushl   %gs
 119        pushl   %fs
 120        pushl   %es
 121        pushl   %ds
 122
 123        pushl   %eax
 124        pushl   %ebp
 125        pushl   %edi
 126        pushl   %esi
 127        pushl   %edx
 128        pushl   %ecx
 129        pushl   %ebx
 130
 131        ENCODE_FRAME_POINTER
 132
 133        movl    PT_EIP(%esp), %eax      # 1st argument: IP
 134        subl    $MCOUNT_INSN_SIZE, %eax
 135        movl    21*4(%esp), %edx        # 2nd argument: parent ip
 136        movl    function_trace_op, %ecx # 3rd argument: ftrace_pos
 137        pushl   %esp                    # 4th argument: pt_regs
 138
 139SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
 140        call    ftrace_stub
 141
 142        addl    $4, %esp                # skip 4th argument
 143
 144        /* place IP below the new SP */
 145        movl    PT_OLDESP(%esp), %eax
 146        movl    PT_EIP(%esp), %ecx
 147        movl    %ecx, -4(%eax)
 148
 149        /* place EAX below that */
 150        movl    PT_EAX(%esp), %ecx
 151        movl    %ecx, -8(%eax)
 152
 153        popl    %ebx
 154        popl    %ecx
 155        popl    %edx
 156        popl    %esi
 157        popl    %edi
 158        popl    %ebp
 159
 160        lea     -8(%eax), %esp
 161        popl    %eax
 162
 163        jmp     .Lftrace_ret
 164SYM_CODE_END(ftrace_regs_caller)
 165
 166#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 167SYM_CODE_START(ftrace_graph_caller)
 168        pushl   %eax
 169        pushl   %ecx
 170        pushl   %edx
 171        movl    3*4(%esp), %eax
 172        /* Even with frame pointers, fentry doesn't have one here */
 173        lea     4*4(%esp), %edx
 174        movl    $0, %ecx
 175        subl    $MCOUNT_INSN_SIZE, %eax
 176        call    prepare_ftrace_return
 177        popl    %edx
 178        popl    %ecx
 179        popl    %eax
 180        ret
 181SYM_CODE_END(ftrace_graph_caller)
 182
 183.globl return_to_handler
 184return_to_handler:
 185        pushl   %eax
 186        pushl   %edx
 187        movl    $0, %eax
 188        call    ftrace_return_to_handler
 189        movl    %eax, %ecx
 190        popl    %edx
 191        popl    %eax
 192        JMP_NOSPEC ecx
 193#endif
 194