LXR linux/arch/x86/kernel/ftrace

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
   4 */
   5
   6#include <linux/linkage.h>
   7#include <asm/page_types.h>
   8#include <asm/segment.h>
   9#include <asm/export.h>
  10#include <asm/ftrace.h>
  11#include <asm/nospec-branch.h>
  12#include <asm/frame.h>
  13#include <asm/asm-offsets.h>
  14
  15# define function_hook  __fentry__
  16EXPORT_SYMBOL(__fentry__)
  17
  18#ifdef CONFIG_FRAME_POINTER
  19# define MCOUNT_FRAME                   1       /* using frame = true  */
  20#else
  21# define MCOUNT_FRAME                   0       /* using frame = false */
  22#endif
  23
  24ENTRY(function_hook)
  25        ret
  26END(function_hook)
  27
  28ENTRY(ftrace_caller)
  29
  30#ifdef CONFIG_FRAME_POINTER
  31        /*
  32         * Frame pointers are of ip followed by bp.
  33         * Since fentry is an immediate jump, we are left with
  34         * parent-ip, function-ip. We need to add a frame with
  35         * parent-ip followed by ebp.
  36         */
  37        pushl   4(%esp)                         /* parent ip */
  38        pushl   %ebp
  39        movl    %esp, %ebp
  40        pushl   2*4(%esp)                       /* function ip */
  41
  42        /* For mcount, the function ip is directly above */
  43        pushl   %ebp
  44        movl    %esp, %ebp
  45#endif
  46        pushl   %eax
  47        pushl   %ecx
  48        pushl   %edx
  49        pushl   $0                              /* Pass NULL as regs pointer */
  50
  51#ifdef CONFIG_FRAME_POINTER
  52        /* Load parent ebp into edx */
  53        movl    4*4(%esp), %edx
  54#else
  55        /* There's no frame pointer, load the appropriate stack addr instead */
  56        lea     4*4(%esp), %edx
  57#endif
  58
  59        movl    (MCOUNT_FRAME+4)*4(%esp), %eax  /* load the rip */
  60        /* Get the parent ip */
  61        movl    4(%edx), %edx                   /* edx has ebp */
  62
  63        movl    function_trace_op, %ecx
  64        subl    $MCOUNT_INSN_SIZE, %eax
  65
  66.globl ftrace_call
  67ftrace_call:
  68        call    ftrace_stub
  69
  70        addl    $4, %esp                        /* skip NULL pointer */
  71        popl    %edx
  72        popl    %ecx
  73        popl    %eax
  74#ifdef CONFIG_FRAME_POINTER
  75        popl    %ebp
  76        addl    $4,%esp                         /* skip function ip */
  77        popl    %ebp                            /* this is the orig bp */
  78        addl    $4, %esp                        /* skip parent ip */
  79#endif
  80.Lftrace_ret:
  81#ifdef CONFIG_FUNCTION_GRAPH_TRACER
  82.globl ftrace_graph_call
  83ftrace_graph_call:
  84        jmp     ftrace_stub
  85#endif
  86
  87/* This is weak to keep gas from relaxing the jumps */
  88WEAK(ftrace_stub)
  89        ret
  90END(ftrace_caller)
  91
  92ENTRY(ftrace_regs_caller)
  93        /*
  94         * We're here from an mcount/fentry CALL, and the stack frame looks like:
  95         *
  96         *  <previous context>
  97         *  RET-IP
  98         *
  99         * The purpose of this function is to call out in an emulated INT3
 100         * environment with a stack frame like:
 101         *
 102         *  <previous context>
 103         *  gap / RET-IP
 104         *  gap
 105         *  gap
 106         *  gap
 107         *  pt_regs
 108         *
 109         * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds
 110         */
 111        subl    $3*4, %esp      # RET-IP + 3 gaps
 112        pushl   %ss             # ss
 113        pushl   %esp            # points at ss
 114        addl    $5*4, (%esp)    #   make it point at <previous context>
 115        pushfl                  # flags
 116        pushl   $__KERNEL_CS    # cs
 117        pushl   7*4(%esp)       # ip <- RET-IP
 118        pushl   $0              # orig_eax
 119
 120        pushl   %gs
 121        pushl   %fs
 122        pushl   %es
 123        pushl   %ds
 124
 125        pushl   %eax
 126        pushl   %ebp
 127        pushl   %edi
 128        pushl   %esi
 129        pushl   %edx
 130        pushl   %ecx
 131        pushl   %ebx
 132
 133        ENCODE_FRAME_POINTER
 134
 135        movl    PT_EIP(%esp), %eax      # 1st argument: IP
 136        subl    $MCOUNT_INSN_SIZE, %eax
 137        movl    21*4(%esp), %edx        # 2nd argument: parent ip
 138        movl    function_trace_op, %ecx # 3rd argument: ftrace_pos
 139        pushl   %esp                    # 4th argument: pt_regs
 140
 141GLOBAL(ftrace_regs_call)
 142        call    ftrace_stub
 143
 144        addl    $4, %esp                # skip 4th argument
 145
 146        /* place IP below the new SP */
 147        movl    PT_OLDESP(%esp), %eax
 148        movl    PT_EIP(%esp), %ecx
 149        movl    %ecx, -4(%eax)
 150
 151        /* place EAX below that */
 152        movl    PT_EAX(%esp), %ecx
 153        movl    %ecx, -8(%eax)
 154
 155        popl    %ebx
 156        popl    %ecx
 157        popl    %edx
 158        popl    %esi
 159        popl    %edi
 160        popl    %ebp
 161
 162        lea     -8(%eax), %esp
 163        popl    %eax
 164
 165        jmp     .Lftrace_ret
 166
 167#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 168ENTRY(ftrace_graph_caller)
 169        pushl   %eax
 170        pushl   %ecx
 171        pushl   %edx
 172        movl    3*4(%esp), %eax
 173        /* Even with frame pointers, fentry doesn't have one here */
 174        lea     4*4(%esp), %edx
 175        movl    $0, %ecx
 176        subl    $MCOUNT_INSN_SIZE, %eax
 177        call    prepare_ftrace_return
 178        popl    %edx
 179        popl    %ecx
 180        popl    %eax
 181        ret
 182END(ftrace_graph_caller)
 183
 184.globl return_to_handler
 185return_to_handler:
 186        pushl   %eax
 187        pushl   %edx
 188        movl    $0, %eax
 189        call    ftrace_return_to_handler
 190        movl    %eax, %ecx
 191        popl    %edx
 192        popl    %eax
 193        JMP_NOSPEC %ecx
 194#endif
 195