linux/arch/x86/kernel/ftrace_32.S
<<
>>
Prefs 
   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
   4 */
   5
   6#include <linux/linkage.h>
   7#include <asm/page_types.h>
   8#include <asm/segment.h>
   9#include <asm/export.h>
  10#include <asm/ftrace.h>
  11#include <asm/nospec-branch.h>
  12
  13#ifdef CC_USING_FENTRY
  14# define function_hook  __fentry__
  15EXPORT_SYMBOL(__fentry__)
  16#else
  17# define function_hook  mcount
  18EXPORT_SYMBOL(mcount)
  19#endif
  20
  21#ifdef CONFIG_DYNAMIC_FTRACE
  22
  23/* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */
  24#if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER)
  25# define USING_FRAME_POINTER
  26#endif
  27
  28#ifdef USING_FRAME_POINTER
  29# define MCOUNT_FRAME                   1       /* using frame = true  */
  30#else
  31# define MCOUNT_FRAME                   0       /* using frame = false */
  32#endif
  33
  34ENTRY(function_hook)
  35        ret
  36END(function_hook)
  37
  38ENTRY(ftrace_caller)
  39
  40#ifdef USING_FRAME_POINTER
  41# ifdef CC_USING_FENTRY
  42        /*
  43         * Frame pointers are of ip followed by bp.
  44         * Since fentry is an immediate jump, we are left with
  45         * parent-ip, function-ip. We need to add a frame with
  46         * parent-ip followed by ebp.
  47         */
  48        pushl   4(%esp)                         /* parent ip */
  49        pushl   %ebp
  50        movl    %esp, %ebp
  51        pushl   2*4(%esp)                       /* function ip */
  52# endif
  53        /* For mcount, the function ip is directly above */
  54        pushl   %ebp
  55        movl    %esp, %ebp
  56#endif
  57        pushl   %eax
  58        pushl   %ecx
  59        pushl   %edx
  60        pushl   $0                              /* Pass NULL as regs pointer */
  61
  62#ifdef USING_FRAME_POINTER
  63        /* Load parent ebp into edx */
  64        movl    4*4(%esp), %edx
  65#else
  66        /* There's no frame pointer, load the appropriate stack addr instead */
  67        lea     4*4(%esp), %edx
  68#endif
  69
  70        movl    (MCOUNT_FRAME+4)*4(%esp), %eax  /* load the rip */
  71        /* Get the parent ip */
  72        movl    4(%edx), %edx                   /* edx has ebp */
  73
  74        movl    function_trace_op, %ecx
  75        subl    $MCOUNT_INSN_SIZE, %eax
  76
  77.globl ftrace_call
  78ftrace_call:
  79        call    ftrace_stub
  80
  81        addl    $4, %esp                        /* skip NULL pointer */
  82        popl    %edx
  83        popl    %ecx
  84        popl    %eax
  85#ifdef USING_FRAME_POINTER
  86        popl    %ebp
  87# ifdef CC_USING_FENTRY
  88        addl    $4,%esp                         /* skip function ip */
  89        popl    %ebp                            /* this is the orig bp */
  90        addl    $4, %esp                        /* skip parent ip */
  91# endif
  92#endif
  93.Lftrace_ret:
  94#ifdef CONFIG_FUNCTION_GRAPH_TRACER
  95.globl ftrace_graph_call
  96ftrace_graph_call:
  97        jmp     ftrace_stub
  98#endif
  99
 100/* This is weak to keep gas from relaxing the jumps */
 101WEAK(ftrace_stub)
 102        ret
 103END(ftrace_caller)
 104
 105ENTRY(ftrace_regs_caller)
 106        /*
 107         * i386 does not save SS and ESP when coming from kernel.
 108         * Instead, to get sp, &regs->sp is used (see ptrace.h).
 109         * Unfortunately, that means eflags must be at the same location
 110         * as the current return ip is. We move the return ip into the
 111         * regs->ip location, and move flags into the return ip location.
 112         */
 113        pushl   $__KERNEL_CS
 114        pushl   4(%esp)                         /* Save the return ip */
 115        pushl   $0                              /* Load 0 into orig_ax */
 116        pushl   %gs
 117        pushl   %fs
 118        pushl   %es
 119        pushl   %ds
 120        pushl   %eax
 121
 122        /* Get flags and place them into the return ip slot */
 123        pushf
 124        popl    %eax
 125        movl    %eax, 8*4(%esp)
 126
 127        pushl   %ebp
 128        pushl   %edi
 129        pushl   %esi
 130        pushl   %edx
 131        pushl   %ecx
 132        pushl   %ebx
 133
 134        movl    12*4(%esp), %eax                /* Load ip (1st parameter) */
 135        subl    $MCOUNT_INSN_SIZE, %eax         /* Adjust ip */
 136#ifdef CC_USING_FENTRY
 137        movl    15*4(%esp), %edx                /* Load parent ip (2nd parameter) */
 138#else
 139        movl    0x4(%ebp), %edx                 /* Load parent ip (2nd parameter) */
 140#endif
 141        movl    function_trace_op, %ecx         /* Save ftrace_pos in 3rd parameter */
 142        pushl   %esp                            /* Save pt_regs as 4th parameter */
 143
 144GLOBAL(ftrace_regs_call)
 145        call    ftrace_stub
 146
 147        addl    $4, %esp                        /* Skip pt_regs */
 148
 149        /* restore flags */
 150        push    14*4(%esp)
 151        popf
 152
 153        /* Move return ip back to its original location */
 154        movl    12*4(%esp), %eax
 155        movl    %eax, 14*4(%esp)
 156
 157        popl    %ebx
 158        popl    %ecx
 159        popl    %edx
 160        popl    %esi
 161        popl    %edi
 162        popl    %ebp
 163        popl    %eax
 164        popl    %ds
 165        popl    %es
 166        popl    %fs
 167        popl    %gs
 168
 169        /* use lea to not affect flags */
 170        lea     3*4(%esp), %esp                 /* Skip orig_ax, ip and cs */
 171
 172        jmp     .Lftrace_ret
 173#else /* ! CONFIG_DYNAMIC_FTRACE */
 174
 175ENTRY(function_hook)
 176        cmpl    $__PAGE_OFFSET, %esp
 177        jb      ftrace_stub                     /* Paging not enabled yet? */
 178
 179        cmpl    $ftrace_stub, ftrace_trace_function
 180        jnz     .Ltrace
 181#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 182        cmpl    $ftrace_stub, ftrace_graph_return
 183        jnz     ftrace_graph_caller
 184
 185        cmpl    $ftrace_graph_entry_stub, ftrace_graph_entry
 186        jnz     ftrace_graph_caller
 187#endif
 188.globl ftrace_stub
 189ftrace_stub:
 190        ret
 191
 192        /* taken from glibc */
 193.Ltrace:
 194        pushl   %eax
 195        pushl   %ecx
 196        pushl   %edx
 197        movl    0xc(%esp), %eax
 198        movl    0x4(%ebp), %edx
 199        subl    $MCOUNT_INSN_SIZE, %eax
 200
 201        movl    ftrace_trace_function, %ecx
 202        CALL_NOSPEC %ecx
 203
 204        popl    %edx
 205        popl    %ecx
 206        popl    %eax
 207        jmp     ftrace_stub
 208END(function_hook)
 209#endif /* CONFIG_DYNAMIC_FTRACE */
 210
 211#ifdef CONFIG_FUNCTION_GRAPH_TRACER
 212ENTRY(ftrace_graph_caller)
 213        pushl   %eax
 214        pushl   %ecx
 215        pushl   %edx
 216        movl    3*4(%esp), %eax
 217        /* Even with frame pointers, fentry doesn't have one here */
 218#ifdef CC_USING_FENTRY
 219        lea     4*4(%esp), %edx
 220        movl    $0, %ecx
 221#else
 222        lea     0x4(%ebp), %edx
 223        movl    (%ebp), %ecx
 224#endif
 225        subl    $MCOUNT_INSN_SIZE, %eax
 226        call    prepare_ftrace_return
 227        popl    %edx
 228        popl    %ecx
 229        popl    %eax
 230        ret
 231END(ftrace_graph_caller)
 232
 233.globl return_to_handler
 234return_to_handler:
 235        pushl   %eax
 236        pushl   %edx
 237#ifdef CC_USING_FENTRY
 238        movl    $0, %eax
 239#else
 240        movl    %ebp, %eax
 241#endif
 242        call    ftrace_return_to_handler
 243        movl    %eax, %ecx
 244        popl    %edx
 245        popl    %eax
 246        JMP_NOSPEC %ecx
 247#endif
 248
Hosted by Missing Link Electronics. The original LXR software by the LXR community, this experimental version by lxr@linux.no.