LXR linux/arch/sparc/kernel/rtrap

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * rtrap.S: Preparing for return from trap on Sparc V9.
   4 *
   5 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   6 * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
   7 */
   8
   9
  10#include <asm/asi.h>
  11#include <asm/pstate.h>
  12#include <asm/ptrace.h>
  13#include <asm/spitfire.h>
  14#include <asm/head.h>
  15#include <asm/visasm.h>
  16#include <asm/processor.h>
  17
  18#ifdef CONFIG_CONTEXT_TRACKING
  19# define SCHEDULE_USER schedule_user
  20#else
  21# define SCHEDULE_USER schedule
  22#endif
  23
  24                .text
  25                .align                  32
  26__handle_preemption:
  27                call                    SCHEDULE_USER
  28661:             wrpr                   %g0, RTRAP_PSTATE, %pstate
  29                /* If userspace is using ADI, it could potentially pass
  30                 * a pointer with version tag embedded in it. To maintain
  31                 * the ADI security, we must re-enable PSTATE.mcde before
  32                 * we continue execution in the kernel for another thread.
  33                 */
  34                .section .sun_m7_1insn_patch, "ax"
  35                .word   661b
  36                 wrpr                   %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate
  37                .previous
  38                ba,pt                   %xcc, __handle_preemption_continue
  39                 wrpr                   %g0, RTRAP_PSTATE_IRQOFF, %pstate
  40
  41__handle_user_windows:
  42                call                    fault_in_user_windows
  43661:             wrpr                   %g0, RTRAP_PSTATE, %pstate
  44                /* If userspace is using ADI, it could potentially pass
  45                 * a pointer with version tag embedded in it. To maintain
  46                 * the ADI security, we must re-enable PSTATE.mcde before
  47                 * we continue execution in the kernel for another thread.
  48                 */
  49                .section .sun_m7_1insn_patch, "ax"
  50                .word   661b
  51                 wrpr                   %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate
  52                .previous
  53                ba,pt                   %xcc, __handle_preemption_continue
  54                 wrpr                   %g0, RTRAP_PSTATE_IRQOFF, %pstate
  55
  56__handle_userfpu:
  57                rd                      %fprs, %l5
  58                andcc                   %l5, FPRS_FEF, %g0
  59                sethi                   %hi(TSTATE_PEF), %o0
  60                be,a,pn                 %icc, __handle_userfpu_continue
  61                 andn                   %l1, %o0, %l1
  62                ba,a,pt                 %xcc, __handle_userfpu_continue
  63
  64__handle_signal:
  65                mov                     %l5, %o1
  66                add                     %sp, PTREGS_OFF, %o0
  67                mov                     %l0, %o2
  68                call                    do_notify_resume
  69661:             wrpr                   %g0, RTRAP_PSTATE, %pstate
  70                /* If userspace is using ADI, it could potentially pass
  71                 * a pointer with version tag embedded in it. To maintain
  72                 * the ADI security, we must re-enable PSTATE.mcde before
  73                 * we continue execution in the kernel for another thread.
  74                 */
  75                .section .sun_m7_1insn_patch, "ax"
  76                .word   661b
  77                 wrpr                   %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate
  78                .previous
  79                wrpr                    %g0, RTRAP_PSTATE_IRQOFF, %pstate
  80
  81                /* Signal delivery can modify pt_regs tstate, so we must
  82                 * reload it.
  83                 */
  84                ldx                     [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
  85                sethi                   %hi(0xf << 20), %l4
  86                and                     %l1, %l4, %l4
  87                andn                    %l1, %l4, %l1
  88                ba,pt                   %xcc, __handle_preemption_continue
  89                 srl                    %l4, 20, %l4
  90
  91                /* When returning from a NMI (%pil==15) interrupt we want to
  92                 * avoid running softirqs, doing IRQ tracing, preempting, etc.
  93                 */
  94                .globl                  rtrap_nmi
  95rtrap_nmi:      ldx                     [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
  96                sethi                   %hi(0xf << 20), %l4
  97                and                     %l1, %l4, %l4
  98                andn                    %l1, %l4, %l1
  99                srl                     %l4, 20, %l4
 100                ba,pt                   %xcc, rtrap_no_irq_enable
 101                nop
 102                /* Do not actually set the %pil here.  We will do that
 103                 * below after we clear PSTATE_IE in the %pstate register.
 104                 * If we re-enable interrupts here, we can recurse down
 105                 * the hardirq stack potentially endlessly, causing a
 106                 * stack overflow.
 107                 */
 108
 109                .align                  64
 110                .globl                  rtrap_irq, rtrap, irqsz_patchme, rtrap_xcall
 111rtrap_irq:
 112rtrap:
 113                /* mm/ultra.S:xcall_report_regs KNOWS about this load. */
 114                ldx                     [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
 115rtrap_xcall:
 116                sethi                   %hi(0xf << 20), %l4
 117                and                     %l1, %l4, %l4
 118                andn                    %l1, %l4, %l1
 119                srl                     %l4, 20, %l4
 120#ifdef CONFIG_TRACE_IRQFLAGS
 121                brnz,pn                 %l4, rtrap_no_irq_enable
 122                 nop
 123                call                    trace_hardirqs_on
 124                 nop
 125                /* Do not actually set the %pil here.  We will do that
 126                 * below after we clear PSTATE_IE in the %pstate register.
 127                 * If we re-enable interrupts here, we can recurse down
 128                 * the hardirq stack potentially endlessly, causing a
 129                 * stack overflow.
 130                 *
 131                 * It is tempting to put this test and trace_hardirqs_on
 132                 * call at the 'rt_continue' label, but that will not work
 133                 * as that path hits unconditionally and we do not want to
 134                 * execute this in NMI return paths, for example.
 135                 */
 136#endif
 137rtrap_no_irq_enable:
 138                andcc                   %l1, TSTATE_PRIV, %l3
 139                bne,pn                  %icc, to_kernel
 140                 nop
 141
 142                /* We must hold IRQs off and atomically test schedule+signal
 143                 * state, then hold them off all the way back to userspace.
 144                 * If we are returning to kernel, none of this matters.  Note
 145                 * that we are disabling interrupts via PSTATE_IE, not using
 146                 * %pil.
 147                 *
 148                 * If we do not do this, there is a window where we would do
 149                 * the tests, later the signal/resched event arrives but we do
 150                 * not process it since we are still in kernel mode.  It would
 151                 * take until the next local IRQ before the signal/resched
 152                 * event would be handled.
 153                 *
 154                 * This also means that if we have to deal with user
 155                 * windows, we have to redo all of these sched+signal checks
 156                 * with IRQs disabled.
 157                 */
 158to_user:        wrpr                    %g0, RTRAP_PSTATE_IRQOFF, %pstate
 159                wrpr                    0, %pil
 160__handle_preemption_continue:
 161                ldx                     [%g6 + TI_FLAGS], %l0
 162                sethi                   %hi(_TIF_USER_WORK_MASK), %o0
 163                or                      %o0, %lo(_TIF_USER_WORK_MASK), %o0
 164                andcc                   %l0, %o0, %g0
 165                sethi                   %hi(TSTATE_PEF), %o0
 166                be,pt                   %xcc, user_nowork
 167                 andcc                  %l1, %o0, %g0
 168                andcc                   %l0, _TIF_NEED_RESCHED, %g0
 169                bne,pn                  %xcc, __handle_preemption
 170                 andcc                  %l0, _TIF_DO_NOTIFY_RESUME_MASK, %g0
 171                bne,pn                  %xcc, __handle_signal
 172                 ldub                   [%g6 + TI_WSAVED], %o2
 173                brnz,pn                 %o2, __handle_user_windows
 174                 nop
 175                sethi                   %hi(TSTATE_PEF), %o0
 176                andcc                   %l1, %o0, %g0
 177
 178                /* This fpdepth clear is necessary for non-syscall rtraps only */
 179user_nowork:
 180                bne,pn                  %xcc, __handle_userfpu
 181                 stb                    %g0, [%g6 + TI_FPDEPTH]
 182__handle_userfpu_continue:
 183
 184rt_continue:    ldx                     [%sp + PTREGS_OFF + PT_V9_G1], %g1
 185                ldx                     [%sp + PTREGS_OFF + PT_V9_G2], %g2
 186
 187                ldx                     [%sp + PTREGS_OFF + PT_V9_G3], %g3
 188                ldx                     [%sp + PTREGS_OFF + PT_V9_G4], %g4
 189                ldx                     [%sp + PTREGS_OFF + PT_V9_G5], %g5
 190                brz,pt                  %l3, 1f
 191                mov                     %g6, %l2
 192
 193                /* Must do this before thread reg is clobbered below.  */
 194                LOAD_PER_CPU_BASE(%g5, %g6, %i0, %i1, %i2)
 1951:
 196                ldx                     [%sp + PTREGS_OFF + PT_V9_G6], %g6
 197                ldx                     [%sp + PTREGS_OFF + PT_V9_G7], %g7
 198
 199                /* Normal globals are restored, go to trap globals.  */
 200661:            wrpr                    %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate
 201                nop
 202                .section                .sun4v_2insn_patch, "ax"
 203                .word                   661b
 204                wrpr                    %g0, RTRAP_PSTATE_IRQOFF, %pstate
 205                SET_GL(1)
 206                .previous
 207
 208                mov                     %l2, %g6
 209
 210                ldx                     [%sp + PTREGS_OFF + PT_V9_I0], %i0
 211                ldx                     [%sp + PTREGS_OFF + PT_V9_I1], %i1
 212
 213                ldx                     [%sp + PTREGS_OFF + PT_V9_I2], %i2
 214                ldx                     [%sp + PTREGS_OFF + PT_V9_I3], %i3
 215                ldx                     [%sp + PTREGS_OFF + PT_V9_I4], %i4
 216                ldx                     [%sp + PTREGS_OFF + PT_V9_I5], %i5
 217                ldx                     [%sp + PTREGS_OFF + PT_V9_I6], %i6
 218                ldx                     [%sp + PTREGS_OFF + PT_V9_I7], %i7
 219                ldx                     [%sp + PTREGS_OFF + PT_V9_TPC], %l2
 220                ldx                     [%sp + PTREGS_OFF + PT_V9_TNPC], %o2
 221
 222                ld                      [%sp + PTREGS_OFF + PT_V9_Y], %o3
 223                wr                      %o3, %g0, %y
 224                wrpr                    %l4, 0x0, %pil
 225                wrpr                    %g0, 0x1, %tl
 226                andn                    %l1, TSTATE_SYSCALL, %l1
 227                wrpr                    %l1, %g0, %tstate
 228                wrpr                    %l2, %g0, %tpc
 229                wrpr                    %o2, %g0, %tnpc
 230
 231                brnz,pn                 %l3, kern_rtt
 232                 mov                    PRIMARY_CONTEXT, %l7
 233
 234661:            ldxa                    [%l7 + %l7] ASI_DMMU, %l0
 235                .section                .sun4v_1insn_patch, "ax"
 236                .word                   661b
 237                ldxa                    [%l7 + %l7] ASI_MMU, %l0
 238                .previous
 239
 240                sethi                   %hi(sparc64_kern_pri_nuc_bits), %l1
 241                ldx                     [%l1 + %lo(sparc64_kern_pri_nuc_bits)], %l1
 242                or                      %l0, %l1, %l0
 243
 244661:            stxa                    %l0, [%l7] ASI_DMMU
 245                .section                .sun4v_1insn_patch, "ax"
 246                .word                   661b
 247                stxa                    %l0, [%l7] ASI_MMU
 248                .previous
 249
 250                sethi                   %hi(KERNBASE), %l7
 251                flush                   %l7
 252                rdpr                    %wstate, %l1
 253                rdpr                    %otherwin, %l2
 254                srl                     %l1, 3, %l1
 255
 256661:            wrpr                    %l2, %g0, %canrestore
 257                .section                .fast_win_ctrl_1insn_patch, "ax"
 258                .word                   661b
 259                .word                   0x89880000      ! normalw
 260                .previous
 261
 262                wrpr                    %l1, %g0, %wstate
 263                brnz,pt                 %l2, user_rtt_restore
 264661:             wrpr                   %g0, %g0, %otherwin
 265                .section                .fast_win_ctrl_1insn_patch, "ax"
 266                .word                   661b
 267                 nop
 268                .previous
 269
 270                ldx                     [%g6 + TI_FLAGS], %g3
 271                wr                      %g0, ASI_AIUP, %asi
 272                rdpr                    %cwp, %g1
 273                andcc                   %g3, _TIF_32BIT, %g0
 274                sub                     %g1, 1, %g1
 275                bne,pt                  %xcc, user_rtt_fill_32bit
 276                 wrpr                   %g1, %cwp
 277                ba,a,pt                 %xcc, user_rtt_fill_64bit
 278                 nop
 279
 280user_rtt_fill_fixup_dax:
 281                ba,pt   %xcc, user_rtt_fill_fixup_common
 282                 mov    1, %g3
 283
 284user_rtt_fill_fixup_mna:
 285                ba,pt   %xcc, user_rtt_fill_fixup_common
 286                 mov    2, %g3
 287
 288user_rtt_fill_fixup:
 289                ba,pt   %xcc, user_rtt_fill_fixup_common
 290                 clr    %g3
 291
 292user_rtt_pre_restore:
 293                add                     %g1, 1, %g1
 294                wrpr                    %g1, 0x0, %cwp
 295
 296user_rtt_restore:
 297                restore
 298                rdpr                    %canrestore, %g1
 299                wrpr                    %g1, 0x0, %cleanwin
 300                retry
 301                nop
 302
 303kern_rtt:       rdpr                    %canrestore, %g1
 304                brz,pn                  %g1, kern_rtt_fill
 305                 nop
 306kern_rtt_restore:
 307                stw                     %g0, [%sp + PTREGS_OFF + PT_V9_MAGIC]
 308                restore
 309                retry
 310
 311to_kernel:
 312#ifdef CONFIG_PREEMPT
 313                ldsw                    [%g6 + TI_PRE_COUNT], %l5
 314                brnz                    %l5, kern_fpucheck
 315                 ldx                    [%g6 + TI_FLAGS], %l5
 316                andcc                   %l5, _TIF_NEED_RESCHED, %g0
 317                be,pt                   %xcc, kern_fpucheck
 318                 nop
 319                cmp                     %l4, 0
 320                bne,pn                  %xcc, kern_fpucheck
 321                 nop
 322                call                    preempt_schedule_irq
 323                 nop
 324                ba,pt                   %xcc, rtrap
 325#endif
 326kern_fpucheck:  ldub                    [%g6 + TI_FPDEPTH], %l5
 327                brz,pt                  %l5, rt_continue
 328                 srl                    %l5, 1, %o0
 329                add                     %g6, TI_FPSAVED, %l6
 330                ldub                    [%l6 + %o0], %l2
 331                sub                     %l5, 2, %l5
 332
 333                add                     %g6, TI_GSR, %o1
 334                andcc                   %l2, (FPRS_FEF|FPRS_DU), %g0
 335                be,pt                   %icc, 2f
 336                 and                    %l2, FPRS_DL, %l6
 337                andcc                   %l2, FPRS_FEF, %g0
 338                be,pn                   %icc, 5f
 339                 sll                    %o0, 3, %o5
 340                rd                      %fprs, %g1
 341
 342                wr                      %g1, FPRS_FEF, %fprs
 343                ldx                     [%o1 + %o5], %g1
 344                add                     %g6, TI_XFSR, %o1
 345                sll                     %o0, 8, %o2
 346                add                     %g6, TI_FPREGS, %o3
 347                brz,pn                  %l6, 1f
 348                 add                    %g6, TI_FPREGS+0x40, %o4
 349
 350                membar                  #Sync
 351                ldda                    [%o3 + %o2] ASI_BLK_P, %f0
 352                ldda                    [%o4 + %o2] ASI_BLK_P, %f16
 353                membar                  #Sync
 3541:              andcc                   %l2, FPRS_DU, %g0
 355                be,pn                   %icc, 1f
 356                 wr                     %g1, 0, %gsr
 357                add                     %o2, 0x80, %o2
 358                membar                  #Sync
 359                ldda                    [%o3 + %o2] ASI_BLK_P, %f32
 360                ldda                    [%o4 + %o2] ASI_BLK_P, %f48
 3611:              membar                  #Sync
 362                ldx                     [%o1 + %o5], %fsr
 3632:              stb                     %l5, [%g6 + TI_FPDEPTH]
 364                ba,pt                   %xcc, rt_continue
 365                 nop
 3665:              wr                      %g0, FPRS_FEF, %fprs
 367                sll                     %o0, 8, %o2
 368
 369                add                     %g6, TI_FPREGS+0x80, %o3
 370                add                     %g6, TI_FPREGS+0xc0, %o4
 371                membar                  #Sync
 372                ldda                    [%o3 + %o2] ASI_BLK_P, %f32
 373                ldda                    [%o4 + %o2] ASI_BLK_P, %f48
 374                membar                  #Sync
 375                wr                      %g0, FPRS_DU, %fprs
 376                ba,pt                   %xcc, rt_continue
 377                 stb                    %l5, [%g6 + TI_FPDEPTH]
 378