1
2
3#include <linux/compiler.h>
4#include <linux/context_tracking.h>
5#include <linux/errno.h>
6#include <linux/nospec.h>
7#include <linux/ptrace.h>
8#include <linux/randomize_kstack.h>
9#include <linux/syscalls.h>
10
11#include <asm/daifflags.h>
12#include <asm/debug-monitors.h>
13#include <asm/exception.h>
14#include <asm/fpsimd.h>
15#include <asm/syscall.h>
16#include <asm/thread_info.h>
17#include <asm/unistd.h>
18
19long compat_arm_syscall(struct pt_regs *regs, int scno);
20long sys_ni_syscall(void);
21
22static long do_ni_syscall(struct pt_regs *regs, int scno)
23{
24#ifdef CONFIG_COMPAT
25 long ret;
26 if (is_compat_task()) {
27 ret = compat_arm_syscall(regs, scno);
28 if (ret != -ENOSYS)
29 return ret;
30 }
31#endif
32
33 return sys_ni_syscall();
34}
35
36static long __invoke_syscall(struct pt_regs *regs, syscall_fn_t syscall_fn)
37{
38 return syscall_fn(regs);
39}
40
41static void invoke_syscall(struct pt_regs *regs, unsigned int scno,
42 unsigned int sc_nr,
43 const syscall_fn_t syscall_table[])
44{
45 long ret;
46
47 add_random_kstack_offset();
48
49 if (scno < sc_nr) {
50 syscall_fn_t syscall_fn;
51 syscall_fn = syscall_table[array_index_nospec(scno, sc_nr)];
52 ret = __invoke_syscall(regs, syscall_fn);
53 } else {
54 ret = do_ni_syscall(regs, scno);
55 }
56
57 syscall_set_return_value(current, regs, 0, ret);
58
59
60
61
62
63
64
65
66
67
68
69
70 choose_random_kstack_offset(get_random_int() & 0x1FF);
71}
72
73static inline bool has_syscall_work(unsigned long flags)
74{
75 return unlikely(flags & _TIF_SYSCALL_WORK);
76}
77
78int syscall_trace_enter(struct pt_regs *regs);
79void syscall_trace_exit(struct pt_regs *regs);
80
81static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
82 const syscall_fn_t syscall_table[])
83{
84 unsigned long flags = current_thread_info()->flags;
85
86 regs->orig_x0 = regs->regs[0];
87 regs->syscallno = scno;
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107 local_daif_restore(DAIF_PROCCTX);
108
109 if (flags & _TIF_MTE_ASYNC_FAULT) {
110
111
112
113
114
115 syscall_set_return_value(current, regs, -ERESTARTNOINTR, 0);
116 return;
117 }
118
119 if (has_syscall_work(flags)) {
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135 if (scno == NO_SYSCALL)
136 syscall_set_return_value(current, regs, -ENOSYS, 0);
137 scno = syscall_trace_enter(regs);
138 if (scno == NO_SYSCALL)
139 goto trace_exit;
140 }
141
142 invoke_syscall(regs, scno, sc_nr, syscall_table);
143
144
145
146
147
148
149 if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
150 local_daif_mask();
151 flags = current_thread_info()->flags;
152 if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
153 return;
154 local_daif_restore(DAIF_PROCCTX);
155 }
156
157trace_exit:
158 syscall_trace_exit(regs);
159}
160
161static inline void sve_user_discard(void)
162{
163 if (!system_supports_sve())
164 return;
165
166 clear_thread_flag(TIF_SVE);
167
168
169
170
171
172
173
174
175 sve_user_disable();
176}
177
178void do_el0_svc(struct pt_regs *regs)
179{
180 sve_user_discard();
181 el0_svc_common(regs, regs->regs[8], __NR_syscalls, sys_call_table);
182}
183
184#ifdef CONFIG_COMPAT
185void do_el0_svc_compat(struct pt_regs *regs)
186{
187 el0_svc_common(regs, regs->regs[7], __NR_compat_syscalls,
188 compat_sys_call_table);
189}
190#endif
191