1
2#ifndef __ASM_ALTERNATIVE_H
3#define __ASM_ALTERNATIVE_H
4
5#include <asm/cpucaps.h>
6#include <asm/insn.h>
7
8#define ARM64_CB_PATCH ARM64_NCAPS
9
10#ifndef __ASSEMBLY__
11
12#include <linux/init.h>
13#include <linux/types.h>
14#include <linux/stddef.h>
15#include <linux/stringify.h>
16
17struct alt_instr {
18 s32 orig_offset;
19 s32 alt_offset;
20 u16 cpufeature;
21 u8 orig_len;
22 u8 alt_len;
23};
24
25typedef void (*alternative_cb_t)(struct alt_instr *alt,
26 __le32 *origptr, __le32 *updptr, int nr_inst);
27
28void __init apply_boot_alternatives(void);
29void __init apply_alternatives_all(void);
30bool alternative_is_applied(u16 cpufeature);
31
32#ifdef CONFIG_MODULES
33void apply_alternatives_module(void *start, size_t length);
34#else
35static inline void apply_alternatives_module(void *start, size_t length) { }
36#endif
37
38#define ALTINSTR_ENTRY(feature,cb) \
39 " .word 661b - .\n" \
40 " .if " __stringify(cb) " == 0\n" \
41 " .word 663f - .\n" \
42 " .else\n" \
43 " .word " __stringify(cb) "- .\n" \
44 " .endif\n" \
45 " .hword " __stringify(feature) "\n" \
46 " .byte 662b-661b\n" \
47 " .byte 664f-663f\n"
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled, cb) \
66 ".if "__stringify(cfg_enabled)" == 1\n" \
67 "661:\n\t" \
68 oldinstr "\n" \
69 "662:\n" \
70 ".pushsection .altinstructions,\"a\"\n" \
71 ALTINSTR_ENTRY(feature,cb) \
72 ".popsection\n" \
73 " .if " __stringify(cb) " == 0\n" \
74 ".pushsection .altinstr_replacement, \"a\"\n" \
75 "663:\n\t" \
76 newinstr "\n" \
77 "664:\n\t" \
78 ".popsection\n\t" \
79 ".org . - (664b-663b) + (662b-661b)\n\t" \
80 ".org . - (662b-661b) + (664b-663b)\n" \
81 ".else\n\t" \
82 "663:\n\t" \
83 "664:\n\t" \
84 ".endif\n" \
85 ".endif\n"
86
87#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
88 __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg), 0)
89
90#define ALTERNATIVE_CB(oldinstr, cb) \
91 __ALTERNATIVE_CFG(oldinstr, "NOT_AN_INSTRUCTION", ARM64_CB_PATCH, 1, cb)
92#else
93
94#include <asm/assembler.h>
95
96.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
97 .word \orig_offset - .
98 .word \alt_offset - .
99 .hword \feature
100 .byte \orig_len
101 .byte \alt_len
102.endm
103
104.macro alternative_insn insn1, insn2, cap, enable = 1
105 .if \enable
106661: \insn1
107662: .pushsection .altinstructions, "a"
108 altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
109 .popsection
110 .pushsection .altinstr_replacement, "ax"
111663: \insn2
112664: .popsection
113 .org . - (664b-663b) + (662b-661b)
114 .org . - (662b-661b) + (664b-663b)
115 .endif
116.endm
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140.macro alternative_if_not cap
141 .set .Lasm_alt_mode, 0
142 .pushsection .altinstructions, "a"
143 altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
144 .popsection
145661:
146.endm
147
148.macro alternative_if cap
149 .set .Lasm_alt_mode, 1
150 .pushsection .altinstructions, "a"
151 altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
152 .popsection
153 .pushsection .altinstr_replacement, "ax"
154 .align 2
155661:
156.endm
157
158.macro alternative_cb cb
159 .set .Lasm_alt_mode, 0
160 .pushsection .altinstructions, "a"
161 altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
162 .popsection
163661:
164.endm
165
166
167
168
169.macro alternative_else
170662:
171 .if .Lasm_alt_mode==0
172 .pushsection .altinstr_replacement, "ax"
173 .else
174 .popsection
175 .endif
176663:
177.endm
178
179
180
181
182.macro alternative_endif
183664:
184 .if .Lasm_alt_mode==0
185 .popsection
186 .endif
187 .org . - (664b-663b) + (662b-661b)
188 .org . - (662b-661b) + (664b-663b)
189.endm
190
191
192
193
194.macro alternative_cb_end
195662:
196.endm
197
198
199
200
201
202
203.macro alternative_else_nop_endif
204alternative_else
205 nops (662b-661b) / AARCH64_INSN_SIZE
206alternative_endif
207.endm
208
209#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
210 alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
211
212.macro user_alt, label, oldinstr, newinstr, cond
2139999: alternative_insn "\oldinstr", "\newinstr", \cond
214 _ASM_EXTABLE 9999b, \label
215.endm
216
217
218
219
220
221
222#ifdef CONFIG_ARM64_UAO
223 .macro uao_ldp l, reg1, reg2, addr, post_inc
224 alternative_if_not ARM64_HAS_UAO
2258888: ldp \reg1, \reg2, [\addr], \post_inc;
2268889: nop;
227 nop;
228 alternative_else
229 ldtr \reg1, [\addr];
230 ldtr \reg2, [\addr, #8];
231 add \addr, \addr, \post_inc;
232 alternative_endif
233
234 _asm_extable 8888b,\l;
235 _asm_extable 8889b,\l;
236 .endm
237
238 .macro uao_stp l, reg1, reg2, addr, post_inc
239 alternative_if_not ARM64_HAS_UAO
2408888: stp \reg1, \reg2, [\addr], \post_inc;
2418889: nop;
242 nop;
243 alternative_else
244 sttr \reg1, [\addr];
245 sttr \reg2, [\addr, #8];
246 add \addr, \addr, \post_inc;
247 alternative_endif
248
249 _asm_extable 8888b,\l;
250 _asm_extable 8889b,\l;
251 .endm
252
253 .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
254 alternative_if_not ARM64_HAS_UAO
2558888: \inst \reg, [\addr], \post_inc;
256 nop;
257 alternative_else
258 \alt_inst \reg, [\addr];
259 add \addr, \addr, \post_inc;
260 alternative_endif
261
262 _asm_extable 8888b,\l;
263 .endm
264#else
265 .macro uao_ldp l, reg1, reg2, addr, post_inc
266 USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
267 .endm
268 .macro uao_stp l, reg1, reg2, addr, post_inc
269 USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
270 .endm
271 .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
272 USER(\l, \inst \reg, [\addr], \post_inc)
273 .endm
274#endif
275
276#endif
277
278
279
280
281
282
283
284
285#define ALTERNATIVE(oldinstr, newinstr, ...) \
286 _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
287
288#endif
289