1#ifndef __ASM_ALTERNATIVE_H
2#define __ASM_ALTERNATIVE_H
3
4#include <asm/cpufeature.h>
5
6#ifndef __ASSEMBLY__
7
8#include <linux/init.h>
9#include <linux/kconfig.h>
10#include <linux/types.h>
11#include <linux/stddef.h>
12#include <linux/stringify.h>
13
14struct alt_instr {
15 s32 orig_offset;
16 s32 alt_offset;
17 u16 cpufeature;
18 u8 orig_len;
19 u8 alt_len;
20};
21
22void __init apply_alternatives_all(void);
23void apply_alternatives(void *start, size_t length);
24
25#define ALTINSTR_ENTRY(feature) \
26 " .word 661b - .\n" \
27 " .word 663f - .\n" \
28 " .hword " __stringify(feature) "\n" \
29 " .byte 662b-661b\n" \
30 " .byte 664f-663f\n"
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \
47 ".if "__stringify(cfg_enabled)" == 1\n" \
48 "661:\n\t" \
49 oldinstr "\n" \
50 "662:\n" \
51 ".pushsection .altinstructions,\"a\"\n" \
52 ALTINSTR_ENTRY(feature) \
53 ".popsection\n" \
54 ".pushsection .altinstr_replacement, \"a\"\n" \
55 "663:\n\t" \
56 newinstr "\n" \
57 "664:\n\t" \
58 ".popsection\n\t" \
59 ".org . - (664b-663b) + (662b-661b)\n\t" \
60 ".org . - (662b-661b) + (664b-663b)\n" \
61 ".endif\n"
62
63#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
64 __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
65
66#else
67
68#include <asm/assembler.h>
69
70.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
71 .word \orig_offset - .
72 .word \alt_offset - .
73 .hword \feature
74 .byte \orig_len
75 .byte \alt_len
76.endm
77
78.macro alternative_insn insn1, insn2, cap, enable = 1
79 .if \enable
80661: \insn1
81662: .pushsection .altinstructions, "a"
82 altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
83 .popsection
84 .pushsection .altinstr_replacement, "ax"
85663: \insn2
86664: .popsection
87 .org . - (664b-663b) + (662b-661b)
88 .org . - (662b-661b) + (664b-663b)
89 .endif
90.endm
91
92
93
94
95
96
97
98.macro alternative_if_not cap, enable = 1
99 .if \enable
100 .pushsection .altinstructions, "a"
101 altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
102 .popsection
103661:
104 .endif
105.endm
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121.macro alternative_else, enable = 1
122 .if \enable
123662: .pushsection .altinstr_replacement, "ax"
124663:
125 .endif
126.endm
127
128
129
130
131.macro alternative_endif, enable = 1
132 .if \enable
133664: .popsection
134 .org . - (664b-663b) + (662b-661b)
135 .org . - (662b-661b) + (664b-663b)
136 .endif
137.endm
138
139#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
140 alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
141
142
143
144
145
146
147
148#ifdef CONFIG_ARM64_UAO
149 .macro uao_ldp l, reg1, reg2, addr, post_inc
150 alternative_if_not ARM64_HAS_UAO
1518888: ldp \reg1, \reg2, [\addr], \post_inc;
1528889: nop;
153 nop;
154 alternative_else
155 ldtr \reg1, [\addr];
156 ldtr \reg2, [\addr, #8];
157 add \addr, \addr, \post_inc;
158 alternative_endif
159
160 _asm_extable 8888b,\l;
161 _asm_extable 8889b,\l;
162 .endm
163
164 .macro uao_stp l, reg1, reg2, addr, post_inc
165 alternative_if_not ARM64_HAS_UAO
1668888: stp \reg1, \reg2, [\addr], \post_inc;
1678889: nop;
168 nop;
169 alternative_else
170 sttr \reg1, [\addr];
171 sttr \reg2, [\addr, #8];
172 add \addr, \addr, \post_inc;
173 alternative_endif
174
175 _asm_extable 8888b,\l;
176 _asm_extable 8889b,\l;
177 .endm
178
179 .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
180 alternative_if_not ARM64_HAS_UAO
1818888: \inst \reg, [\addr], \post_inc;
182 nop;
183 alternative_else
184 \alt_inst \reg, [\addr];
185 add \addr, \addr, \post_inc;
186 alternative_endif
187
188 _asm_extable 8888b,\l;
189 .endm
190#else
191 .macro uao_ldp l, reg1, reg2, addr, post_inc
192 USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
193 .endm
194 .macro uao_stp l, reg1, reg2, addr, post_inc
195 USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
196 .endm
197 .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
198 USER(\l, \inst \reg, [\addr], \post_inc)
199 .endm
200#endif
201
202#endif
203
204
205
206
207
208
209
210
211#define ALTERNATIVE(oldinstr, newinstr, ...) \
212 _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
213
214#endif
215