Contents of /trunk/kernel-alx-legacy/patches-4.9/0409-4.9.310-all-fixes.patch
Parent Directory | Revision Log
Revision 3711 -
(show annotations)
(download)
Mon Oct 24 14:08:24 2022 UTC (19 months, 3 weeks ago) by niro
File size: 95653 byte(s)
Mon Oct 24 14:08:24 2022 UTC (19 months, 3 weeks ago) by niro
File size: 95653 byte(s)
-linux-4.9.310
1 | diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt |
2 | index ac9489fad31b8..47df2c25302ac 100644 |
3 | --- a/Documentation/arm64/silicon-errata.txt |
4 | +++ b/Documentation/arm64/silicon-errata.txt |
5 | @@ -55,6 +55,7 @@ stable kernels. |
6 | | ARM | Cortex-A57 | #834220 | ARM64_ERRATUM_834220 | |
7 | | ARM | Cortex-A72 | #853709 | N/A | |
8 | | ARM | Cortex-A55 | #1024718 | ARM64_ERRATUM_1024718 | |
9 | +| ARM | Cortex-A76 | #1188873 | ARM64_ERRATUM_1188873 | |
10 | | ARM | MMU-500 | #841119,#826419 | N/A | |
11 | | | | | | |
12 | | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 | |
13 | diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt |
14 | index 6c0957c67d207..f2b10986ab889 100644 |
15 | --- a/Documentation/kernel-parameters.txt |
16 | +++ b/Documentation/kernel-parameters.txt |
17 | @@ -751,15 +751,6 @@ bytes respectively. Such letter suffixes can also be entirely omitted. |
18 | loops can be debugged more effectively on production |
19 | systems. |
20 | |
21 | - clocksource.arm_arch_timer.fsl-a008585= |
22 | - [ARM64] |
23 | - Format: <bool> |
24 | - Enable/disable the workaround of Freescale/NXP |
25 | - erratum A-008585. This can be useful for KVM |
26 | - guests, if the guest device tree doesn't show the |
27 | - erratum. If unspecified, the workaround is |
28 | - enabled based on the device tree. |
29 | - |
30 | clearcpuid=BITNUM [X86] |
31 | Disable CPUID feature X for the kernel. See |
32 | arch/x86/include/asm/cpufeatures.h for the valid bit |
33 | diff --git a/Makefile b/Makefile |
34 | index c1d4ddadbcb51..72a8c786cb777 100644 |
35 | --- a/Makefile |
36 | +++ b/Makefile |
37 | @@ -1,6 +1,6 @@ |
38 | VERSION = 4 |
39 | PATCHLEVEL = 9 |
40 | -SUBLEVEL = 309 |
41 | +SUBLEVEL = 310 |
42 | EXTRAVERSION = |
43 | NAME = Roaring Lionus |
44 | |
45 | diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h |
46 | index 2fda7e905754b..82c71a147f21d 100644 |
47 | --- a/arch/arm/include/asm/kvm_host.h |
48 | +++ b/arch/arm/include/asm/kvm_host.h |
49 | @@ -349,4 +349,9 @@ static inline int kvm_arm_have_ssbd(void) |
50 | return KVM_SSBD_UNKNOWN; |
51 | } |
52 | |
53 | +static inline bool kvm_arm_spectre_bhb_mitigated(void) |
54 | +{ |
55 | + /* 32bit guests don't need firmware for this */ |
56 | + return false; |
57 | +} |
58 | #endif /* __ARM_KVM_HOST_H__ */ |
59 | diff --git a/arch/arm/kvm/psci.c b/arch/arm/kvm/psci.c |
60 | index 83365bec04b69..a262c175456d3 100644 |
61 | --- a/arch/arm/kvm/psci.c |
62 | +++ b/arch/arm/kvm/psci.c |
63 | @@ -431,6 +431,10 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) |
64 | break; |
65 | } |
66 | break; |
67 | + case ARM_SMCCC_ARCH_WORKAROUND_3: |
68 | + if (kvm_arm_spectre_bhb_mitigated()) |
69 | + val = SMCCC_RET_SUCCESS; |
70 | + break; |
71 | } |
72 | break; |
73 | default: |
74 | diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig |
75 | index b12275be0e139..6d12c3b787771 100644 |
76 | --- a/arch/arm64/Kconfig |
77 | +++ b/arch/arm64/Kconfig |
78 | @@ -441,6 +441,20 @@ config ARM64_ERRATUM_1024718 |
79 | |
80 | If unsure, say Y. |
81 | |
82 | +config ARM64_ERRATUM_1188873 |
83 | + bool "Cortex-A76: MRC read following MRRC read of specific Generic Timer in AArch32 might give incorrect result" |
84 | + default y |
85 | + depends on COMPAT |
86 | + select ARM_ARCH_TIMER_OOL_WORKAROUND |
87 | + help |
88 | + This option adds work arounds for ARM Cortex-A76 erratum 1188873 |
89 | + |
90 | + Affected Cortex-A76 cores (r0p0, r1p0, r2p0) could cause |
91 | + register corruption when accessing the timer registers from |
92 | + AArch32 userspace. |
93 | + |
94 | + If unsure, say Y. |
95 | + |
96 | config CAVIUM_ERRATUM_22375 |
97 | bool "Cavium erratum 22375, 24313" |
98 | default y |
99 | @@ -785,6 +799,16 @@ config ARM64_SSBD |
100 | |
101 | If unsure, say Y. |
102 | |
103 | +config MITIGATE_SPECTRE_BRANCH_HISTORY |
104 | + bool "Mitigate Spectre style attacks against branch history" if EXPERT |
105 | + default y |
106 | + depends on HARDEN_BRANCH_PREDICTOR || !KVM |
107 | + help |
108 | + Speculation attacks against some high-performance processors can |
109 | + make use of branch history to influence future speculation. |
110 | + When taking an exception from user-space, a sequence of branches |
111 | + or a firmware call overwrites the branch history. |
112 | + |
113 | menuconfig ARMV8_DEPRECATED |
114 | bool "Emulate deprecated/obsolete ARMv8 instructions" |
115 | depends on COMPAT |
116 | diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h |
117 | index eaa5bbe3fa875..1b0d7e994e0c7 100644 |
118 | --- a/arch/arm64/include/asm/arch_timer.h |
119 | +++ b/arch/arm64/include/asm/arch_timer.h |
120 | @@ -29,41 +29,35 @@ |
121 | |
122 | #include <clocksource/arm_arch_timer.h> |
123 | |
124 | -#if IS_ENABLED(CONFIG_FSL_ERRATUM_A008585) |
125 | +#if IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND) |
126 | extern struct static_key_false arch_timer_read_ool_enabled; |
127 | -#define needs_fsl_a008585_workaround() \ |
128 | +#define needs_unstable_timer_counter_workaround() \ |
129 | static_branch_unlikely(&arch_timer_read_ool_enabled) |
130 | #else |
131 | -#define needs_fsl_a008585_workaround() false |
132 | +#define needs_unstable_timer_counter_workaround() false |
133 | #endif |
134 | |
135 | -u32 __fsl_a008585_read_cntp_tval_el0(void); |
136 | -u32 __fsl_a008585_read_cntv_tval_el0(void); |
137 | -u64 __fsl_a008585_read_cntvct_el0(void); |
138 | +enum arch_timer_erratum_match_type { |
139 | + ate_match_dt, |
140 | + ate_match_local_cap_id, |
141 | +}; |
142 | |
143 | -/* |
144 | - * The number of retries is an arbitrary value well beyond the highest number |
145 | - * of iterations the loop has been observed to take. |
146 | - */ |
147 | -#define __fsl_a008585_read_reg(reg) ({ \ |
148 | - u64 _old, _new; \ |
149 | - int _retries = 200; \ |
150 | - \ |
151 | - do { \ |
152 | - _old = read_sysreg(reg); \ |
153 | - _new = read_sysreg(reg); \ |
154 | - _retries--; \ |
155 | - } while (unlikely(_old != _new) && _retries); \ |
156 | - \ |
157 | - WARN_ON_ONCE(!_retries); \ |
158 | - _new; \ |
159 | -}) |
160 | +struct arch_timer_erratum_workaround { |
161 | + enum arch_timer_erratum_match_type match_type; |
162 | + const void *id; |
163 | + const char *desc; |
164 | + u32 (*read_cntp_tval_el0)(void); |
165 | + u32 (*read_cntv_tval_el0)(void); |
166 | + u64 (*read_cntvct_el0)(void); |
167 | +}; |
168 | + |
169 | +extern const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround; |
170 | |
171 | #define arch_timer_reg_read_stable(reg) \ |
172 | ({ \ |
173 | u64 _val; \ |
174 | - if (needs_fsl_a008585_workaround()) \ |
175 | - _val = __fsl_a008585_read_##reg(); \ |
176 | + if (needs_unstable_timer_counter_workaround()) \ |
177 | + _val = timer_unstable_counter_workaround->read_##reg();\ |
178 | else \ |
179 | _val = read_sysreg(reg); \ |
180 | _val; \ |
181 | diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h |
182 | index 3f85bbcd7e408..a6aaeb871d5fa 100644 |
183 | --- a/arch/arm64/include/asm/assembler.h |
184 | +++ b/arch/arm64/include/asm/assembler.h |
185 | @@ -94,6 +94,13 @@ |
186 | hint #20 |
187 | .endm |
188 | |
189 | +/* |
190 | + * Clear Branch History instruction |
191 | + */ |
192 | + .macro clearbhb |
193 | + hint #22 |
194 | + .endm |
195 | + |
196 | /* |
197 | * Sanitise a 64-bit bounded index wrt speculation, returning zero if out |
198 | * of bounds. |
199 | @@ -494,4 +501,31 @@ alternative_endif |
200 | .Ldone\@: |
201 | .endm |
202 | |
203 | + .macro __mitigate_spectre_bhb_loop tmp |
204 | +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
205 | +alternative_cb spectre_bhb_patch_loop_iter |
206 | + mov \tmp, #32 // Patched to correct the immediate |
207 | +alternative_cb_end |
208 | +.Lspectre_bhb_loop\@: |
209 | + b . + 4 |
210 | + subs \tmp, \tmp, #1 |
211 | + b.ne .Lspectre_bhb_loop\@ |
212 | + dsb nsh |
213 | + isb |
214 | +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
215 | + .endm |
216 | + |
217 | + /* Save/restores x0-x3 to the stack */ |
218 | + .macro __mitigate_spectre_bhb_fw |
219 | +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
220 | + stp x0, x1, [sp, #-16]! |
221 | + stp x2, x3, [sp, #-16]! |
222 | + mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 |
223 | +alternative_cb arm64_update_smccc_conduit |
224 | + nop // Patched to SMC/HVC #0 |
225 | +alternative_cb_end |
226 | + ldp x2, x3, [sp], #16 |
227 | + ldp x0, x1, [sp], #16 |
228 | +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
229 | + .endm |
230 | #endif /* __ASM_ASSEMBLER_H */ |
231 | diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h |
232 | index 889226b4c6e1c..c7f17e663e729 100644 |
233 | --- a/arch/arm64/include/asm/cpu.h |
234 | +++ b/arch/arm64/include/asm/cpu.h |
235 | @@ -36,6 +36,7 @@ struct cpuinfo_arm64 { |
236 | u64 reg_id_aa64dfr1; |
237 | u64 reg_id_aa64isar0; |
238 | u64 reg_id_aa64isar1; |
239 | + u64 reg_id_aa64isar2; |
240 | u64 reg_id_aa64mmfr0; |
241 | u64 reg_id_aa64mmfr1; |
242 | u64 reg_id_aa64mmfr2; |
243 | diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h |
244 | index 8c7c4b23a8b18..9935e55a3cc75 100644 |
245 | --- a/arch/arm64/include/asm/cpucaps.h |
246 | +++ b/arch/arm64/include/asm/cpucaps.h |
247 | @@ -38,7 +38,9 @@ |
248 | #define ARM64_HARDEN_BRANCH_PREDICTOR 17 |
249 | #define ARM64_SSBD 18 |
250 | #define ARM64_MISMATCHED_CACHE_TYPE 19 |
251 | +#define ARM64_WORKAROUND_1188873 20 |
252 | +#define ARM64_SPECTRE_BHB 21 |
253 | |
254 | -#define ARM64_NCAPS 20 |
255 | +#define ARM64_NCAPS 22 |
256 | |
257 | #endif /* __ASM_CPUCAPS_H */ |
258 | diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h |
259 | index e7bef3d936d87..58a32511da8f1 100644 |
260 | --- a/arch/arm64/include/asm/cpufeature.h |
261 | +++ b/arch/arm64/include/asm/cpufeature.h |
262 | @@ -10,6 +10,7 @@ |
263 | #define __ASM_CPUFEATURE_H |
264 | |
265 | #include <asm/cpucaps.h> |
266 | +#include <asm/cputype.h> |
267 | #include <asm/hwcap.h> |
268 | #include <asm/sysreg.h> |
269 | |
270 | @@ -66,24 +67,173 @@ struct arm64_ftr_reg { |
271 | |
272 | extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; |
273 | |
274 | -/* scope of capability check */ |
275 | -enum { |
276 | - SCOPE_SYSTEM, |
277 | - SCOPE_LOCAL_CPU, |
278 | -}; |
279 | +/* |
280 | + * CPU capabilities: |
281 | + * |
282 | + * We use arm64_cpu_capabilities to represent system features, errata work |
283 | + * arounds (both used internally by kernel and tracked in cpu_hwcaps) and |
284 | + * ELF HWCAPs (which are exposed to user). |
285 | + * |
286 | + * To support systems with heterogeneous CPUs, we need to make sure that we |
287 | + * detect the capabilities correctly on the system and take appropriate |
288 | + * measures to ensure there are no incompatibilities. |
289 | + * |
290 | + * This comment tries to explain how we treat the capabilities. |
291 | + * Each capability has the following list of attributes : |
292 | + * |
293 | + * 1) Scope of Detection : The system detects a given capability by |
294 | + * performing some checks at runtime. This could be, e.g, checking the |
295 | + * value of a field in CPU ID feature register or checking the cpu |
296 | + * model. The capability provides a call back ( @matches() ) to |
297 | + * perform the check. Scope defines how the checks should be performed. |
298 | + * There are two cases: |
299 | + * |
300 | + * a) SCOPE_LOCAL_CPU: check all the CPUs and "detect" if at least one |
301 | + * matches. This implies, we have to run the check on all the |
302 | + * booting CPUs, until the system decides that state of the |
303 | + * capability is finalised. (See section 2 below) |
304 | + * Or |
305 | + * b) SCOPE_SYSTEM: check all the CPUs and "detect" if all the CPUs |
306 | + * matches. This implies, we run the check only once, when the |
307 | + * system decides to finalise the state of the capability. If the |
308 | + * capability relies on a field in one of the CPU ID feature |
309 | + * registers, we use the sanitised value of the register from the |
310 | + * CPU feature infrastructure to make the decision. |
311 | + * |
312 | + * The process of detection is usually denoted by "update" capability |
313 | + * state in the code. |
314 | + * |
315 | + * 2) Finalise the state : The kernel should finalise the state of a |
316 | + * capability at some point during its execution and take necessary |
317 | + * actions if any. Usually, this is done, after all the boot-time |
318 | + * enabled CPUs are brought up by the kernel, so that it can make |
319 | + * better decision based on the available set of CPUs. However, there |
320 | + * are some special cases, where the action is taken during the early |
321 | + * boot by the primary boot CPU. (e.g, running the kernel at EL2 with |
322 | + * Virtualisation Host Extensions). The kernel usually disallows any |
323 | + * changes to the state of a capability once it finalises the capability |
324 | + * and takes any action, as it may be impossible to execute the actions |
325 | + * safely. A CPU brought up after a capability is "finalised" is |
326 | + * referred to as "Late CPU" w.r.t the capability. e.g, all secondary |
327 | + * CPUs are treated "late CPUs" for capabilities determined by the boot |
328 | + * CPU. |
329 | + * |
330 | + * 3) Verification: When a CPU is brought online (e.g, by user or by the |
331 | + * kernel), the kernel should make sure that it is safe to use the CPU, |
332 | + * by verifying that the CPU is compliant with the state of the |
333 | + * capabilities finalised already. This happens via : |
334 | + * |
335 | + * secondary_start_kernel()-> check_local_cpu_capabilities() |
336 | + * |
337 | + * As explained in (2) above, capabilities could be finalised at |
338 | + * different points in the execution. Each CPU is verified against the |
339 | + * "finalised" capabilities and if there is a conflict, the kernel takes |
340 | + * an action, based on the severity (e.g, a CPU could be prevented from |
341 | + * booting or cause a kernel panic). The CPU is allowed to "affect" the |
342 | + * state of the capability, if it has not been finalised already. |
343 | + * See section 5 for more details on conflicts. |
344 | + * |
345 | + * 4) Action: As mentioned in (2), the kernel can take an action for each |
346 | + * detected capability, on all CPUs on the system. Appropriate actions |
347 | + * include, turning on an architectural feature, modifying the control |
348 | + * registers (e.g, SCTLR, TCR etc.) or patching the kernel via |
349 | + * alternatives. The kernel patching is batched and performed at later |
350 | + * point. The actions are always initiated only after the capability |
351 | + * is finalised. This is usally denoted by "enabling" the capability. |
352 | + * The actions are initiated as follows : |
353 | + * a) Action is triggered on all online CPUs, after the capability is |
354 | + * finalised, invoked within the stop_machine() context from |
355 | + * enable_cpu_capabilitie(). |
356 | + * |
357 | + * b) Any late CPU, brought up after (1), the action is triggered via: |
358 | + * |
359 | + * check_local_cpu_capabilities() -> verify_local_cpu_capabilities() |
360 | + * |
361 | + * 5) Conflicts: Based on the state of the capability on a late CPU vs. |
362 | + * the system state, we could have the following combinations : |
363 | + * |
364 | + * x-----------------------------x |
365 | + * | Type | System | Late CPU | |
366 | + * |-----------------------------| |
367 | + * | a | y | n | |
368 | + * |-----------------------------| |
369 | + * | b | n | y | |
370 | + * x-----------------------------x |
371 | + * |
372 | + * Two separate flag bits are defined to indicate whether each kind of |
373 | + * conflict can be allowed: |
374 | + * ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU - Case(a) is allowed |
375 | + * ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU - Case(b) is allowed |
376 | + * |
377 | + * Case (a) is not permitted for a capability that the system requires |
378 | + * all CPUs to have in order for the capability to be enabled. This is |
379 | + * typical for capabilities that represent enhanced functionality. |
380 | + * |
381 | + * Case (b) is not permitted for a capability that must be enabled |
382 | + * during boot if any CPU in the system requires it in order to run |
383 | + * safely. This is typical for erratum work arounds that cannot be |
384 | + * enabled after the corresponding capability is finalised. |
385 | + * |
386 | + * In some non-typical cases either both (a) and (b), or neither, |
387 | + * should be permitted. This can be described by including neither |
388 | + * or both flags in the capability's type field. |
389 | + */ |
390 | + |
391 | + |
392 | +/* Decide how the capability is detected. On a local CPU vs System wide */ |
393 | +#define ARM64_CPUCAP_SCOPE_LOCAL_CPU ((u16)BIT(0)) |
394 | +#define ARM64_CPUCAP_SCOPE_SYSTEM ((u16)BIT(1)) |
395 | +#define ARM64_CPUCAP_SCOPE_MASK \ |
396 | + (ARM64_CPUCAP_SCOPE_SYSTEM | \ |
397 | + ARM64_CPUCAP_SCOPE_LOCAL_CPU) |
398 | + |
399 | +#define SCOPE_SYSTEM ARM64_CPUCAP_SCOPE_SYSTEM |
400 | +#define SCOPE_LOCAL_CPU ARM64_CPUCAP_SCOPE_LOCAL_CPU |
401 | + |
402 | +/* |
403 | + * Is it permitted for a late CPU to have this capability when system |
404 | + * hasn't already enabled it ? |
405 | + */ |
406 | +#define ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU ((u16)BIT(4)) |
407 | +/* Is it safe for a late CPU to miss this capability when system has it */ |
408 | +#define ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU ((u16)BIT(5)) |
409 | + |
410 | +/* |
411 | + * CPU errata workarounds that need to be enabled at boot time if one or |
412 | + * more CPUs in the system requires it. When one of these capabilities |
413 | + * has been enabled, it is safe to allow any CPU to boot that doesn't |
414 | + * require the workaround. However, it is not safe if a "late" CPU |
415 | + * requires a workaround and the system hasn't enabled it already. |
416 | + */ |
417 | +#define ARM64_CPUCAP_LOCAL_CPU_ERRATUM \ |
418 | + (ARM64_CPUCAP_SCOPE_LOCAL_CPU | ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU) |
419 | +/* |
420 | + * CPU feature detected at boot time based on system-wide value of a |
421 | + * feature. It is safe for a late CPU to have this feature even though |
422 | + * the system hasn't enabled it, although the featuer will not be used |
423 | + * by Linux in this case. If the system has enabled this feature already, |
424 | + * then every late CPU must have it. |
425 | + */ |
426 | +#define ARM64_CPUCAP_SYSTEM_FEATURE \ |
427 | + (ARM64_CPUCAP_SCOPE_SYSTEM | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU) |
428 | |
429 | struct arm64_cpu_capabilities { |
430 | const char *desc; |
431 | u16 capability; |
432 | - int def_scope; /* default scope */ |
433 | + u16 type; |
434 | bool (*matches)(const struct arm64_cpu_capabilities *caps, int scope); |
435 | - int (*enable)(void *); /* Called on all active CPUs */ |
436 | + /* |
437 | + * Take the appropriate actions to enable this capability for this CPU. |
438 | + * For each successfully booted CPU, this method is called for each |
439 | + * globally detected capability. |
440 | + */ |
441 | + void (*cpu_enable)(const struct arm64_cpu_capabilities *cap); |
442 | union { |
443 | struct { /* To be used for erratum handling only */ |
444 | - u32 midr_model; |
445 | - u32 midr_range_min, midr_range_max; |
446 | + struct midr_range midr_range; |
447 | }; |
448 | |
449 | + const struct midr_range *midr_range_list; |
450 | struct { /* Feature register checking */ |
451 | u32 sys_reg; |
452 | u8 field_pos; |
453 | @@ -95,6 +245,23 @@ struct arm64_cpu_capabilities { |
454 | }; |
455 | }; |
456 | |
457 | +static inline int cpucap_default_scope(const struct arm64_cpu_capabilities *cap) |
458 | +{ |
459 | + return cap->type & ARM64_CPUCAP_SCOPE_MASK; |
460 | +} |
461 | + |
462 | +static inline bool |
463 | +cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap) |
464 | +{ |
465 | + return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU); |
466 | +} |
467 | + |
468 | +static inline bool |
469 | +cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap) |
470 | +{ |
471 | + return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU); |
472 | +} |
473 | + |
474 | extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); |
475 | extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS]; |
476 | extern struct static_key_false arm64_const_caps_ready; |
477 | @@ -195,15 +362,8 @@ static inline bool id_aa64pfr0_32bit_el0(u64 pfr0) |
478 | } |
479 | |
480 | void __init setup_cpu_features(void); |
481 | - |
482 | -void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, |
483 | - const char *info); |
484 | -void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps); |
485 | void check_local_cpu_capabilities(void); |
486 | |
487 | -void update_cpu_errata_workarounds(void); |
488 | -void __init enable_errata_workarounds(void); |
489 | -void verify_local_cpu_errata_workarounds(void); |
490 | |
491 | u64 read_system_reg(u32 id); |
492 | |
493 | @@ -212,6 +372,34 @@ static inline bool cpu_supports_mixed_endian_el0(void) |
494 | return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); |
495 | } |
496 | |
497 | +static inline bool supports_csv2p3(int scope) |
498 | +{ |
499 | + u64 pfr0; |
500 | + u8 csv2_val; |
501 | + |
502 | + if (scope == SCOPE_LOCAL_CPU) |
503 | + pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1); |
504 | + else |
505 | + pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1); |
506 | + |
507 | + csv2_val = cpuid_feature_extract_unsigned_field(pfr0, |
508 | + ID_AA64PFR0_CSV2_SHIFT); |
509 | + return csv2_val == 3; |
510 | +} |
511 | + |
512 | +static inline bool supports_clearbhb(int scope) |
513 | +{ |
514 | + u64 isar2; |
515 | + |
516 | + if (scope == SCOPE_LOCAL_CPU) |
517 | + isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1); |
518 | + else |
519 | + isar2 = read_system_reg(SYS_ID_AA64ISAR2_EL1); |
520 | + |
521 | + return cpuid_feature_extract_unsigned_field(isar2, |
522 | + ID_AA64ISAR2_CLEARBHB_SHIFT); |
523 | +} |
524 | + |
525 | static inline bool system_supports_32bit_el0(void) |
526 | { |
527 | return cpus_have_const_cap(ARM64_HAS_32BIT_EL0); |
528 | @@ -244,6 +432,18 @@ void arm64_set_ssbd_mitigation(bool state); |
529 | static inline void arm64_set_ssbd_mitigation(bool state) {} |
530 | #endif |
531 | |
532 | +/* Watch out, ordering is important here. */ |
533 | +enum mitigation_state { |
534 | + SPECTRE_UNAFFECTED, |
535 | + SPECTRE_MITIGATED, |
536 | + SPECTRE_VULNERABLE, |
537 | +}; |
538 | + |
539 | +enum mitigation_state arm64_get_spectre_bhb_state(void); |
540 | +bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope); |
541 | +u8 spectre_bhb_loop_affected(int scope); |
542 | +void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry); |
543 | + |
544 | #endif /* __ASSEMBLY__ */ |
545 | |
546 | #endif |
547 | diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h |
548 | index 39d1db68748db..7ffa41caa4178 100644 |
549 | --- a/arch/arm64/include/asm/cputype.h |
550 | +++ b/arch/arm64/include/asm/cputype.h |
551 | @@ -83,6 +83,18 @@ |
552 | #define ARM_CPU_PART_CORTEX_A53 0xD03 |
553 | #define ARM_CPU_PART_CORTEX_A73 0xD09 |
554 | #define ARM_CPU_PART_CORTEX_A75 0xD0A |
555 | +#define ARM_CPU_PART_CORTEX_A35 0xD04 |
556 | +#define ARM_CPU_PART_CORTEX_A55 0xD05 |
557 | +#define ARM_CPU_PART_CORTEX_A76 0xD0B |
558 | +#define ARM_CPU_PART_NEOVERSE_N1 0xD0C |
559 | +#define ARM_CPU_PART_CORTEX_A77 0xD0D |
560 | +#define ARM_CPU_PART_NEOVERSE_V1 0xD40 |
561 | +#define ARM_CPU_PART_CORTEX_A78 0xD41 |
562 | +#define ARM_CPU_PART_CORTEX_X1 0xD44 |
563 | +#define ARM_CPU_PART_CORTEX_A710 0xD47 |
564 | +#define ARM_CPU_PART_CORTEX_X2 0xD48 |
565 | +#define ARM_CPU_PART_NEOVERSE_N2 0xD49 |
566 | +#define ARM_CPU_PART_CORTEX_A78C 0xD4B |
567 | |
568 | #define APM_CPU_PART_POTENZA 0x000 |
569 | |
570 | @@ -98,6 +110,18 @@ |
571 | #define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) |
572 | #define MIDR_CORTEX_A73 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73) |
573 | #define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75) |
574 | +#define MIDR_CORTEX_A35 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35) |
575 | +#define MIDR_CORTEX_A55 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55) |
576 | +#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) |
577 | +#define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1) |
578 | +#define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77) |
579 | +#define MIDR_NEOVERSE_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V1) |
580 | +#define MIDR_CORTEX_A78 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78) |
581 | +#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1) |
582 | +#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) |
583 | +#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2) |
584 | +#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2) |
585 | +#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C) |
586 | #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) |
587 | #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) |
588 | #define MIDR_CAVIUM_THUNDERX2 MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2) |
589 | @@ -109,6 +133,45 @@ |
590 | |
591 | #define read_cpuid(reg) read_sysreg_s(SYS_ ## reg) |
592 | |
593 | +/* |
594 | + * Represent a range of MIDR values for a given CPU model and a |
595 | + * range of variant/revision values. |
596 | + * |
597 | + * @model - CPU model as defined by MIDR_CPU_MODEL |
598 | + * @rv_min - Minimum value for the revision/variant as defined by |
599 | + * MIDR_CPU_VAR_REV |
600 | + * @rv_max - Maximum value for the variant/revision for the range. |
601 | + */ |
602 | +struct midr_range { |
603 | + u32 model; |
604 | + u32 rv_min; |
605 | + u32 rv_max; |
606 | +}; |
607 | + |
608 | +#define MIDR_RANGE(m, v_min, r_min, v_max, r_max) \ |
609 | + { \ |
610 | + .model = m, \ |
611 | + .rv_min = MIDR_CPU_VAR_REV(v_min, r_min), \ |
612 | + .rv_max = MIDR_CPU_VAR_REV(v_max, r_max), \ |
613 | + } |
614 | + |
615 | +#define MIDR_ALL_VERSIONS(m) MIDR_RANGE(m, 0, 0, 0xf, 0xf) |
616 | + |
617 | +static inline bool is_midr_in_range(u32 midr, struct midr_range const *range) |
618 | +{ |
619 | + return MIDR_IS_CPU_MODEL_RANGE(midr, range->model, |
620 | + range->rv_min, range->rv_max); |
621 | +} |
622 | + |
623 | +static inline bool |
624 | +is_midr_in_range_list(u32 midr, struct midr_range const *ranges) |
625 | +{ |
626 | + while (ranges->model) |
627 | + if (is_midr_in_range(midr, ranges++)) |
628 | + return true; |
629 | + return false; |
630 | +} |
631 | + |
632 | /* |
633 | * The CPU ID never changes at run time, so we might as well tell the |
634 | * compiler that it's constant. Use this function to read the CPU ID |
635 | diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h |
636 | index d8e58051f32d4..4ffe0d698fa72 100644 |
637 | --- a/arch/arm64/include/asm/fixmap.h |
638 | +++ b/arch/arm64/include/asm/fixmap.h |
639 | @@ -53,9 +53,11 @@ enum fixed_addresses { |
640 | FIX_TEXT_POKE0, |
641 | |
642 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
643 | + FIX_ENTRY_TRAMP_TEXT3, |
644 | + FIX_ENTRY_TRAMP_TEXT2, |
645 | + FIX_ENTRY_TRAMP_TEXT1, |
646 | FIX_ENTRY_TRAMP_DATA, |
647 | - FIX_ENTRY_TRAMP_TEXT, |
648 | -#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT)) |
649 | +#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1)) |
650 | #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ |
651 | __end_of_permanent_fixed_addresses, |
652 | |
653 | diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h |
654 | index bc853663dd517..aecc07e09a18b 100644 |
655 | --- a/arch/arm64/include/asm/insn.h |
656 | +++ b/arch/arm64/include/asm/insn.h |
657 | @@ -332,6 +332,8 @@ bool aarch64_insn_is_branch(u32 insn); |
658 | u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn); |
659 | u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, |
660 | u32 insn, u64 imm); |
661 | +u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type, |
662 | + u32 insn); |
663 | u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, |
664 | enum aarch64_insn_branch_type type); |
665 | u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr, |
666 | diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h |
667 | index a152a7bbc85ab..a75f02e5f0fdc 100644 |
668 | --- a/arch/arm64/include/asm/kvm_host.h |
669 | +++ b/arch/arm64/include/asm/kvm_host.h |
670 | @@ -452,4 +452,8 @@ static inline int kvm_arm_have_ssbd(void) |
671 | } |
672 | } |
673 | |
674 | +static inline bool kvm_arm_spectre_bhb_mitigated(void) |
675 | +{ |
676 | + return arm64_get_spectre_bhb_state() == SPECTRE_MITIGATED; |
677 | +} |
678 | #endif /* __ARM64_KVM_HOST_H__ */ |
679 | diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h |
680 | index ff721659eb94b..4a2c958548568 100644 |
681 | --- a/arch/arm64/include/asm/kvm_mmu.h |
682 | +++ b/arch/arm64/include/asm/kvm_mmu.h |
683 | @@ -362,7 +362,7 @@ static inline void *kvm_get_hyp_vector(void) |
684 | struct bp_hardening_data *data = arm64_get_bp_hardening_data(); |
685 | void *vect = kvm_ksym_ref(__kvm_hyp_vector); |
686 | |
687 | - if (data->fn) { |
688 | + if (data->template_start) { |
689 | vect = __bp_harden_hyp_vecs_start + |
690 | data->hyp_vectors_slot * SZ_2K; |
691 | |
692 | diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h |
693 | index 6ac34c75f4e13..f4377b005cba9 100644 |
694 | --- a/arch/arm64/include/asm/mmu.h |
695 | +++ b/arch/arm64/include/asm/mmu.h |
696 | @@ -34,7 +34,7 @@ typedef struct { |
697 | */ |
698 | #define ASID(mm) ((mm)->context.id.counter & 0xffff) |
699 | |
700 | -static inline bool arm64_kernel_unmapped_at_el0(void) |
701 | +static __always_inline bool arm64_kernel_unmapped_at_el0(void) |
702 | { |
703 | return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) && |
704 | cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0); |
705 | @@ -45,6 +45,12 @@ typedef void (*bp_hardening_cb_t)(void); |
706 | struct bp_hardening_data { |
707 | int hyp_vectors_slot; |
708 | bp_hardening_cb_t fn; |
709 | + |
710 | + /* |
711 | + * template_start is only used by the BHB mitigation to identify the |
712 | + * hyp_vectors_slot sequence. |
713 | + */ |
714 | + const char *template_start; |
715 | }; |
716 | |
717 | #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
718 | diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h |
719 | index 9ee660013e5cb..367141e05c346 100644 |
720 | --- a/arch/arm64/include/asm/processor.h |
721 | +++ b/arch/arm64/include/asm/processor.h |
722 | @@ -37,6 +37,7 @@ |
723 | #include <linux/string.h> |
724 | |
725 | #include <asm/alternative.h> |
726 | +#include <asm/cpufeature.h> |
727 | #include <asm/fpsimd.h> |
728 | #include <asm/hw_breakpoint.h> |
729 | #include <asm/lse.h> |
730 | @@ -219,9 +220,8 @@ static inline void spin_lock_prefetch(const void *ptr) |
731 | |
732 | #endif |
733 | |
734 | -int cpu_enable_pan(void *__unused); |
735 | -int cpu_enable_uao(void *__unused); |
736 | -int cpu_enable_cache_maint_trap(void *__unused); |
737 | +void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused); |
738 | +void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused); |
739 | |
740 | #endif /* __ASSEMBLY__ */ |
741 | #endif /* __ASM_PROCESSOR_H */ |
742 | diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h |
743 | index 4e7e7067afdb0..09ebd37d5aa3c 100644 |
744 | --- a/arch/arm64/include/asm/sections.h |
745 | +++ b/arch/arm64/include/asm/sections.h |
746 | @@ -26,5 +26,11 @@ extern char __hyp_text_start[], __hyp_text_end[]; |
747 | extern char __idmap_text_start[], __idmap_text_end[]; |
748 | extern char __irqentry_text_start[], __irqentry_text_end[]; |
749 | extern char __mmuoff_data_start[], __mmuoff_data_end[]; |
750 | +extern char __entry_tramp_text_start[], __entry_tramp_text_end[]; |
751 | + |
752 | +static inline size_t entry_tramp_text_size(void) |
753 | +{ |
754 | + return __entry_tramp_text_end - __entry_tramp_text_start; |
755 | +} |
756 | |
757 | #endif /* __ASM_SECTIONS_H */ |
758 | diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h |
759 | index ae1b31d02784c..46e97be12e028 100644 |
760 | --- a/arch/arm64/include/asm/sysreg.h |
761 | +++ b/arch/arm64/include/asm/sysreg.h |
762 | @@ -70,6 +70,7 @@ |
763 | |
764 | #define SYS_ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0) |
765 | #define SYS_ID_AA64ISAR1_EL1 sys_reg(3, 0, 0, 6, 1) |
766 | +#define SYS_ID_AA64ISAR2_EL1 sys_reg(3, 0, 0, 6, 2) |
767 | |
768 | #define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0) |
769 | #define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1) |
770 | @@ -173,6 +174,9 @@ |
771 | #define ID_AA64ISAR0_SHA1_SHIFT 8 |
772 | #define ID_AA64ISAR0_AES_SHIFT 4 |
773 | |
774 | +/* id_aa64isar2 */ |
775 | +#define ID_AA64ISAR2_CLEARBHB_SHIFT 28 |
776 | + |
777 | /* id_aa64pfr0 */ |
778 | #define ID_AA64PFR0_CSV3_SHIFT 60 |
779 | #define ID_AA64PFR0_CSV2_SHIFT 56 |
780 | @@ -211,6 +215,7 @@ |
781 | #define ID_AA64MMFR0_TGRAN16_SUPPORTED 0x1 |
782 | |
783 | /* id_aa64mmfr1 */ |
784 | +#define ID_AA64MMFR1_ECBHB_SHIFT 60 |
785 | #define ID_AA64MMFR1_PAN_SHIFT 20 |
786 | #define ID_AA64MMFR1_LOR_SHIFT 16 |
787 | #define ID_AA64MMFR1_HPD_SHIFT 12 |
788 | diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h |
789 | new file mode 100644 |
790 | index 0000000000000..695583b9a145b |
791 | --- /dev/null |
792 | +++ b/arch/arm64/include/asm/vectors.h |
793 | @@ -0,0 +1,74 @@ |
794 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
795 | +/* |
796 | + * Copyright (C) 2022 ARM Ltd. |
797 | + */ |
798 | +#ifndef __ASM_VECTORS_H |
799 | +#define __ASM_VECTORS_H |
800 | + |
801 | +#include <linux/bug.h> |
802 | +#include <linux/percpu.h> |
803 | + |
804 | +#include <asm/fixmap.h> |
805 | +#include <asm/mmu.h> |
806 | + |
807 | +extern char vectors[]; |
808 | +extern char tramp_vectors[]; |
809 | +extern char __bp_harden_el1_vectors[]; |
810 | + |
811 | +/* |
812 | + * Note: the order of this enum corresponds to two arrays in entry.S: |
813 | + * tramp_vecs and __bp_harden_el1_vectors. By default the canonical |
814 | + * 'full fat' vectors are used directly. |
815 | + */ |
816 | +enum arm64_bp_harden_el1_vectors { |
817 | +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
818 | + /* |
819 | + * Perform the BHB loop mitigation, before branching to the canonical |
820 | + * vectors. |
821 | + */ |
822 | + EL1_VECTOR_BHB_LOOP, |
823 | + |
824 | + /* |
825 | + * Make the SMC call for firmware mitigation, before branching to the |
826 | + * canonical vectors. |
827 | + */ |
828 | + EL1_VECTOR_BHB_FW, |
829 | + |
830 | + /* |
831 | + * Use the ClearBHB instruction, before branching to the canonical |
832 | + * vectors. |
833 | + */ |
834 | + EL1_VECTOR_BHB_CLEAR_INSN, |
835 | +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
836 | + |
837 | + /* |
838 | + * Remap the kernel before branching to the canonical vectors. |
839 | + */ |
840 | + EL1_VECTOR_KPTI, |
841 | +}; |
842 | + |
843 | +#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
844 | +#define EL1_VECTOR_BHB_LOOP -1 |
845 | +#define EL1_VECTOR_BHB_FW -1 |
846 | +#define EL1_VECTOR_BHB_CLEAR_INSN -1 |
847 | +#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
848 | + |
849 | +/* The vectors to use on return from EL0. e.g. to remap the kernel */ |
850 | +DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector); |
851 | + |
852 | +#ifndef CONFIG_UNMAP_KERNEL_AT_EL0 |
853 | +#define TRAMP_VALIAS 0 |
854 | +#endif |
855 | + |
856 | +static inline const char * |
857 | +arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot) |
858 | +{ |
859 | + if (arm64_kernel_unmapped_at_el0()) |
860 | + return (char *)TRAMP_VALIAS + SZ_2K * slot; |
861 | + |
862 | + WARN_ON_ONCE(slot == EL1_VECTOR_KPTI); |
863 | + |
864 | + return __bp_harden_el1_vectors + SZ_2K * slot; |
865 | +} |
866 | + |
867 | +#endif /* __ASM_VECTORS_H */ |
868 | diff --git a/arch/arm64/kernel/bpi.S b/arch/arm64/kernel/bpi.S |
869 | index dc4eb154e33b5..d3fd8bf42d868 100644 |
870 | --- a/arch/arm64/kernel/bpi.S |
871 | +++ b/arch/arm64/kernel/bpi.S |
872 | @@ -73,3 +73,58 @@ ENTRY(__smccc_workaround_1_smc_end) |
873 | ENTRY(__smccc_workaround_1_hvc_start) |
874 | smccc_workaround_1 hvc |
875 | ENTRY(__smccc_workaround_1_hvc_end) |
876 | + |
877 | +ENTRY(__smccc_workaround_3_smc_start) |
878 | + sub sp, sp, #(8 * 4) |
879 | + stp x2, x3, [sp, #(8 * 0)] |
880 | + stp x0, x1, [sp, #(8 * 2)] |
881 | + mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 |
882 | + smc #0 |
883 | + ldp x2, x3, [sp, #(8 * 0)] |
884 | + ldp x0, x1, [sp, #(8 * 2)] |
885 | + add sp, sp, #(8 * 4) |
886 | +ENTRY(__smccc_workaround_3_smc_end) |
887 | + |
888 | +ENTRY(__spectre_bhb_loop_k8_start) |
889 | + sub sp, sp, #(8 * 2) |
890 | + stp x0, x1, [sp, #(8 * 0)] |
891 | + mov x0, #8 |
892 | +2: b . + 4 |
893 | + subs x0, x0, #1 |
894 | + b.ne 2b |
895 | + dsb nsh |
896 | + isb |
897 | + ldp x0, x1, [sp, #(8 * 0)] |
898 | + add sp, sp, #(8 * 2) |
899 | +ENTRY(__spectre_bhb_loop_k8_end) |
900 | + |
901 | +ENTRY(__spectre_bhb_loop_k24_start) |
902 | + sub sp, sp, #(8 * 2) |
903 | + stp x0, x1, [sp, #(8 * 0)] |
904 | + mov x0, #24 |
905 | +2: b . + 4 |
906 | + subs x0, x0, #1 |
907 | + b.ne 2b |
908 | + dsb nsh |
909 | + isb |
910 | + ldp x0, x1, [sp, #(8 * 0)] |
911 | + add sp, sp, #(8 * 2) |
912 | +ENTRY(__spectre_bhb_loop_k24_end) |
913 | + |
914 | +ENTRY(__spectre_bhb_loop_k32_start) |
915 | + sub sp, sp, #(8 * 2) |
916 | + stp x0, x1, [sp, #(8 * 0)] |
917 | + mov x0, #32 |
918 | +2: b . + 4 |
919 | + subs x0, x0, #1 |
920 | + b.ne 2b |
921 | + dsb nsh |
922 | + isb |
923 | + ldp x0, x1, [sp, #(8 * 0)] |
924 | + add sp, sp, #(8 * 2) |
925 | +ENTRY(__spectre_bhb_loop_k32_end) |
926 | + |
927 | +ENTRY(__spectre_bhb_clearbhb_start) |
928 | + hint #22 /* aka clearbhb */ |
929 | + isb |
930 | +ENTRY(__spectre_bhb_clearbhb_end) |
931 | diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c |
932 | index 3b680a32886b9..f0cdf21b1006b 100644 |
933 | --- a/arch/arm64/kernel/cpu_errata.c |
934 | +++ b/arch/arm64/kernel/cpu_errata.c |
935 | @@ -23,14 +23,23 @@ |
936 | #include <asm/cpu.h> |
937 | #include <asm/cputype.h> |
938 | #include <asm/cpufeature.h> |
939 | +#include <asm/vectors.h> |
940 | |
941 | static bool __maybe_unused |
942 | is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope) |
943 | { |
944 | + u32 midr = read_cpuid_id(); |
945 | + |
946 | WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); |
947 | - return MIDR_IS_CPU_MODEL_RANGE(read_cpuid_id(), entry->midr_model, |
948 | - entry->midr_range_min, |
949 | - entry->midr_range_max); |
950 | + return is_midr_in_range(midr, &entry->midr_range); |
951 | +} |
952 | + |
953 | +static bool __maybe_unused |
954 | +is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry, |
955 | + int scope) |
956 | +{ |
957 | + WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); |
958 | + return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list); |
959 | } |
960 | |
961 | static bool |
962 | @@ -48,17 +57,18 @@ has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry, |
963 | (arm64_ftr_reg_ctrel0.sys_val & mask); |
964 | } |
965 | |
966 | -static int cpu_enable_trap_ctr_access(void *__unused) |
967 | +static void |
968 | +cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused) |
969 | { |
970 | /* Clear SCTLR_EL1.UCT */ |
971 | config_sctlr_el1(SCTLR_EL1_UCT, 0); |
972 | - return 0; |
973 | } |
974 | |
975 | #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
976 | #include <asm/mmu_context.h> |
977 | #include <asm/cacheflush.h> |
978 | |
979 | +static bool __hardenbp_enab; |
980 | DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); |
981 | |
982 | #ifdef CONFIG_KVM |
983 | @@ -66,6 +76,16 @@ extern char __smccc_workaround_1_smc_start[]; |
984 | extern char __smccc_workaround_1_smc_end[]; |
985 | extern char __smccc_workaround_1_hvc_start[]; |
986 | extern char __smccc_workaround_1_hvc_end[]; |
987 | +extern char __smccc_workaround_3_smc_start[]; |
988 | +extern char __smccc_workaround_3_smc_end[]; |
989 | +extern char __spectre_bhb_loop_k8_start[]; |
990 | +extern char __spectre_bhb_loop_k8_end[]; |
991 | +extern char __spectre_bhb_loop_k24_start[]; |
992 | +extern char __spectre_bhb_loop_k24_end[]; |
993 | +extern char __spectre_bhb_loop_k32_start[]; |
994 | +extern char __spectre_bhb_loop_k32_end[]; |
995 | +extern char __spectre_bhb_clearbhb_start[]; |
996 | +extern char __spectre_bhb_clearbhb_end[]; |
997 | |
998 | static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start, |
999 | const char *hyp_vecs_end) |
1000 | @@ -79,12 +99,14 @@ static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start, |
1001 | flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K); |
1002 | } |
1003 | |
1004 | +static DEFINE_SPINLOCK(bp_lock); |
1005 | +static int last_slot = -1; |
1006 | + |
1007 | static void __install_bp_hardening_cb(bp_hardening_cb_t fn, |
1008 | const char *hyp_vecs_start, |
1009 | const char *hyp_vecs_end) |
1010 | { |
1011 | - static int last_slot = -1; |
1012 | - static DEFINE_SPINLOCK(bp_lock); |
1013 | + |
1014 | int cpu, slot = -1; |
1015 | |
1016 | spin_lock(&bp_lock); |
1017 | @@ -105,6 +127,8 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn, |
1018 | |
1019 | __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot); |
1020 | __this_cpu_write(bp_hardening_data.fn, fn); |
1021 | + __this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start); |
1022 | + __hardenbp_enab = true; |
1023 | spin_unlock(&bp_lock); |
1024 | } |
1025 | #else |
1026 | @@ -118,6 +142,7 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn, |
1027 | const char *hyp_vecs_end) |
1028 | { |
1029 | __this_cpu_write(bp_hardening_data.fn, fn); |
1030 | + __hardenbp_enab = true; |
1031 | } |
1032 | #endif /* CONFIG_KVM */ |
1033 | |
1034 | @@ -152,25 +177,25 @@ static void call_hvc_arch_workaround_1(void) |
1035 | arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); |
1036 | } |
1037 | |
1038 | -static int enable_smccc_arch_workaround_1(void *data) |
1039 | +static void |
1040 | +enable_smccc_arch_workaround_1(const struct arm64_cpu_capabilities *entry) |
1041 | { |
1042 | - const struct arm64_cpu_capabilities *entry = data; |
1043 | bp_hardening_cb_t cb; |
1044 | void *smccc_start, *smccc_end; |
1045 | struct arm_smccc_res res; |
1046 | |
1047 | if (!entry->matches(entry, SCOPE_LOCAL_CPU)) |
1048 | - return 0; |
1049 | + return; |
1050 | |
1051 | if (psci_ops.smccc_version == SMCCC_VERSION_1_0) |
1052 | - return 0; |
1053 | + return; |
1054 | |
1055 | switch (psci_ops.conduit) { |
1056 | case PSCI_CONDUIT_HVC: |
1057 | arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
1058 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); |
1059 | if ((int)res.a0 < 0) |
1060 | - return 0; |
1061 | + return; |
1062 | cb = call_hvc_arch_workaround_1; |
1063 | smccc_start = __smccc_workaround_1_hvc_start; |
1064 | smccc_end = __smccc_workaround_1_hvc_end; |
1065 | @@ -180,22 +205,44 @@ static int enable_smccc_arch_workaround_1(void *data) |
1066 | arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
1067 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); |
1068 | if ((int)res.a0 < 0) |
1069 | - return 0; |
1070 | + return; |
1071 | cb = call_smc_arch_workaround_1; |
1072 | smccc_start = __smccc_workaround_1_smc_start; |
1073 | smccc_end = __smccc_workaround_1_smc_end; |
1074 | break; |
1075 | |
1076 | default: |
1077 | - return 0; |
1078 | + return; |
1079 | } |
1080 | |
1081 | install_bp_hardening_cb(entry, cb, smccc_start, smccc_end); |
1082 | |
1083 | - return 0; |
1084 | + return; |
1085 | } |
1086 | #endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */ |
1087 | |
1088 | +void __init arm64_update_smccc_conduit(struct alt_instr *alt, |
1089 | + __le32 *origptr, __le32 *updptr, |
1090 | + int nr_inst) |
1091 | +{ |
1092 | + u32 insn; |
1093 | + |
1094 | + BUG_ON(nr_inst != 1); |
1095 | + |
1096 | + switch (psci_ops.conduit) { |
1097 | + case PSCI_CONDUIT_HVC: |
1098 | + insn = aarch64_insn_get_hvc_value(); |
1099 | + break; |
1100 | + case PSCI_CONDUIT_SMC: |
1101 | + insn = aarch64_insn_get_smc_value(); |
1102 | + break; |
1103 | + default: |
1104 | + return; |
1105 | + } |
1106 | + |
1107 | + *updptr = cpu_to_le32(insn); |
1108 | +} |
1109 | + |
1110 | #ifdef CONFIG_ARM64_SSBD |
1111 | DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required); |
1112 | |
1113 | @@ -231,28 +278,6 @@ static int __init ssbd_cfg(char *buf) |
1114 | } |
1115 | early_param("ssbd", ssbd_cfg); |
1116 | |
1117 | -void __init arm64_update_smccc_conduit(struct alt_instr *alt, |
1118 | - __le32 *origptr, __le32 *updptr, |
1119 | - int nr_inst) |
1120 | -{ |
1121 | - u32 insn; |
1122 | - |
1123 | - BUG_ON(nr_inst != 1); |
1124 | - |
1125 | - switch (psci_ops.conduit) { |
1126 | - case PSCI_CONDUIT_HVC: |
1127 | - insn = aarch64_insn_get_hvc_value(); |
1128 | - break; |
1129 | - case PSCI_CONDUIT_SMC: |
1130 | - insn = aarch64_insn_get_smc_value(); |
1131 | - break; |
1132 | - default: |
1133 | - return; |
1134 | - } |
1135 | - |
1136 | - *updptr = cpu_to_le32(insn); |
1137 | -} |
1138 | - |
1139 | void __init arm64_enable_wa2_handling(struct alt_instr *alt, |
1140 | __le32 *origptr, __le32 *updptr, |
1141 | int nr_inst) |
1142 | @@ -368,19 +393,60 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry, |
1143 | } |
1144 | #endif /* CONFIG_ARM64_SSBD */ |
1145 | |
1146 | -#define MIDR_RANGE(model, min, max) \ |
1147 | - .def_scope = SCOPE_LOCAL_CPU, \ |
1148 | - .matches = is_affected_midr_range, \ |
1149 | - .midr_model = model, \ |
1150 | - .midr_range_min = min, \ |
1151 | - .midr_range_max = max |
1152 | +#define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max) \ |
1153 | + .matches = is_affected_midr_range, \ |
1154 | + .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max) |
1155 | + |
1156 | +#define CAP_MIDR_ALL_VERSIONS(model) \ |
1157 | + .matches = is_affected_midr_range, \ |
1158 | + .midr_range = MIDR_ALL_VERSIONS(model) |
1159 | + |
1160 | +#define MIDR_FIXED(rev, revidr_mask) \ |
1161 | + .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}} |
1162 | + |
1163 | +#define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max) \ |
1164 | + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, \ |
1165 | + CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max) |
1166 | + |
1167 | +#define CAP_MIDR_RANGE_LIST(list) \ |
1168 | + .matches = is_affected_midr_range_list, \ |
1169 | + .midr_range_list = list |
1170 | |
1171 | -#define MIDR_ALL_VERSIONS(model) \ |
1172 | - .def_scope = SCOPE_LOCAL_CPU, \ |
1173 | - .matches = is_affected_midr_range, \ |
1174 | - .midr_model = model, \ |
1175 | - .midr_range_min = 0, \ |
1176 | - .midr_range_max = (MIDR_VARIANT_MASK | MIDR_REVISION_MASK) |
1177 | +/* Errata affecting a range of revisions of given model variant */ |
1178 | +#define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max) \ |
1179 | + ERRATA_MIDR_RANGE(m, var, r_min, var, r_max) |
1180 | + |
1181 | +/* Errata affecting a single variant/revision of a model */ |
1182 | +#define ERRATA_MIDR_REV(model, var, rev) \ |
1183 | + ERRATA_MIDR_RANGE(model, var, rev, var, rev) |
1184 | + |
1185 | +/* Errata affecting all variants/revisions of a given a model */ |
1186 | +#define ERRATA_MIDR_ALL_VERSIONS(model) \ |
1187 | + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, \ |
1188 | + CAP_MIDR_ALL_VERSIONS(model) |
1189 | + |
1190 | +/* Errata affecting a list of midr ranges, with same work around */ |
1191 | +#define ERRATA_MIDR_RANGE_LIST(midr_list) \ |
1192 | + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, \ |
1193 | + CAP_MIDR_RANGE_LIST(midr_list) |
1194 | + |
1195 | +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
1196 | + |
1197 | +/* |
1198 | + * List of CPUs where we need to issue a psci call to |
1199 | + * harden the branch predictor. |
1200 | + */ |
1201 | +static const struct midr_range arm64_bp_harden_smccc_cpus[] = { |
1202 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), |
1203 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), |
1204 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), |
1205 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), |
1206 | + MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN), |
1207 | + MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2), |
1208 | + {}, |
1209 | +}; |
1210 | + |
1211 | +#endif |
1212 | |
1213 | const struct arm64_cpu_capabilities arm64_errata[] = { |
1214 | #if defined(CONFIG_ARM64_ERRATUM_826319) || \ |
1215 | @@ -390,8 +456,8 @@ const struct arm64_cpu_capabilities arm64_errata[] = { |
1216 | /* Cortex-A53 r0p[012] */ |
1217 | .desc = "ARM errata 826319, 827319, 824069", |
1218 | .capability = ARM64_WORKAROUND_CLEAN_CACHE, |
1219 | - MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x02), |
1220 | - .enable = cpu_enable_cache_maint_trap, |
1221 | + ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2), |
1222 | + .cpu_enable = cpu_enable_cache_maint_trap, |
1223 | }, |
1224 | #endif |
1225 | #ifdef CONFIG_ARM64_ERRATUM_819472 |
1226 | @@ -399,8 +465,8 @@ const struct arm64_cpu_capabilities arm64_errata[] = { |
1227 | /* Cortex-A53 r0p[01] */ |
1228 | .desc = "ARM errata 819472", |
1229 | .capability = ARM64_WORKAROUND_CLEAN_CACHE, |
1230 | - MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x01), |
1231 | - .enable = cpu_enable_cache_maint_trap, |
1232 | + ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1), |
1233 | + .cpu_enable = cpu_enable_cache_maint_trap, |
1234 | }, |
1235 | #endif |
1236 | #ifdef CONFIG_ARM64_ERRATUM_832075 |
1237 | @@ -408,8 +474,9 @@ const struct arm64_cpu_capabilities arm64_errata[] = { |
1238 | /* Cortex-A57 r0p0 - r1p2 */ |
1239 | .desc = "ARM erratum 832075", |
1240 | .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE, |
1241 | - MIDR_RANGE(MIDR_CORTEX_A57, 0x00, |
1242 | - (1 << MIDR_VARIANT_SHIFT) | 2), |
1243 | + ERRATA_MIDR_RANGE(MIDR_CORTEX_A57, |
1244 | + 0, 0, |
1245 | + 1, 2), |
1246 | }, |
1247 | #endif |
1248 | #ifdef CONFIG_ARM64_ERRATUM_834220 |
1249 | @@ -417,8 +484,9 @@ const struct arm64_cpu_capabilities arm64_errata[] = { |
1250 | /* Cortex-A57 r0p0 - r1p2 */ |
1251 | .desc = "ARM erratum 834220", |
1252 | .capability = ARM64_WORKAROUND_834220, |
1253 | - MIDR_RANGE(MIDR_CORTEX_A57, 0x00, |
1254 | - (1 << MIDR_VARIANT_SHIFT) | 2), |
1255 | + ERRATA_MIDR_RANGE(MIDR_CORTEX_A57, |
1256 | + 0, 0, |
1257 | + 1, 2), |
1258 | }, |
1259 | #endif |
1260 | #ifdef CONFIG_ARM64_ERRATUM_845719 |
1261 | @@ -426,7 +494,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { |
1262 | /* Cortex-A53 r0p[01234] */ |
1263 | .desc = "ARM erratum 845719", |
1264 | .capability = ARM64_WORKAROUND_845719, |
1265 | - MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04), |
1266 | + ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4), |
1267 | }, |
1268 | #endif |
1269 | #ifdef CONFIG_CAVIUM_ERRATUM_23154 |
1270 | @@ -434,7 +502,7 @@ const struct arm64_cpu_capabilities arm64_errata[] = { |
1271 | /* Cavium ThunderX, pass 1.x */ |
1272 | .desc = "Cavium erratum 23154", |
1273 | .capability = ARM64_WORKAROUND_CAVIUM_23154, |
1274 | - MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01), |
1275 | + ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1), |
1276 | }, |
1277 | #endif |
1278 | #ifdef CONFIG_CAVIUM_ERRATUM_27456 |
1279 | @@ -442,103 +510,402 @@ const struct arm64_cpu_capabilities arm64_errata[] = { |
1280 | /* Cavium ThunderX, T88 pass 1.x - 2.1 */ |
1281 | .desc = "Cavium erratum 27456", |
1282 | .capability = ARM64_WORKAROUND_CAVIUM_27456, |
1283 | - MIDR_RANGE(MIDR_THUNDERX, 0x00, |
1284 | - (1 << MIDR_VARIANT_SHIFT) | 1), |
1285 | + ERRATA_MIDR_RANGE(MIDR_THUNDERX, |
1286 | + 0, 0, |
1287 | + 1, 1), |
1288 | }, |
1289 | { |
1290 | /* Cavium ThunderX, T81 pass 1.0 */ |
1291 | .desc = "Cavium erratum 27456", |
1292 | .capability = ARM64_WORKAROUND_CAVIUM_27456, |
1293 | - MIDR_RANGE(MIDR_THUNDERX_81XX, 0x00, 0x00), |
1294 | + ERRATA_MIDR_REV(MIDR_THUNDERX_81XX, 0, 0), |
1295 | }, |
1296 | #endif |
1297 | { |
1298 | .desc = "Mismatched cache line size", |
1299 | .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE, |
1300 | .matches = has_mismatched_cache_type, |
1301 | - .def_scope = SCOPE_LOCAL_CPU, |
1302 | - .enable = cpu_enable_trap_ctr_access, |
1303 | + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, |
1304 | + .cpu_enable = cpu_enable_trap_ctr_access, |
1305 | }, |
1306 | { |
1307 | .desc = "Mismatched cache type", |
1308 | .capability = ARM64_MISMATCHED_CACHE_TYPE, |
1309 | .matches = has_mismatched_cache_type, |
1310 | - .def_scope = SCOPE_LOCAL_CPU, |
1311 | - .enable = cpu_enable_trap_ctr_access, |
1312 | + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, |
1313 | + .cpu_enable = cpu_enable_trap_ctr_access, |
1314 | }, |
1315 | #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
1316 | { |
1317 | .capability = ARM64_HARDEN_BRANCH_PREDICTOR, |
1318 | - MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), |
1319 | - .enable = enable_smccc_arch_workaround_1, |
1320 | - }, |
1321 | - { |
1322 | - .capability = ARM64_HARDEN_BRANCH_PREDICTOR, |
1323 | - MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), |
1324 | - .enable = enable_smccc_arch_workaround_1, |
1325 | - }, |
1326 | - { |
1327 | - .capability = ARM64_HARDEN_BRANCH_PREDICTOR, |
1328 | - MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), |
1329 | - .enable = enable_smccc_arch_workaround_1, |
1330 | - }, |
1331 | - { |
1332 | - .capability = ARM64_HARDEN_BRANCH_PREDICTOR, |
1333 | - MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), |
1334 | - .enable = enable_smccc_arch_workaround_1, |
1335 | - }, |
1336 | - { |
1337 | - .capability = ARM64_HARDEN_BRANCH_PREDICTOR, |
1338 | - MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN), |
1339 | - .enable = enable_smccc_arch_workaround_1, |
1340 | - }, |
1341 | - { |
1342 | - .capability = ARM64_HARDEN_BRANCH_PREDICTOR, |
1343 | - MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2), |
1344 | - .enable = enable_smccc_arch_workaround_1, |
1345 | + ERRATA_MIDR_RANGE_LIST(arm64_bp_harden_smccc_cpus), |
1346 | + .cpu_enable = enable_smccc_arch_workaround_1, |
1347 | }, |
1348 | #endif |
1349 | #ifdef CONFIG_ARM64_SSBD |
1350 | { |
1351 | .desc = "Speculative Store Bypass Disable", |
1352 | - .def_scope = SCOPE_LOCAL_CPU, |
1353 | + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, |
1354 | .capability = ARM64_SSBD, |
1355 | .matches = has_ssbd_mitigation, |
1356 | }, |
1357 | #endif |
1358 | +#ifdef CONFIG_ARM64_ERRATUM_1188873 |
1359 | + { |
1360 | + /* Cortex-A76 r0p0 to r2p0 */ |
1361 | + .desc = "ARM erratum 1188873", |
1362 | + .capability = ARM64_WORKAROUND_1188873, |
1363 | + ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0), |
1364 | + }, |
1365 | +#endif |
1366 | + { |
1367 | + .desc = "Spectre-BHB", |
1368 | + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, |
1369 | + .capability = ARM64_SPECTRE_BHB, |
1370 | + .matches = is_spectre_bhb_affected, |
1371 | +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
1372 | + .cpu_enable = spectre_bhb_enable_mitigation, |
1373 | +#endif |
1374 | + }, |
1375 | { |
1376 | } |
1377 | }; |
1378 | |
1379 | /* |
1380 | - * The CPU Errata work arounds are detected and applied at boot time |
1381 | - * and the related information is freed soon after. If the new CPU requires |
1382 | - * an errata not detected at boot, fail this CPU. |
1383 | + * We try to ensure that the mitigation state can never change as the result of |
1384 | + * onlining a late CPU. |
1385 | */ |
1386 | -void verify_local_cpu_errata_workarounds(void) |
1387 | +static void __maybe_unused update_mitigation_state(enum mitigation_state *oldp, |
1388 | + enum mitigation_state new) |
1389 | { |
1390 | - const struct arm64_cpu_capabilities *caps = arm64_errata; |
1391 | - |
1392 | - for (; caps->matches; caps++) { |
1393 | - if (cpus_have_cap(caps->capability)) { |
1394 | - if (caps->enable) |
1395 | - caps->enable((void *)caps); |
1396 | - } else if (caps->matches(caps, SCOPE_LOCAL_CPU)) { |
1397 | - pr_crit("CPU%d: Requires work around for %s, not detected" |
1398 | - " at boot time\n", |
1399 | - smp_processor_id(), |
1400 | - caps->desc ? : "an erratum"); |
1401 | - cpu_die_early(); |
1402 | + enum mitigation_state state; |
1403 | + |
1404 | + do { |
1405 | + state = READ_ONCE(*oldp); |
1406 | + if (new <= state) |
1407 | + break; |
1408 | + } while (cmpxchg_relaxed(oldp, state, new) != state); |
1409 | +} |
1410 | + |
1411 | +/* |
1412 | + * Spectre BHB. |
1413 | + * |
1414 | + * A CPU is either: |
1415 | + * - Mitigated by a branchy loop a CPU specific number of times, and listed |
1416 | + * in our "loop mitigated list". |
1417 | + * - Mitigated in software by the firmware Spectre v2 call. |
1418 | + * - Has the ClearBHB instruction to perform the mitigation. |
1419 | + * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no |
1420 | + * software mitigation in the vectors is needed. |
1421 | + * - Has CSV2.3, so is unaffected. |
1422 | + */ |
1423 | +static enum mitigation_state spectre_bhb_state; |
1424 | + |
1425 | +enum mitigation_state arm64_get_spectre_bhb_state(void) |
1426 | +{ |
1427 | + return spectre_bhb_state; |
1428 | +} |
1429 | + |
1430 | +/* |
1431 | + * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any |
1432 | + * SCOPE_SYSTEM call will give the right answer. |
1433 | + */ |
1434 | +u8 spectre_bhb_loop_affected(int scope) |
1435 | +{ |
1436 | + u8 k = 0; |
1437 | + static u8 max_bhb_k; |
1438 | + |
1439 | + if (scope == SCOPE_LOCAL_CPU) { |
1440 | + static const struct midr_range spectre_bhb_k32_list[] = { |
1441 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), |
1442 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), |
1443 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), |
1444 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), |
1445 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), |
1446 | + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), |
1447 | + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), |
1448 | + {}, |
1449 | + }; |
1450 | + static const struct midr_range spectre_bhb_k24_list[] = { |
1451 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), |
1452 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), |
1453 | + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), |
1454 | + {}, |
1455 | + }; |
1456 | + static const struct midr_range spectre_bhb_k8_list[] = { |
1457 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), |
1458 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), |
1459 | + {}, |
1460 | + }; |
1461 | + |
1462 | + if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list)) |
1463 | + k = 32; |
1464 | + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list)) |
1465 | + k = 24; |
1466 | + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list)) |
1467 | + k = 8; |
1468 | + |
1469 | + max_bhb_k = max(max_bhb_k, k); |
1470 | + } else { |
1471 | + k = max_bhb_k; |
1472 | + } |
1473 | + |
1474 | + return k; |
1475 | +} |
1476 | + |
1477 | +static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void) |
1478 | +{ |
1479 | + int ret; |
1480 | + struct arm_smccc_res res; |
1481 | + |
1482 | + if (psci_ops.smccc_version == SMCCC_VERSION_1_0) |
1483 | + return SPECTRE_VULNERABLE; |
1484 | + |
1485 | + switch (psci_ops.conduit) { |
1486 | + case PSCI_CONDUIT_HVC: |
1487 | + arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
1488 | + ARM_SMCCC_ARCH_WORKAROUND_3, &res); |
1489 | + break; |
1490 | + |
1491 | + case PSCI_CONDUIT_SMC: |
1492 | + arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
1493 | + ARM_SMCCC_ARCH_WORKAROUND_3, &res); |
1494 | + break; |
1495 | + |
1496 | + default: |
1497 | + return SPECTRE_VULNERABLE; |
1498 | + } |
1499 | + |
1500 | + ret = res.a0; |
1501 | + switch (ret) { |
1502 | + case SMCCC_RET_SUCCESS: |
1503 | + return SPECTRE_MITIGATED; |
1504 | + case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: |
1505 | + return SPECTRE_UNAFFECTED; |
1506 | + default: |
1507 | + case SMCCC_RET_NOT_SUPPORTED: |
1508 | + return SPECTRE_VULNERABLE; |
1509 | + } |
1510 | +} |
1511 | + |
1512 | +static bool is_spectre_bhb_fw_affected(int scope) |
1513 | +{ |
1514 | + static bool system_affected; |
1515 | + enum mitigation_state fw_state; |
1516 | + bool has_smccc = (psci_ops.smccc_version >= SMCCC_VERSION_1_1); |
1517 | + static const struct midr_range spectre_bhb_firmware_mitigated_list[] = { |
1518 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), |
1519 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), |
1520 | + {}, |
1521 | + }; |
1522 | + bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(), |
1523 | + spectre_bhb_firmware_mitigated_list); |
1524 | + |
1525 | + if (scope != SCOPE_LOCAL_CPU) |
1526 | + return system_affected; |
1527 | + |
1528 | + fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); |
1529 | + if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) { |
1530 | + system_affected = true; |
1531 | + return true; |
1532 | + } |
1533 | + |
1534 | + return false; |
1535 | +} |
1536 | + |
1537 | +static bool __maybe_unused supports_ecbhb(int scope) |
1538 | +{ |
1539 | + u64 mmfr1; |
1540 | + |
1541 | + if (scope == SCOPE_LOCAL_CPU) |
1542 | + mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1); |
1543 | + else |
1544 | + mmfr1 = read_system_reg(SYS_ID_AA64MMFR1_EL1); |
1545 | + |
1546 | + return cpuid_feature_extract_unsigned_field(mmfr1, |
1547 | + ID_AA64MMFR1_ECBHB_SHIFT); |
1548 | +} |
1549 | + |
1550 | +bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, |
1551 | + int scope) |
1552 | +{ |
1553 | + WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); |
1554 | + |
1555 | + if (supports_csv2p3(scope)) |
1556 | + return false; |
1557 | + |
1558 | + if (supports_clearbhb(scope)) |
1559 | + return true; |
1560 | + |
1561 | + if (spectre_bhb_loop_affected(scope)) |
1562 | + return true; |
1563 | + |
1564 | + if (is_spectre_bhb_fw_affected(scope)) |
1565 | + return true; |
1566 | + |
1567 | + return false; |
1568 | +} |
1569 | + |
1570 | +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
1571 | +static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) |
1572 | +{ |
1573 | + const char *v = arm64_get_bp_hardening_vector(slot); |
1574 | + |
1575 | + if (slot < 0) |
1576 | + return; |
1577 | + |
1578 | + __this_cpu_write(this_cpu_vector, v); |
1579 | + |
1580 | + /* |
1581 | + * When KPTI is in use, the vectors are switched when exiting to |
1582 | + * user-space. |
1583 | + */ |
1584 | + if (arm64_kernel_unmapped_at_el0()) |
1585 | + return; |
1586 | + |
1587 | + write_sysreg(v, vbar_el1); |
1588 | + isb(); |
1589 | +} |
1590 | + |
1591 | +#ifdef CONFIG_KVM |
1592 | +static const char *kvm_bhb_get_vecs_end(const char *start) |
1593 | +{ |
1594 | + if (start == __smccc_workaround_3_smc_start) |
1595 | + return __smccc_workaround_3_smc_end; |
1596 | + else if (start == __spectre_bhb_loop_k8_start) |
1597 | + return __spectre_bhb_loop_k8_end; |
1598 | + else if (start == __spectre_bhb_loop_k24_start) |
1599 | + return __spectre_bhb_loop_k24_end; |
1600 | + else if (start == __spectre_bhb_loop_k32_start) |
1601 | + return __spectre_bhb_loop_k32_end; |
1602 | + else if (start == __spectre_bhb_clearbhb_start) |
1603 | + return __spectre_bhb_clearbhb_end; |
1604 | + |
1605 | + return NULL; |
1606 | +} |
1607 | + |
1608 | +static void kvm_setup_bhb_slot(const char *hyp_vecs_start) |
1609 | +{ |
1610 | + int cpu, slot = -1; |
1611 | + const char *hyp_vecs_end; |
1612 | + |
1613 | + if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available()) |
1614 | + return; |
1615 | + |
1616 | + hyp_vecs_end = kvm_bhb_get_vecs_end(hyp_vecs_start); |
1617 | + if (WARN_ON_ONCE(!hyp_vecs_start || !hyp_vecs_end)) |
1618 | + return; |
1619 | + |
1620 | + spin_lock(&bp_lock); |
1621 | + for_each_possible_cpu(cpu) { |
1622 | + if (per_cpu(bp_hardening_data.template_start, cpu) == hyp_vecs_start) { |
1623 | + slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu); |
1624 | + break; |
1625 | } |
1626 | } |
1627 | + |
1628 | + if (slot == -1) { |
1629 | + last_slot++; |
1630 | + BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start) |
1631 | + / SZ_2K) <= last_slot); |
1632 | + slot = last_slot; |
1633 | + __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end); |
1634 | + } |
1635 | + |
1636 | + __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot); |
1637 | + __this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start); |
1638 | + spin_unlock(&bp_lock); |
1639 | } |
1640 | +#else |
1641 | +#define __smccc_workaround_3_smc_start NULL |
1642 | +#define __spectre_bhb_loop_k8_start NULL |
1643 | +#define __spectre_bhb_loop_k24_start NULL |
1644 | +#define __spectre_bhb_loop_k32_start NULL |
1645 | +#define __spectre_bhb_clearbhb_start NULL |
1646 | + |
1647 | +static void kvm_setup_bhb_slot(const char *hyp_vecs_start) { }; |
1648 | +#endif /* CONFIG_KVM */ |
1649 | |
1650 | -void update_cpu_errata_workarounds(void) |
1651 | +static bool is_spectrev2_safe(void) |
1652 | { |
1653 | - update_cpu_capabilities(arm64_errata, "enabling workaround for"); |
1654 | + return !is_midr_in_range_list(read_cpuid_id(), |
1655 | + arm64_bp_harden_smccc_cpus); |
1656 | } |
1657 | |
1658 | -void __init enable_errata_workarounds(void) |
1659 | +void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) |
1660 | { |
1661 | - enable_cpu_capabilities(arm64_errata); |
1662 | + enum mitigation_state fw_state, state = SPECTRE_VULNERABLE; |
1663 | + |
1664 | + if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU)) |
1665 | + return; |
1666 | + |
1667 | + if (!is_spectrev2_safe() && !__hardenbp_enab) { |
1668 | + /* No point mitigating Spectre-BHB alone. */ |
1669 | + } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { |
1670 | + pr_info_once("spectre-bhb mitigation disabled by compile time option\n"); |
1671 | + } else if (cpu_mitigations_off()) { |
1672 | + pr_info_once("spectre-bhb mitigation disabled by command line option\n"); |
1673 | + } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { |
1674 | + state = SPECTRE_MITIGATED; |
1675 | + } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) { |
1676 | + kvm_setup_bhb_slot(__spectre_bhb_clearbhb_start); |
1677 | + this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN); |
1678 | + |
1679 | + state = SPECTRE_MITIGATED; |
1680 | + } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) { |
1681 | + switch (spectre_bhb_loop_affected(SCOPE_SYSTEM)) { |
1682 | + case 8: |
1683 | + kvm_setup_bhb_slot(__spectre_bhb_loop_k8_start); |
1684 | + break; |
1685 | + case 24: |
1686 | + kvm_setup_bhb_slot(__spectre_bhb_loop_k24_start); |
1687 | + break; |
1688 | + case 32: |
1689 | + kvm_setup_bhb_slot(__spectre_bhb_loop_k32_start); |
1690 | + break; |
1691 | + default: |
1692 | + WARN_ON_ONCE(1); |
1693 | + } |
1694 | + this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP); |
1695 | + |
1696 | + state = SPECTRE_MITIGATED; |
1697 | + } else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) { |
1698 | + fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); |
1699 | + if (fw_state == SPECTRE_MITIGATED) { |
1700 | + kvm_setup_bhb_slot(__smccc_workaround_3_smc_start); |
1701 | + this_cpu_set_vectors(EL1_VECTOR_BHB_FW); |
1702 | + |
1703 | + /* |
1704 | + * With WA3 in the vectors, the WA1 calls can be |
1705 | + * removed. |
1706 | + */ |
1707 | + __this_cpu_write(bp_hardening_data.fn, NULL); |
1708 | + |
1709 | + state = SPECTRE_MITIGATED; |
1710 | + } |
1711 | + } |
1712 | + |
1713 | + update_mitigation_state(&spectre_bhb_state, state); |
1714 | +} |
1715 | + |
1716 | +/* Patched to correct the immediate */ |
1717 | +void __init spectre_bhb_patch_loop_iter(struct alt_instr *alt, |
1718 | + __le32 *origptr, __le32 *updptr, int nr_inst) |
1719 | +{ |
1720 | + u8 rd; |
1721 | + u32 insn; |
1722 | + u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM); |
1723 | + |
1724 | + BUG_ON(nr_inst != 1); /* MOV -> MOV */ |
1725 | + |
1726 | + if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) |
1727 | + return; |
1728 | + |
1729 | + insn = le32_to_cpu(*origptr); |
1730 | + rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); |
1731 | + insn = aarch64_insn_gen_movewide(rd, loop_count, 0, |
1732 | + AARCH64_INSN_VARIANT_64BIT, |
1733 | + AARCH64_INSN_MOVEWIDE_ZERO); |
1734 | + *updptr++ = cpu_to_le32(insn); |
1735 | } |
1736 | +#endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */ |
1737 | diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c |
1738 | index 8cf001baee219..9b7e7d2f236ee 100644 |
1739 | --- a/arch/arm64/kernel/cpufeature.c |
1740 | +++ b/arch/arm64/kernel/cpufeature.c |
1741 | @@ -20,15 +20,18 @@ |
1742 | |
1743 | #include <linux/bsearch.h> |
1744 | #include <linux/cpumask.h> |
1745 | +#include <linux/percpu.h> |
1746 | #include <linux/sort.h> |
1747 | #include <linux/stop_machine.h> |
1748 | #include <linux/types.h> |
1749 | + |
1750 | #include <asm/cpu.h> |
1751 | #include <asm/cpufeature.h> |
1752 | #include <asm/cpu_ops.h> |
1753 | #include <asm/mmu_context.h> |
1754 | #include <asm/processor.h> |
1755 | #include <asm/sysreg.h> |
1756 | +#include <asm/vectors.h> |
1757 | #include <asm/virt.h> |
1758 | |
1759 | unsigned long elf_hwcap __read_mostly; |
1760 | @@ -49,6 +52,8 @@ unsigned int compat_elf_hwcap2 __read_mostly; |
1761 | DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); |
1762 | EXPORT_SYMBOL(cpu_hwcaps); |
1763 | |
1764 | +DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors; |
1765 | + |
1766 | DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS); |
1767 | EXPORT_SYMBOL(cpu_hwcap_keys); |
1768 | |
1769 | @@ -93,6 +98,11 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = { |
1770 | ARM64_FTR_END, |
1771 | }; |
1772 | |
1773 | +static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { |
1774 | + ARM64_FTR_BITS(FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0), |
1775 | + ARM64_FTR_END, |
1776 | +}; |
1777 | + |
1778 | static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { |
1779 | ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0), |
1780 | ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0), |
1781 | @@ -327,6 +337,7 @@ static const struct __ftr_reg_entry { |
1782 | /* Op1 = 0, CRn = 0, CRm = 6 */ |
1783 | ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0), |
1784 | ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_aa64raz), |
1785 | + ARM64_FTR_REG(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2), |
1786 | |
1787 | /* Op1 = 0, CRn = 0, CRm = 7 */ |
1788 | ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0), |
1789 | @@ -439,6 +450,9 @@ static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new) |
1790 | reg->strict_mask = strict_mask; |
1791 | } |
1792 | |
1793 | +extern const struct arm64_cpu_capabilities arm64_errata[]; |
1794 | +static void update_cpu_errata_workarounds(void); |
1795 | + |
1796 | void __init init_cpu_features(struct cpuinfo_arm64 *info) |
1797 | { |
1798 | /* Before we start using the tables, make sure it is sorted */ |
1799 | @@ -451,6 +465,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) |
1800 | init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1); |
1801 | init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0); |
1802 | init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1); |
1803 | + init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2); |
1804 | init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0); |
1805 | init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1); |
1806 | init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2); |
1807 | @@ -476,6 +491,11 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) |
1808 | init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2); |
1809 | } |
1810 | |
1811 | + /* |
1812 | + * Run the errata work around checks on the boot CPU, once we have |
1813 | + * initialised the cpu feature infrastructure. |
1814 | + */ |
1815 | + update_cpu_errata_workarounds(); |
1816 | } |
1817 | |
1818 | static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new) |
1819 | @@ -557,6 +577,8 @@ void update_cpu_features(int cpu, |
1820 | info->reg_id_aa64isar0, boot->reg_id_aa64isar0); |
1821 | taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu, |
1822 | info->reg_id_aa64isar1, boot->reg_id_aa64isar1); |
1823 | + taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu, |
1824 | + info->reg_id_aa64isar2, boot->reg_id_aa64isar2); |
1825 | |
1826 | /* |
1827 | * Differing PARange support is fine as long as all peripherals and |
1828 | @@ -676,6 +698,7 @@ static u64 __raw_read_system_reg(u32 sys_id) |
1829 | case SYS_ID_AA64MMFR2_EL1: return read_cpuid(ID_AA64MMFR2_EL1); |
1830 | case SYS_ID_AA64ISAR0_EL1: return read_cpuid(ID_AA64ISAR0_EL1); |
1831 | case SYS_ID_AA64ISAR1_EL1: return read_cpuid(ID_AA64ISAR1_EL1); |
1832 | + case SYS_ID_AA64ISAR2_EL1: return read_cpuid(ID_AA64ISAR2_EL1); |
1833 | |
1834 | case SYS_CNTFRQ_EL0: return read_cpuid(CNTFRQ_EL0); |
1835 | case SYS_CTR_EL0: return read_cpuid(CTR_EL0); |
1836 | @@ -728,13 +751,11 @@ static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, |
1837 | static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused) |
1838 | { |
1839 | u32 midr = read_cpuid_id(); |
1840 | - u32 rv_min, rv_max; |
1841 | |
1842 | /* Cavium ThunderX pass 1.x and 2.x */ |
1843 | - rv_min = 0; |
1844 | - rv_max = (1 << MIDR_VARIANT_SHIFT) | MIDR_REVISION_MASK; |
1845 | - |
1846 | - return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX, rv_min, rv_max); |
1847 | + return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX, |
1848 | + MIDR_CPU_VAR_REV(0, 0), |
1849 | + MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK)); |
1850 | } |
1851 | |
1852 | static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused) |
1853 | @@ -761,6 +782,17 @@ static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */ |
1854 | static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry, |
1855 | int __unused) |
1856 | { |
1857 | + /* List of CPUs that are not vulnerable and don't need KPTI */ |
1858 | + static const struct midr_range kpti_safe_list[] = { |
1859 | + MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2), |
1860 | + MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN), |
1861 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A35), |
1862 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A53), |
1863 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), |
1864 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), |
1865 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), |
1866 | + MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), |
1867 | + }; |
1868 | char const *str = "command line option"; |
1869 | u64 pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1); |
1870 | |
1871 | @@ -786,23 +818,16 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry, |
1872 | return true; |
1873 | |
1874 | /* Don't force KPTI for CPUs that are not vulnerable */ |
1875 | - switch (read_cpuid_id() & MIDR_CPU_MODEL_MASK) { |
1876 | - case MIDR_CAVIUM_THUNDERX2: |
1877 | - case MIDR_BRCM_VULCAN: |
1878 | - case MIDR_CORTEX_A53: |
1879 | - case MIDR_CORTEX_A55: |
1880 | - case MIDR_CORTEX_A57: |
1881 | - case MIDR_CORTEX_A72: |
1882 | - case MIDR_CORTEX_A73: |
1883 | + if (is_midr_in_range_list(read_cpuid_id(), kpti_safe_list)) |
1884 | return false; |
1885 | - } |
1886 | |
1887 | /* Defer to CPU feature registers */ |
1888 | return !cpuid_feature_extract_unsigned_field(pfr0, |
1889 | ID_AA64PFR0_CSV3_SHIFT); |
1890 | } |
1891 | |
1892 | -static int kpti_install_ng_mappings(void *__unused) |
1893 | +static void |
1894 | +kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) |
1895 | { |
1896 | typedef void (kpti_remap_fn)(int, int, phys_addr_t); |
1897 | extern kpti_remap_fn idmap_kpti_install_ng_mappings; |
1898 | @@ -811,8 +836,14 @@ static int kpti_install_ng_mappings(void *__unused) |
1899 | static bool kpti_applied = false; |
1900 | int cpu = smp_processor_id(); |
1901 | |
1902 | + if (__this_cpu_read(this_cpu_vector) == vectors) { |
1903 | + const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI); |
1904 | + |
1905 | + __this_cpu_write(this_cpu_vector, v); |
1906 | + } |
1907 | + |
1908 | if (kpti_applied) |
1909 | - return 0; |
1910 | + return; |
1911 | |
1912 | remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); |
1913 | |
1914 | @@ -823,7 +854,7 @@ static int kpti_install_ng_mappings(void *__unused) |
1915 | if (!cpu) |
1916 | kpti_applied = true; |
1917 | |
1918 | - return 0; |
1919 | + return; |
1920 | } |
1921 | |
1922 | static int __init parse_kpti(char *str) |
1923 | @@ -840,7 +871,7 @@ static int __init parse_kpti(char *str) |
1924 | early_param("kpti", parse_kpti); |
1925 | #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ |
1926 | |
1927 | -static int cpu_copy_el2regs(void *__unused) |
1928 | +static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused) |
1929 | { |
1930 | /* |
1931 | * Copy register values that aren't redirected by hardware. |
1932 | @@ -852,15 +883,13 @@ static int cpu_copy_el2regs(void *__unused) |
1933 | */ |
1934 | if (!alternatives_applied) |
1935 | write_sysreg(read_sysreg(tpidr_el1), tpidr_el2); |
1936 | - |
1937 | - return 0; |
1938 | } |
1939 | |
1940 | static const struct arm64_cpu_capabilities arm64_features[] = { |
1941 | { |
1942 | .desc = "GIC system register CPU interface", |
1943 | .capability = ARM64_HAS_SYSREG_GIC_CPUIF, |
1944 | - .def_scope = SCOPE_SYSTEM, |
1945 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
1946 | .matches = has_useable_gicv3_cpuif, |
1947 | .sys_reg = SYS_ID_AA64PFR0_EL1, |
1948 | .field_pos = ID_AA64PFR0_GIC_SHIFT, |
1949 | @@ -871,20 +900,20 @@ static const struct arm64_cpu_capabilities arm64_features[] = { |
1950 | { |
1951 | .desc = "Privileged Access Never", |
1952 | .capability = ARM64_HAS_PAN, |
1953 | - .def_scope = SCOPE_SYSTEM, |
1954 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
1955 | .matches = has_cpuid_feature, |
1956 | .sys_reg = SYS_ID_AA64MMFR1_EL1, |
1957 | .field_pos = ID_AA64MMFR1_PAN_SHIFT, |
1958 | .sign = FTR_UNSIGNED, |
1959 | .min_field_value = 1, |
1960 | - .enable = cpu_enable_pan, |
1961 | + .cpu_enable = cpu_enable_pan, |
1962 | }, |
1963 | #endif /* CONFIG_ARM64_PAN */ |
1964 | #if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS) |
1965 | { |
1966 | .desc = "LSE atomic instructions", |
1967 | .capability = ARM64_HAS_LSE_ATOMICS, |
1968 | - .def_scope = SCOPE_SYSTEM, |
1969 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
1970 | .matches = has_cpuid_feature, |
1971 | .sys_reg = SYS_ID_AA64ISAR0_EL1, |
1972 | .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT, |
1973 | @@ -895,39 +924,42 @@ static const struct arm64_cpu_capabilities arm64_features[] = { |
1974 | { |
1975 | .desc = "Software prefetching using PRFM", |
1976 | .capability = ARM64_HAS_NO_HW_PREFETCH, |
1977 | - .def_scope = SCOPE_SYSTEM, |
1978 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
1979 | .matches = has_no_hw_prefetch, |
1980 | }, |
1981 | #ifdef CONFIG_ARM64_UAO |
1982 | { |
1983 | .desc = "User Access Override", |
1984 | .capability = ARM64_HAS_UAO, |
1985 | - .def_scope = SCOPE_SYSTEM, |
1986 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
1987 | .matches = has_cpuid_feature, |
1988 | .sys_reg = SYS_ID_AA64MMFR2_EL1, |
1989 | .field_pos = ID_AA64MMFR2_UAO_SHIFT, |
1990 | .min_field_value = 1, |
1991 | - .enable = cpu_enable_uao, |
1992 | + /* |
1993 | + * We rely on stop_machine() calling uao_thread_switch() to set |
1994 | + * UAO immediately after patching. |
1995 | + */ |
1996 | }, |
1997 | #endif /* CONFIG_ARM64_UAO */ |
1998 | #ifdef CONFIG_ARM64_PAN |
1999 | { |
2000 | .capability = ARM64_ALT_PAN_NOT_UAO, |
2001 | - .def_scope = SCOPE_SYSTEM, |
2002 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2003 | .matches = cpufeature_pan_not_uao, |
2004 | }, |
2005 | #endif /* CONFIG_ARM64_PAN */ |
2006 | { |
2007 | .desc = "Virtualization Host Extensions", |
2008 | .capability = ARM64_HAS_VIRT_HOST_EXTN, |
2009 | - .def_scope = SCOPE_SYSTEM, |
2010 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2011 | .matches = runs_at_el2, |
2012 | - .enable = cpu_copy_el2regs, |
2013 | + .cpu_enable = cpu_copy_el2regs, |
2014 | }, |
2015 | { |
2016 | .desc = "32-bit EL0 Support", |
2017 | .capability = ARM64_HAS_32BIT_EL0, |
2018 | - .def_scope = SCOPE_SYSTEM, |
2019 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2020 | .matches = has_cpuid_feature, |
2021 | .sys_reg = SYS_ID_AA64PFR0_EL1, |
2022 | .sign = FTR_UNSIGNED, |
2023 | @@ -937,31 +969,31 @@ static const struct arm64_cpu_capabilities arm64_features[] = { |
2024 | { |
2025 | .desc = "Reduced HYP mapping offset", |
2026 | .capability = ARM64_HYP_OFFSET_LOW, |
2027 | - .def_scope = SCOPE_SYSTEM, |
2028 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2029 | .matches = hyp_offset_low, |
2030 | }, |
2031 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
2032 | { |
2033 | .desc = "Kernel page table isolation (KPTI)", |
2034 | .capability = ARM64_UNMAP_KERNEL_AT_EL0, |
2035 | - .def_scope = SCOPE_SYSTEM, |
2036 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
2037 | .matches = unmap_kernel_at_el0, |
2038 | - .enable = kpti_install_ng_mappings, |
2039 | + .cpu_enable = kpti_install_ng_mappings, |
2040 | }, |
2041 | #endif |
2042 | {}, |
2043 | }; |
2044 | |
2045 | -#define HWCAP_CAP(reg, field, s, min_value, type, cap) \ |
2046 | +#define HWCAP_CAP(reg, field, s, min_value, cap_type, cap) \ |
2047 | { \ |
2048 | .desc = #cap, \ |
2049 | - .def_scope = SCOPE_SYSTEM, \ |
2050 | + .type = ARM64_CPUCAP_SYSTEM_FEATURE, \ |
2051 | .matches = has_cpuid_feature, \ |
2052 | .sys_reg = reg, \ |
2053 | .field_pos = field, \ |
2054 | .sign = s, \ |
2055 | .min_field_value = min_value, \ |
2056 | - .hwcap_type = type, \ |
2057 | + .hwcap_type = cap_type, \ |
2058 | .hwcap = cap, \ |
2059 | } |
2060 | |
2061 | @@ -1038,7 +1070,7 @@ static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap) |
2062 | static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps) |
2063 | { |
2064 | for (; hwcaps->matches; hwcaps++) |
2065 | - if (hwcaps->matches(hwcaps, hwcaps->def_scope)) |
2066 | + if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps))) |
2067 | cap_set_elf_hwcap(hwcaps); |
2068 | } |
2069 | |
2070 | @@ -1061,11 +1093,11 @@ static bool __this_cpu_has_cap(const struct arm64_cpu_capabilities *cap_array, |
2071 | return false; |
2072 | } |
2073 | |
2074 | -void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, |
2075 | - const char *info) |
2076 | +static void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, |
2077 | + const char *info) |
2078 | { |
2079 | for (; caps->matches; caps++) { |
2080 | - if (!caps->matches(caps, caps->def_scope)) |
2081 | + if (!caps->matches(caps, cpucap_default_scope(caps))) |
2082 | continue; |
2083 | |
2084 | if (!cpus_have_cap(caps->capability) && caps->desc) |
2085 | @@ -1074,11 +1106,20 @@ void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, |
2086 | } |
2087 | } |
2088 | |
2089 | +static int __enable_cpu_capability(void *arg) |
2090 | +{ |
2091 | + const struct arm64_cpu_capabilities *cap = arg; |
2092 | + |
2093 | + cap->cpu_enable(cap); |
2094 | + return 0; |
2095 | +} |
2096 | + |
2097 | /* |
2098 | * Run through the enabled capabilities and enable() it on all active |
2099 | * CPUs |
2100 | */ |
2101 | -void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) |
2102 | +static void __init |
2103 | +enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) |
2104 | { |
2105 | for (; caps->matches; caps++) { |
2106 | unsigned int num = caps->capability; |
2107 | @@ -1089,14 +1130,15 @@ void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) |
2108 | /* Ensure cpus_have_const_cap(num) works */ |
2109 | static_branch_enable(&cpu_hwcap_keys[num]); |
2110 | |
2111 | - if (caps->enable) { |
2112 | + if (caps->cpu_enable) { |
2113 | /* |
2114 | * Use stop_machine() as it schedules the work allowing |
2115 | * us to modify PSTATE, instead of on_each_cpu() which |
2116 | * uses an IPI, giving us a PSTATE that disappears when |
2117 | * we return. |
2118 | */ |
2119 | - stop_machine(caps->enable, (void *)caps, cpu_online_mask); |
2120 | + stop_machine(__enable_cpu_capability, (void *)caps, |
2121 | + cpu_online_mask); |
2122 | } |
2123 | } |
2124 | } |
2125 | @@ -1154,11 +1196,44 @@ verify_local_cpu_features(const struct arm64_cpu_capabilities *caps_list) |
2126 | smp_processor_id(), caps->desc); |
2127 | cpu_die_early(); |
2128 | } |
2129 | - if (caps->enable) |
2130 | - caps->enable((void *)caps); |
2131 | + if (caps->cpu_enable) |
2132 | + caps->cpu_enable(caps); |
2133 | } |
2134 | } |
2135 | |
2136 | +/* |
2137 | + * The CPU Errata work arounds are detected and applied at boot time |
2138 | + * and the related information is freed soon after. If the new CPU requires |
2139 | + * an errata not detected at boot, fail this CPU. |
2140 | + */ |
2141 | +static void verify_local_cpu_errata_workarounds(void) |
2142 | +{ |
2143 | + const struct arm64_cpu_capabilities *caps = arm64_errata; |
2144 | + |
2145 | + for (; caps->matches; caps++) { |
2146 | + if (cpus_have_cap(caps->capability)) { |
2147 | + if (caps->cpu_enable) |
2148 | + caps->cpu_enable(caps); |
2149 | + } else if (caps->matches(caps, SCOPE_LOCAL_CPU)) { |
2150 | + pr_crit("CPU%d: Requires work around for %s, not detected" |
2151 | + " at boot time\n", |
2152 | + smp_processor_id(), |
2153 | + caps->desc ? : "an erratum"); |
2154 | + cpu_die_early(); |
2155 | + } |
2156 | + } |
2157 | +} |
2158 | + |
2159 | +static void update_cpu_errata_workarounds(void) |
2160 | +{ |
2161 | + update_cpu_capabilities(arm64_errata, "enabling workaround for"); |
2162 | +} |
2163 | + |
2164 | +static void __init enable_errata_workarounds(void) |
2165 | +{ |
2166 | + enable_cpu_capabilities(arm64_errata); |
2167 | +} |
2168 | + |
2169 | /* |
2170 | * Run through the enabled system capabilities and enable() it on this CPU. |
2171 | * The capabilities were decided based on the available CPUs at the boot time. |
2172 | diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c |
2173 | index b3d5b3e8fbcb1..4c09f87650f4b 100644 |
2174 | --- a/arch/arm64/kernel/cpuinfo.c |
2175 | +++ b/arch/arm64/kernel/cpuinfo.c |
2176 | @@ -335,6 +335,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) |
2177 | info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1); |
2178 | info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1); |
2179 | info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1); |
2180 | + info->reg_id_aa64isar2 = read_cpuid(ID_AA64ISAR2_EL1); |
2181 | info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); |
2182 | info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1); |
2183 | info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1); |
2184 | diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S |
2185 | index ca978d7d98eb4..1f79abb1e5ddb 100644 |
2186 | --- a/arch/arm64/kernel/entry.S |
2187 | +++ b/arch/arm64/kernel/entry.S |
2188 | @@ -74,26 +74,33 @@ |
2189 | |
2190 | .macro kernel_ventry, el, label, regsize = 64 |
2191 | .align 7 |
2192 | -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
2193 | -alternative_if ARM64_UNMAP_KERNEL_AT_EL0 |
2194 | +.Lventry_start\@: |
2195 | .if \el == 0 |
2196 | + /* |
2197 | + * This must be the first instruction of the EL0 vector entries. It is |
2198 | + * skipped by the trampoline vectors, to trigger the cleanup. |
2199 | + */ |
2200 | + b .Lskip_tramp_vectors_cleanup\@ |
2201 | .if \regsize == 64 |
2202 | mrs x30, tpidrro_el0 |
2203 | msr tpidrro_el0, xzr |
2204 | .else |
2205 | mov x30, xzr |
2206 | .endif |
2207 | +.Lskip_tramp_vectors_cleanup\@: |
2208 | .endif |
2209 | -alternative_else_nop_endif |
2210 | -#endif |
2211 | |
2212 | sub sp, sp, #S_FRAME_SIZE |
2213 | b el\()\el\()_\label |
2214 | +.org .Lventry_start\@ + 128 // Did we overflow the ventry slot? |
2215 | .endm |
2216 | |
2217 | - .macro tramp_alias, dst, sym |
2218 | + .macro tramp_alias, dst, sym, tmp |
2219 | mov_q \dst, TRAMP_VALIAS |
2220 | - add \dst, \dst, #(\sym - .entry.tramp.text) |
2221 | + adr_l \tmp, \sym |
2222 | + add \dst, \dst, \tmp |
2223 | + adr_l \tmp, .entry.tramp.text |
2224 | + sub \dst, \dst, \tmp |
2225 | .endm |
2226 | |
2227 | // This macro corrupts x0-x3. It is the caller's duty |
2228 | @@ -238,21 +245,25 @@ alternative_else_nop_endif |
2229 | ldp x24, x25, [sp, #16 * 12] |
2230 | ldp x26, x27, [sp, #16 * 13] |
2231 | ldp x28, x29, [sp, #16 * 14] |
2232 | - ldr lr, [sp, #S_LR] |
2233 | - add sp, sp, #S_FRAME_SIZE // restore sp |
2234 | |
2235 | .if \el == 0 |
2236 | -alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 |
2237 | +alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 |
2238 | + ldr lr, [sp, #S_LR] |
2239 | + add sp, sp, #S_FRAME_SIZE // restore sp |
2240 | + eret |
2241 | +alternative_else_nop_endif |
2242 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
2243 | bne 4f |
2244 | - msr far_el1, x30 |
2245 | - tramp_alias x30, tramp_exit_native |
2246 | + msr far_el1, x29 |
2247 | + tramp_alias x30, tramp_exit_native, x29 |
2248 | br x30 |
2249 | 4: |
2250 | - tramp_alias x30, tramp_exit_compat |
2251 | + tramp_alias x30, tramp_exit_compat, x29 |
2252 | br x30 |
2253 | #endif |
2254 | .else |
2255 | + ldr lr, [sp, #S_LR] |
2256 | + add sp, sp, #S_FRAME_SIZE // restore sp |
2257 | eret |
2258 | .endif |
2259 | .endm |
2260 | @@ -883,12 +894,7 @@ __ni_sys_trace: |
2261 | |
2262 | .popsection // .entry.text |
2263 | |
2264 | -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
2265 | -/* |
2266 | - * Exception vectors trampoline. |
2267 | - */ |
2268 | - .pushsection ".entry.tramp.text", "ax" |
2269 | - |
2270 | + // Move from tramp_pg_dir to swapper_pg_dir |
2271 | .macro tramp_map_kernel, tmp |
2272 | mrs \tmp, ttbr1_el1 |
2273 | sub \tmp, \tmp, #SWAPPER_DIR_SIZE |
2274 | @@ -908,12 +914,47 @@ __ni_sys_trace: |
2275 | */ |
2276 | .endm |
2277 | |
2278 | - .macro tramp_ventry, regsize = 64 |
2279 | + .macro tramp_data_page dst |
2280 | + adr_l \dst, .entry.tramp.text |
2281 | + sub \dst, \dst, PAGE_SIZE |
2282 | + .endm |
2283 | + |
2284 | + .macro tramp_data_read_var dst, var |
2285 | +#ifdef CONFIG_RANDOMIZE_BASE |
2286 | + tramp_data_page \dst |
2287 | + add \dst, \dst, #:lo12:__entry_tramp_data_\var |
2288 | + ldr \dst, [\dst] |
2289 | +#else |
2290 | + ldr \dst, =\var |
2291 | +#endif |
2292 | + .endm |
2293 | + |
2294 | +#define BHB_MITIGATION_NONE 0 |
2295 | +#define BHB_MITIGATION_LOOP 1 |
2296 | +#define BHB_MITIGATION_FW 2 |
2297 | +#define BHB_MITIGATION_INSN 3 |
2298 | + |
2299 | + .macro tramp_ventry, vector_start, regsize, kpti, bhb |
2300 | .align 7 |
2301 | 1: |
2302 | .if \regsize == 64 |
2303 | msr tpidrro_el0, x30 // Restored in kernel_ventry |
2304 | .endif |
2305 | + |
2306 | + .if \bhb == BHB_MITIGATION_LOOP |
2307 | + /* |
2308 | + * This sequence must appear before the first indirect branch. i.e. the |
2309 | + * ret out of tramp_ventry. It appears here because x30 is free. |
2310 | + */ |
2311 | + __mitigate_spectre_bhb_loop x30 |
2312 | + .endif // \bhb == BHB_MITIGATION_LOOP |
2313 | + |
2314 | + .if \bhb == BHB_MITIGATION_INSN |
2315 | + clearbhb |
2316 | + isb |
2317 | + .endif // \bhb == BHB_MITIGATION_INSN |
2318 | + |
2319 | + .if \kpti == 1 |
2320 | /* |
2321 | * Defend against branch aliasing attacks by pushing a dummy |
2322 | * entry onto the return stack and using a RET instruction to |
2323 | @@ -923,43 +964,74 @@ __ni_sys_trace: |
2324 | b . |
2325 | 2: |
2326 | tramp_map_kernel x30 |
2327 | -#ifdef CONFIG_RANDOMIZE_BASE |
2328 | - adr x30, tramp_vectors + PAGE_SIZE |
2329 | - isb |
2330 | - ldr x30, [x30] |
2331 | -#else |
2332 | - ldr x30, =vectors |
2333 | -#endif |
2334 | - prfm plil1strm, [x30, #(1b - tramp_vectors)] |
2335 | + tramp_data_read_var x30, vectors |
2336 | + prfm plil1strm, [x30, #(1b - \vector_start)] |
2337 | msr vbar_el1, x30 |
2338 | - add x30, x30, #(1b - tramp_vectors) |
2339 | isb |
2340 | + .else |
2341 | + ldr x30, =vectors |
2342 | + .endif // \kpti == 1 |
2343 | + |
2344 | + .if \bhb == BHB_MITIGATION_FW |
2345 | + /* |
2346 | + * The firmware sequence must appear before the first indirect branch. |
2347 | + * i.e. the ret out of tramp_ventry. But it also needs the stack to be |
2348 | + * mapped to save/restore the registers the SMC clobbers. |
2349 | + */ |
2350 | + __mitigate_spectre_bhb_fw |
2351 | + .endif // \bhb == BHB_MITIGATION_FW |
2352 | + |
2353 | + add x30, x30, #(1b - \vector_start + 4) |
2354 | ret |
2355 | +.org 1b + 128 // Did we overflow the ventry slot? |
2356 | .endm |
2357 | |
2358 | .macro tramp_exit, regsize = 64 |
2359 | - adr x30, tramp_vectors |
2360 | + tramp_data_read_var x30, this_cpu_vector |
2361 | +alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
2362 | + mrs x29, tpidr_el1 |
2363 | +alternative_else |
2364 | + mrs x29, tpidr_el2 |
2365 | +alternative_endif |
2366 | + ldr x30, [x30, x29] |
2367 | + |
2368 | msr vbar_el1, x30 |
2369 | - tramp_unmap_kernel x30 |
2370 | + ldr lr, [sp, #S_LR] |
2371 | + tramp_unmap_kernel x29 |
2372 | .if \regsize == 64 |
2373 | - mrs x30, far_el1 |
2374 | + mrs x29, far_el1 |
2375 | .endif |
2376 | + add sp, sp, #S_FRAME_SIZE // restore sp |
2377 | eret |
2378 | .endm |
2379 | |
2380 | - .align 11 |
2381 | -ENTRY(tramp_vectors) |
2382 | + .macro generate_tramp_vector, kpti, bhb |
2383 | +.Lvector_start\@: |
2384 | .space 0x400 |
2385 | |
2386 | - tramp_ventry |
2387 | - tramp_ventry |
2388 | - tramp_ventry |
2389 | - tramp_ventry |
2390 | + .rept 4 |
2391 | + tramp_ventry .Lvector_start\@, 64, \kpti, \bhb |
2392 | + .endr |
2393 | + .rept 4 |
2394 | + tramp_ventry .Lvector_start\@, 32, \kpti, \bhb |
2395 | + .endr |
2396 | + .endm |
2397 | |
2398 | - tramp_ventry 32 |
2399 | - tramp_ventry 32 |
2400 | - tramp_ventry 32 |
2401 | - tramp_ventry 32 |
2402 | +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
2403 | +/* |
2404 | + * Exception vectors trampoline. |
2405 | + * The order must match __bp_harden_el1_vectors and the |
2406 | + * arm64_bp_harden_el1_vectors enum. |
2407 | + */ |
2408 | + .pushsection ".entry.tramp.text", "ax" |
2409 | + .align 11 |
2410 | +ENTRY(tramp_vectors) |
2411 | +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
2412 | + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP |
2413 | + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW |
2414 | + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN |
2415 | +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
2416 | + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE |
2417 | END(tramp_vectors) |
2418 | |
2419 | ENTRY(tramp_exit_native) |
2420 | @@ -977,11 +1049,54 @@ END(tramp_exit_compat) |
2421 | .align PAGE_SHIFT |
2422 | .globl __entry_tramp_data_start |
2423 | __entry_tramp_data_start: |
2424 | +__entry_tramp_data_vectors: |
2425 | .quad vectors |
2426 | +#ifdef CONFIG_ARM_SDE_INTERFACE |
2427 | +__entry_tramp_data___sdei_asm_trampoline_next_handler: |
2428 | + .quad __sdei_asm_handler |
2429 | +#endif /* CONFIG_ARM_SDE_INTERFACE */ |
2430 | +__entry_tramp_data_this_cpu_vector: |
2431 | + .quad this_cpu_vector |
2432 | .popsection // .rodata |
2433 | #endif /* CONFIG_RANDOMIZE_BASE */ |
2434 | #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ |
2435 | |
2436 | +/* |
2437 | + * Exception vectors for spectre mitigations on entry from EL1 when |
2438 | + * kpti is not in use. |
2439 | + */ |
2440 | + .macro generate_el1_vector, bhb |
2441 | +.Lvector_start\@: |
2442 | + kernel_ventry 1, sync_invalid // Synchronous EL1t |
2443 | + kernel_ventry 1, irq_invalid // IRQ EL1t |
2444 | + kernel_ventry 1, fiq_invalid // FIQ EL1t |
2445 | + kernel_ventry 1, error_invalid // Error EL1t |
2446 | + |
2447 | + kernel_ventry 1, sync // Synchronous EL1h |
2448 | + kernel_ventry 1, irq // IRQ EL1h |
2449 | + kernel_ventry 1, fiq_invalid // FIQ EL1h |
2450 | + kernel_ventry 1, error_invalid // Error EL1h |
2451 | + |
2452 | + .rept 4 |
2453 | + tramp_ventry .Lvector_start\@, 64, 0, \bhb |
2454 | + .endr |
2455 | + .rept 4 |
2456 | + tramp_ventry .Lvector_start\@, 32, 0, \bhb |
2457 | + .endr |
2458 | + .endm |
2459 | + |
2460 | +/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */ |
2461 | + .pushsection ".entry.text", "ax" |
2462 | + .align 11 |
2463 | +ENTRY(__bp_harden_el1_vectors) |
2464 | +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
2465 | + generate_el1_vector bhb=BHB_MITIGATION_LOOP |
2466 | + generate_el1_vector bhb=BHB_MITIGATION_FW |
2467 | + generate_el1_vector bhb=BHB_MITIGATION_INSN |
2468 | +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ |
2469 | +END(__bp_harden_el1_vectors) |
2470 | + .popsection |
2471 | + |
2472 | /* |
2473 | * Special system call wrappers. |
2474 | */ |
2475 | diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c |
2476 | index 1d5890f19ca32..ee34be8bed03a 100644 |
2477 | --- a/arch/arm64/kernel/fpsimd.c |
2478 | +++ b/arch/arm64/kernel/fpsimd.c |
2479 | @@ -26,6 +26,7 @@ |
2480 | #include <linux/hardirq.h> |
2481 | |
2482 | #include <asm/fpsimd.h> |
2483 | +#include <asm/cpufeature.h> |
2484 | #include <asm/cputype.h> |
2485 | |
2486 | #define FPEXC_IOF (1 << 0) |
2487 | diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c |
2488 | index 6f2ac4fc66ca2..755b3dd3073e5 100644 |
2489 | --- a/arch/arm64/kernel/insn.c |
2490 | +++ b/arch/arm64/kernel/insn.c |
2491 | @@ -418,6 +418,35 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, |
2492 | return insn; |
2493 | } |
2494 | |
2495 | +u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type, |
2496 | + u32 insn) |
2497 | +{ |
2498 | + int shift; |
2499 | + |
2500 | + switch (type) { |
2501 | + case AARCH64_INSN_REGTYPE_RT: |
2502 | + case AARCH64_INSN_REGTYPE_RD: |
2503 | + shift = 0; |
2504 | + break; |
2505 | + case AARCH64_INSN_REGTYPE_RN: |
2506 | + shift = 5; |
2507 | + break; |
2508 | + case AARCH64_INSN_REGTYPE_RT2: |
2509 | + case AARCH64_INSN_REGTYPE_RA: |
2510 | + shift = 10; |
2511 | + break; |
2512 | + case AARCH64_INSN_REGTYPE_RM: |
2513 | + shift = 16; |
2514 | + break; |
2515 | + default: |
2516 | + pr_err("%s: unknown register type encoding %d\n", __func__, |
2517 | + type); |
2518 | + return 0; |
2519 | + } |
2520 | + |
2521 | + return (insn >> shift) & GENMASK(4, 0); |
2522 | +} |
2523 | + |
2524 | static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type, |
2525 | u32 insn, |
2526 | enum aarch64_insn_register reg) |
2527 | diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c |
2528 | index 13b9c20a84b5c..ea4aedb6bbdca 100644 |
2529 | --- a/arch/arm64/kernel/smp.c |
2530 | +++ b/arch/arm64/kernel/smp.c |
2531 | @@ -444,12 +444,6 @@ void __init smp_prepare_boot_cpu(void) |
2532 | jump_label_init(); |
2533 | cpuinfo_store_boot_cpu(); |
2534 | save_boot_cpu_run_el(); |
2535 | - /* |
2536 | - * Run the errata work around checks on the boot CPU, once we have |
2537 | - * initialised the cpu feature infrastructure from |
2538 | - * cpuinfo_store_boot_cpu() above. |
2539 | - */ |
2540 | - update_cpu_errata_workarounds(); |
2541 | } |
2542 | |
2543 | static u64 __init of_get_cpu_mpidr(struct device_node *dn) |
2544 | diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c |
2545 | index b6fd2a21b015e..adf18b9a2c033 100644 |
2546 | --- a/arch/arm64/kernel/traps.c |
2547 | +++ b/arch/arm64/kernel/traps.c |
2548 | @@ -34,6 +34,7 @@ |
2549 | |
2550 | #include <asm/atomic.h> |
2551 | #include <asm/bug.h> |
2552 | +#include <asm/cpufeature.h> |
2553 | #include <asm/debug-monitors.h> |
2554 | #include <asm/esr.h> |
2555 | #include <asm/insn.h> |
2556 | @@ -432,10 +433,9 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs) |
2557 | force_signal_inject(SIGILL, ILL_ILLOPC, regs, 0); |
2558 | } |
2559 | |
2560 | -int cpu_enable_cache_maint_trap(void *__unused) |
2561 | +void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused) |
2562 | { |
2563 | config_sctlr_el1(SCTLR_EL1_UCI, 0); |
2564 | - return 0; |
2565 | } |
2566 | |
2567 | #define __user_cache_maint(insn, address, res) \ |
2568 | diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S |
2569 | index fa3ffad50a61c..17fc1671b9900 100644 |
2570 | --- a/arch/arm64/kernel/vmlinux.lds.S |
2571 | +++ b/arch/arm64/kernel/vmlinux.lds.S |
2572 | @@ -261,7 +261,7 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1)) |
2573 | <= SZ_4K, "Hibernate exit text too big or misaligned") |
2574 | #endif |
2575 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
2576 | -ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE, |
2577 | +ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE, |
2578 | "Entry trampoline text too big") |
2579 | #endif |
2580 | /* |
2581 | diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S |
2582 | index 7ced1fb93d077..d14a8c89d08e0 100644 |
2583 | --- a/arch/arm64/kvm/hyp/hyp-entry.S |
2584 | +++ b/arch/arm64/kvm/hyp/hyp-entry.S |
2585 | @@ -136,6 +136,10 @@ el1_hvc_guest: |
2586 | /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */ |
2587 | eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \ |
2588 | ARM_SMCCC_ARCH_WORKAROUND_2) |
2589 | + cbz w1, wa_epilogue |
2590 | + |
2591 | + eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \ |
2592 | + ARM_SMCCC_ARCH_WORKAROUND_3) |
2593 | cbnz w1, el1_trap |
2594 | |
2595 | #ifdef CONFIG_ARM64_SSBD |
2596 | diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c |
2597 | index 0a2f37bceab0a..1751d2763cc12 100644 |
2598 | --- a/arch/arm64/kvm/hyp/switch.c |
2599 | +++ b/arch/arm64/kvm/hyp/switch.c |
2600 | @@ -26,7 +26,7 @@ |
2601 | #include <asm/kvm_emulate.h> |
2602 | #include <asm/kvm_hyp.h> |
2603 | #include <asm/uaccess.h> |
2604 | - |
2605 | +#include <asm/vectors.h> |
2606 | extern struct exception_table_entry __start___kvm_ex_table; |
2607 | extern struct exception_table_entry __stop___kvm_ex_table; |
2608 | |
2609 | @@ -107,11 +107,14 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu) |
2610 | |
2611 | static void __hyp_text __deactivate_traps_vhe(void) |
2612 | { |
2613 | - extern char vectors[]; /* kernel exception vectors */ |
2614 | + const char *host_vectors = vectors; |
2615 | |
2616 | write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2); |
2617 | write_sysreg(CPACR_EL1_FPEN, cpacr_el1); |
2618 | - write_sysreg(vectors, vbar_el1); |
2619 | + |
2620 | + if (!arm64_kernel_unmapped_at_el0()) |
2621 | + host_vectors = __this_cpu_read(this_cpu_vector); |
2622 | + write_sysreg(host_vectors, vbar_el1); |
2623 | } |
2624 | |
2625 | static void __hyp_text __deactivate_traps_nvhe(void) |
2626 | diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c |
2627 | index f3d3f2e97adde..a0c3efbc3717e 100644 |
2628 | --- a/arch/arm64/mm/fault.c |
2629 | +++ b/arch/arm64/mm/fault.c |
2630 | @@ -727,7 +727,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr_if_watchpoint, |
2631 | NOKPROBE_SYMBOL(do_debug_exception); |
2632 | |
2633 | #ifdef CONFIG_ARM64_PAN |
2634 | -int cpu_enable_pan(void *__unused) |
2635 | +void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) |
2636 | { |
2637 | /* |
2638 | * We modify PSTATE. This won't work from irq context as the PSTATE |
2639 | @@ -737,20 +737,5 @@ int cpu_enable_pan(void *__unused) |
2640 | |
2641 | config_sctlr_el1(SCTLR_EL1_SPAN, 0); |
2642 | asm(SET_PSTATE_PAN(1)); |
2643 | - return 0; |
2644 | } |
2645 | #endif /* CONFIG_ARM64_PAN */ |
2646 | - |
2647 | -#ifdef CONFIG_ARM64_UAO |
2648 | -/* |
2649 | - * Kernel threads have fs=KERNEL_DS by default, and don't need to call |
2650 | - * set_fs(), devtmpfs in particular relies on this behaviour. |
2651 | - * We need to enable the feature at runtime (instead of adding it to |
2652 | - * PSR_MODE_EL1h) as the feature may not be implemented by the cpu. |
2653 | - */ |
2654 | -int cpu_enable_uao(void *__unused) |
2655 | -{ |
2656 | - asm(SET_PSTATE_UAO(1)); |
2657 | - return 0; |
2658 | -} |
2659 | -#endif /* CONFIG_ARM64_UAO */ |
2660 | diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c |
2661 | index 60be5bc0984ad..36bd50091c4bb 100644 |
2662 | --- a/arch/arm64/mm/mmu.c |
2663 | +++ b/arch/arm64/mm/mmu.c |
2664 | @@ -438,6 +438,7 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, |
2665 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
2666 | static int __init map_entry_trampoline(void) |
2667 | { |
2668 | + int i; |
2669 | extern char __entry_tramp_text_start[]; |
2670 | |
2671 | pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; |
2672 | @@ -448,11 +449,15 @@ static int __init map_entry_trampoline(void) |
2673 | |
2674 | /* Map only the text into the trampoline page table */ |
2675 | memset(tramp_pg_dir, 0, PGD_SIZE); |
2676 | - __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE, |
2677 | - prot, pgd_pgtable_alloc, 0); |
2678 | + __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, |
2679 | + entry_tramp_text_size(), prot, pgd_pgtable_alloc, |
2680 | + 0); |
2681 | |
2682 | /* Map both the text and data into the kernel page table */ |
2683 | - __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot); |
2684 | + for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) |
2685 | + __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, |
2686 | + pa_start + i * PAGE_SIZE, prot); |
2687 | + |
2688 | if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { |
2689 | extern char __entry_tramp_data_start[]; |
2690 | |
2691 | diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig |
2692 | index e2c6e43cf8ca3..3d748eac1a688 100644 |
2693 | --- a/drivers/clocksource/Kconfig |
2694 | +++ b/drivers/clocksource/Kconfig |
2695 | @@ -305,10 +305,14 @@ config ARM_ARCH_TIMER_EVTSTREAM |
2696 | This must be disabled for hardware validation purposes to detect any |
2697 | hardware anomalies of missing events. |
2698 | |
2699 | +config ARM_ARCH_TIMER_OOL_WORKAROUND |
2700 | + bool |
2701 | + |
2702 | config FSL_ERRATUM_A008585 |
2703 | bool "Workaround for Freescale/NXP Erratum A-008585" |
2704 | default y |
2705 | depends on ARM_ARCH_TIMER && ARM64 |
2706 | + select ARM_ARCH_TIMER_OOL_WORKAROUND |
2707 | help |
2708 | This option enables a workaround for Freescale/NXP Erratum |
2709 | A-008585 ("ARM generic timer may contain an erroneous |
2710 | diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c |
2711 | index 5d7f83d27093f..e70d0974470c7 100644 |
2712 | --- a/drivers/clocksource/arm_arch_timer.c |
2713 | +++ b/drivers/clocksource/arm_arch_timer.c |
2714 | @@ -96,41 +96,159 @@ early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); |
2715 | */ |
2716 | |
2717 | #ifdef CONFIG_FSL_ERRATUM_A008585 |
2718 | +/* |
2719 | + * The number of retries is an arbitrary value well beyond the highest number |
2720 | + * of iterations the loop has been observed to take. |
2721 | + */ |
2722 | +#define __fsl_a008585_read_reg(reg) ({ \ |
2723 | + u64 _old, _new; \ |
2724 | + int _retries = 200; \ |
2725 | + \ |
2726 | + do { \ |
2727 | + _old = read_sysreg(reg); \ |
2728 | + _new = read_sysreg(reg); \ |
2729 | + _retries--; \ |
2730 | + } while (unlikely(_old != _new) && _retries); \ |
2731 | + \ |
2732 | + WARN_ON_ONCE(!_retries); \ |
2733 | + _new; \ |
2734 | +}) |
2735 | + |
2736 | +static u32 notrace fsl_a008585_read_cntp_tval_el0(void) |
2737 | +{ |
2738 | + return __fsl_a008585_read_reg(cntp_tval_el0); |
2739 | +} |
2740 | + |
2741 | +static u32 notrace fsl_a008585_read_cntv_tval_el0(void) |
2742 | +{ |
2743 | + return __fsl_a008585_read_reg(cntv_tval_el0); |
2744 | +} |
2745 | + |
2746 | +static u64 notrace fsl_a008585_read_cntvct_el0(void) |
2747 | +{ |
2748 | + return __fsl_a008585_read_reg(cntvct_el0); |
2749 | +} |
2750 | +#endif |
2751 | + |
2752 | +#ifdef CONFIG_ARM64_ERRATUM_1188873 |
2753 | +static u64 notrace arm64_1188873_read_cntvct_el0(void) |
2754 | +{ |
2755 | + return read_sysreg(cntvct_el0); |
2756 | +} |
2757 | +#endif |
2758 | + |
2759 | +#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND |
2760 | +const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround = NULL; |
2761 | +EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); |
2762 | + |
2763 | DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); |
2764 | EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); |
2765 | |
2766 | -static int fsl_a008585_enable = -1; |
2767 | +static const struct arch_timer_erratum_workaround ool_workarounds[] = { |
2768 | +#ifdef CONFIG_FSL_ERRATUM_A008585 |
2769 | + { |
2770 | + .match_type = ate_match_dt, |
2771 | + .id = "fsl,erratum-a008585", |
2772 | + .desc = "Freescale erratum a005858", |
2773 | + .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0, |
2774 | + .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0, |
2775 | + .read_cntvct_el0 = fsl_a008585_read_cntvct_el0, |
2776 | + }, |
2777 | +#endif |
2778 | +#ifdef CONFIG_ARM64_ERRATUM_1188873 |
2779 | + { |
2780 | + .match_type = ate_match_local_cap_id, |
2781 | + .id = (void *)ARM64_WORKAROUND_1188873, |
2782 | + .desc = "ARM erratum 1188873", |
2783 | + .read_cntvct_el0 = arm64_1188873_read_cntvct_el0, |
2784 | + }, |
2785 | +#endif |
2786 | +}; |
2787 | |
2788 | -static int __init early_fsl_a008585_cfg(char *buf) |
2789 | +typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *, |
2790 | + const void *); |
2791 | + |
2792 | +static |
2793 | +bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa, |
2794 | + const void *arg) |
2795 | { |
2796 | - int ret; |
2797 | - bool val; |
2798 | + const struct device_node *np = arg; |
2799 | |
2800 | - ret = strtobool(buf, &val); |
2801 | - if (ret) |
2802 | - return ret; |
2803 | + return of_property_read_bool(np, wa->id); |
2804 | +} |
2805 | |
2806 | - fsl_a008585_enable = val; |
2807 | - return 0; |
2808 | +static |
2809 | +bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa, |
2810 | + const void *arg) |
2811 | +{ |
2812 | + return this_cpu_has_cap((uintptr_t)wa->id); |
2813 | } |
2814 | -early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg); |
2815 | |
2816 | -u32 __fsl_a008585_read_cntp_tval_el0(void) |
2817 | +static const struct arch_timer_erratum_workaround * |
2818 | +arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, |
2819 | + ate_match_fn_t match_fn, |
2820 | + void *arg) |
2821 | { |
2822 | - return __fsl_a008585_read_reg(cntp_tval_el0); |
2823 | + int i; |
2824 | + |
2825 | + for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) { |
2826 | + if (ool_workarounds[i].match_type != type) |
2827 | + continue; |
2828 | + |
2829 | + if (match_fn(&ool_workarounds[i], arg)) |
2830 | + return &ool_workarounds[i]; |
2831 | + } |
2832 | + |
2833 | + return NULL; |
2834 | } |
2835 | |
2836 | -u32 __fsl_a008585_read_cntv_tval_el0(void) |
2837 | +static |
2838 | +void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa) |
2839 | { |
2840 | - return __fsl_a008585_read_reg(cntv_tval_el0); |
2841 | + timer_unstable_counter_workaround = wa; |
2842 | + static_branch_enable(&arch_timer_read_ool_enabled); |
2843 | } |
2844 | |
2845 | -u64 __fsl_a008585_read_cntvct_el0(void) |
2846 | +static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type, |
2847 | + void *arg) |
2848 | { |
2849 | - return __fsl_a008585_read_reg(cntvct_el0); |
2850 | + const struct arch_timer_erratum_workaround *wa; |
2851 | + ate_match_fn_t match_fn = NULL; |
2852 | + bool local = false; |
2853 | + |
2854 | + switch (type) { |
2855 | + case ate_match_dt: |
2856 | + match_fn = arch_timer_check_dt_erratum; |
2857 | + break; |
2858 | + case ate_match_local_cap_id: |
2859 | + match_fn = arch_timer_check_local_cap_erratum; |
2860 | + local = true; |
2861 | + break; |
2862 | + default: |
2863 | + WARN_ON(1); |
2864 | + return; |
2865 | + } |
2866 | + |
2867 | + wa = arch_timer_iterate_errata(type, match_fn, arg); |
2868 | + if (!wa) |
2869 | + return; |
2870 | + |
2871 | + if (needs_unstable_timer_counter_workaround()) { |
2872 | + if (wa != timer_unstable_counter_workaround) |
2873 | + pr_warn("Can't enable workaround for %s (clashes with %s\n)", |
2874 | + wa->desc, |
2875 | + timer_unstable_counter_workaround->desc); |
2876 | + return; |
2877 | + } |
2878 | + |
2879 | + arch_timer_enable_workaround(wa); |
2880 | + pr_info("Enabling %s workaround for %s\n", |
2881 | + local ? "local" : "global", wa->desc); |
2882 | } |
2883 | -EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0); |
2884 | -#endif /* CONFIG_FSL_ERRATUM_A008585 */ |
2885 | + |
2886 | +#else |
2887 | +#define arch_timer_check_ool_workaround(t,a) do { } while(0) |
2888 | +#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ |
2889 | |
2890 | static __always_inline |
2891 | void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, |
2892 | @@ -281,8 +399,8 @@ static __always_inline void set_next_event(const int access, unsigned long evt, |
2893 | arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); |
2894 | } |
2895 | |
2896 | -#ifdef CONFIG_FSL_ERRATUM_A008585 |
2897 | -static __always_inline void fsl_a008585_set_next_event(const int access, |
2898 | +#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND |
2899 | +static __always_inline void erratum_set_next_event_generic(const int access, |
2900 | unsigned long evt, struct clock_event_device *clk) |
2901 | { |
2902 | unsigned long ctrl; |
2903 | @@ -300,20 +418,20 @@ static __always_inline void fsl_a008585_set_next_event(const int access, |
2904 | arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); |
2905 | } |
2906 | |
2907 | -static int fsl_a008585_set_next_event_virt(unsigned long evt, |
2908 | +static int erratum_set_next_event_virt(unsigned long evt, |
2909 | struct clock_event_device *clk) |
2910 | { |
2911 | - fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); |
2912 | + erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk); |
2913 | return 0; |
2914 | } |
2915 | |
2916 | -static int fsl_a008585_set_next_event_phys(unsigned long evt, |
2917 | +static int erratum_set_next_event_phys(unsigned long evt, |
2918 | struct clock_event_device *clk) |
2919 | { |
2920 | - fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); |
2921 | + erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk); |
2922 | return 0; |
2923 | } |
2924 | -#endif /* CONFIG_FSL_ERRATUM_A008585 */ |
2925 | +#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ |
2926 | |
2927 | static int arch_timer_set_next_event_virt(unsigned long evt, |
2928 | struct clock_event_device *clk) |
2929 | @@ -343,16 +461,16 @@ static int arch_timer_set_next_event_phys_mem(unsigned long evt, |
2930 | return 0; |
2931 | } |
2932 | |
2933 | -static void fsl_a008585_set_sne(struct clock_event_device *clk) |
2934 | +static void erratum_workaround_set_sne(struct clock_event_device *clk) |
2935 | { |
2936 | -#ifdef CONFIG_FSL_ERRATUM_A008585 |
2937 | +#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND |
2938 | if (!static_branch_unlikely(&arch_timer_read_ool_enabled)) |
2939 | return; |
2940 | |
2941 | if (arch_timer_uses_ppi == VIRT_PPI) |
2942 | - clk->set_next_event = fsl_a008585_set_next_event_virt; |
2943 | + clk->set_next_event = erratum_set_next_event_virt; |
2944 | else |
2945 | - clk->set_next_event = fsl_a008585_set_next_event_phys; |
2946 | + clk->set_next_event = erratum_set_next_event_phys; |
2947 | #endif |
2948 | } |
2949 | |
2950 | @@ -385,7 +503,9 @@ static void __arch_timer_setup(unsigned type, |
2951 | BUG(); |
2952 | } |
2953 | |
2954 | - fsl_a008585_set_sne(clk); |
2955 | + arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL); |
2956 | + |
2957 | + erratum_workaround_set_sne(clk); |
2958 | } else { |
2959 | clk->features |= CLOCK_EVT_FEAT_DYNIRQ; |
2960 | clk->name = "arch_mem_timer"; |
2961 | @@ -614,7 +734,7 @@ static void __init arch_counter_register(unsigned type) |
2962 | |
2963 | clocksource_counter.archdata.vdso_direct = true; |
2964 | |
2965 | -#ifdef CONFIG_FSL_ERRATUM_A008585 |
2966 | +#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND |
2967 | /* |
2968 | * Don't use the vdso fastpath if errata require using |
2969 | * the out-of-line counter accessor. |
2970 | @@ -902,14 +1022,8 @@ static int __init arch_timer_of_init(struct device_node *np) |
2971 | |
2972 | arch_timer_c3stop = !of_property_read_bool(np, "always-on"); |
2973 | |
2974 | -#ifdef CONFIG_FSL_ERRATUM_A008585 |
2975 | - if (fsl_a008585_enable < 0) |
2976 | - fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585"); |
2977 | - if (fsl_a008585_enable) { |
2978 | - static_branch_enable(&arch_timer_read_ool_enabled); |
2979 | - pr_info("Enabling workaround for FSL erratum A-008585\n"); |
2980 | - } |
2981 | -#endif |
2982 | + /* Check for globally applicable workarounds */ |
2983 | + arch_timer_check_ool_workaround(ate_match_dt, np); |
2984 | |
2985 | /* |
2986 | * If we cannot rely on firmware initializing the timer registers then |
2987 | diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h |
2988 | index 6366b04c7d5f4..0402668914147 100644 |
2989 | --- a/include/linux/arm-smccc.h |
2990 | +++ b/include/linux/arm-smccc.h |
2991 | @@ -85,6 +85,13 @@ |
2992 | ARM_SMCCC_SMC_32, \ |
2993 | 0, 0x7fff) |
2994 | |
2995 | +#define ARM_SMCCC_ARCH_WORKAROUND_3 \ |
2996 | + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ |
2997 | + ARM_SMCCC_SMC_32, \ |
2998 | + 0, 0x3fff) |
2999 | + |
3000 | +#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED 1 |
3001 | + |
3002 | #ifndef __ASSEMBLY__ |
3003 | |
3004 | #include <linux/linkage.h> |