Annotation of /trunk/kernel-alx/patches-5.4/0283-5.4.184-all-fixes.patch
Parent Directory | Revision Log
Revision 3637 -
(hide annotations)
(download)
Mon Oct 24 12:40:44 2022 UTC (18 months, 3 weeks ago) by niro
File size: 70457 byte(s)
Mon Oct 24 12:40:44 2022 UTC (18 months, 3 weeks ago) by niro
File size: 70457 byte(s)
-add missing
1 | niro | 3637 | diff --git a/Documentation/admin-guide/hw-vuln/spectre.rst b/Documentation/admin-guide/hw-vuln/spectre.rst |
2 | index 985181dba0bac..6bd97cd50d625 100644 | ||
3 | --- a/Documentation/admin-guide/hw-vuln/spectre.rst | ||
4 | +++ b/Documentation/admin-guide/hw-vuln/spectre.rst | ||
5 | @@ -60,8 +60,8 @@ privileged data touched during the speculative execution. | ||
6 | Spectre variant 1 attacks take advantage of speculative execution of | ||
7 | conditional branches, while Spectre variant 2 attacks use speculative | ||
8 | execution of indirect branches to leak privileged memory. | ||
9 | -See :ref:`[1] <spec_ref1>` :ref:`[5] <spec_ref5>` :ref:`[7] <spec_ref7>` | ||
10 | -:ref:`[10] <spec_ref10>` :ref:`[11] <spec_ref11>`. | ||
11 | +See :ref:`[1] <spec_ref1>` :ref:`[5] <spec_ref5>` :ref:`[6] <spec_ref6>` | ||
12 | +:ref:`[7] <spec_ref7>` :ref:`[10] <spec_ref10>` :ref:`[11] <spec_ref11>`. | ||
13 | |||
14 | Spectre variant 1 (Bounds Check Bypass) | ||
15 | --------------------------------------- | ||
16 | @@ -131,6 +131,19 @@ steer its indirect branch speculations to gadget code, and measure the | ||
17 | speculative execution's side effects left in level 1 cache to infer the | ||
18 | victim's data. | ||
19 | |||
20 | +Yet another variant 2 attack vector is for the attacker to poison the | ||
21 | +Branch History Buffer (BHB) to speculatively steer an indirect branch | ||
22 | +to a specific Branch Target Buffer (BTB) entry, even if the entry isn't | ||
23 | +associated with the source address of the indirect branch. Specifically, | ||
24 | +the BHB might be shared across privilege levels even in the presence of | ||
25 | +Enhanced IBRS. | ||
26 | + | ||
27 | +Currently the only known real-world BHB attack vector is via | ||
28 | +unprivileged eBPF. Therefore, it's highly recommended to not enable | ||
29 | +unprivileged eBPF, especially when eIBRS is used (without retpolines). | ||
30 | +For a full mitigation against BHB attacks, it's recommended to use | ||
31 | +retpolines (or eIBRS combined with retpolines). | ||
32 | + | ||
33 | Attack scenarios | ||
34 | ---------------- | ||
35 | |||
36 | @@ -364,13 +377,15 @@ The possible values in this file are: | ||
37 | |||
38 | - Kernel status: | ||
39 | |||
40 | - ==================================== ================================= | ||
41 | - 'Not affected' The processor is not vulnerable | ||
42 | - 'Vulnerable' Vulnerable, no mitigation | ||
43 | - 'Mitigation: Full generic retpoline' Software-focused mitigation | ||
44 | - 'Mitigation: Full AMD retpoline' AMD-specific software mitigation | ||
45 | - 'Mitigation: Enhanced IBRS' Hardware-focused mitigation | ||
46 | - ==================================== ================================= | ||
47 | + ======================================== ================================= | ||
48 | + 'Not affected' The processor is not vulnerable | ||
49 | + 'Mitigation: None' Vulnerable, no mitigation | ||
50 | + 'Mitigation: Retpolines' Use Retpoline thunks | ||
51 | + 'Mitigation: LFENCE' Use LFENCE instructions | ||
52 | + 'Mitigation: Enhanced IBRS' Hardware-focused mitigation | ||
53 | + 'Mitigation: Enhanced IBRS + Retpolines' Hardware-focused + Retpolines | ||
54 | + 'Mitigation: Enhanced IBRS + LFENCE' Hardware-focused + LFENCE | ||
55 | + ======================================== ================================= | ||
56 | |||
57 | - Firmware status: Show if Indirect Branch Restricted Speculation (IBRS) is | ||
58 | used to protect against Spectre variant 2 attacks when calling firmware (x86 only). | ||
59 | @@ -584,12 +599,13 @@ kernel command line. | ||
60 | |||
61 | Specific mitigations can also be selected manually: | ||
62 | |||
63 | - retpoline | ||
64 | - replace indirect branches | ||
65 | - retpoline,generic | ||
66 | - google's original retpoline | ||
67 | - retpoline,amd | ||
68 | - AMD-specific minimal thunk | ||
69 | + retpoline auto pick between generic,lfence | ||
70 | + retpoline,generic Retpolines | ||
71 | + retpoline,lfence LFENCE; indirect branch | ||
72 | + retpoline,amd alias for retpoline,lfence | ||
73 | + eibrs enhanced IBRS | ||
74 | + eibrs,retpoline enhanced IBRS + Retpolines | ||
75 | + eibrs,lfence enhanced IBRS + LFENCE | ||
76 | |||
77 | Not specifying this option is equivalent to | ||
78 | spectre_v2=auto. | ||
79 | @@ -730,7 +746,7 @@ AMD white papers: | ||
80 | |||
81 | .. _spec_ref6: | ||
82 | |||
83 | -[6] `Software techniques for managing speculation on AMD processors <https://developer.amd.com/wp-content/resources/90343-B_SoftwareTechniquesforManagingSpeculation_WP_7-18Update_FNL.pdf>`_. | ||
84 | +[6] `Software techniques for managing speculation on AMD processors <https://developer.amd.com/wp-content/resources/Managing-Speculation-on-AMD-Processors.pdf>`_. | ||
85 | |||
86 | ARM white papers: | ||
87 | |||
88 | diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt | ||
89 | index 165abcb656c5b..979423e1b639f 100644 | ||
90 | --- a/Documentation/admin-guide/kernel-parameters.txt | ||
91 | +++ b/Documentation/admin-guide/kernel-parameters.txt | ||
92 | @@ -4493,8 +4493,12 @@ | ||
93 | Specific mitigations can also be selected manually: | ||
94 | |||
95 | retpoline - replace indirect branches | ||
96 | - retpoline,generic - google's original retpoline | ||
97 | - retpoline,amd - AMD-specific minimal thunk | ||
98 | + retpoline,generic - Retpolines | ||
99 | + retpoline,lfence - LFENCE; indirect branch | ||
100 | + retpoline,amd - alias for retpoline,lfence | ||
101 | + eibrs - enhanced IBRS | ||
102 | + eibrs,retpoline - enhanced IBRS + Retpolines | ||
103 | + eibrs,lfence - enhanced IBRS + LFENCE | ||
104 | |||
105 | Not specifying this option is equivalent to | ||
106 | spectre_v2=auto. | ||
107 | diff --git a/Makefile b/Makefile | ||
108 | index a94b5ea499e13..e914e1a8a7d2c 100644 | ||
109 | --- a/Makefile | ||
110 | +++ b/Makefile | ||
111 | @@ -1,7 +1,7 @@ | ||
112 | # SPDX-License-Identifier: GPL-2.0 | ||
113 | VERSION = 5 | ||
114 | PATCHLEVEL = 4 | ||
115 | -SUBLEVEL = 183 | ||
116 | +SUBLEVEL = 184 | ||
117 | EXTRAVERSION = | ||
118 | NAME = Kleptomaniac Octopus | ||
119 | |||
120 | diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h | ||
121 | index 3546d294d55fa..6b3e64e19fb6f 100644 | ||
122 | --- a/arch/arm/include/asm/assembler.h | ||
123 | +++ b/arch/arm/include/asm/assembler.h | ||
124 | @@ -107,6 +107,16 @@ | ||
125 | .endm | ||
126 | #endif | ||
127 | |||
128 | +#if __LINUX_ARM_ARCH__ < 7 | ||
129 | + .macro dsb, args | ||
130 | + mcr p15, 0, r0, c7, c10, 4 | ||
131 | + .endm | ||
132 | + | ||
133 | + .macro isb, args | ||
134 | + mcr p15, 0, r0, c7, c5, 4 | ||
135 | + .endm | ||
136 | +#endif | ||
137 | + | ||
138 | .macro asm_trace_hardirqs_off, save=1 | ||
139 | #if defined(CONFIG_TRACE_IRQFLAGS) | ||
140 | .if \save | ||
141 | diff --git a/arch/arm/include/asm/spectre.h b/arch/arm/include/asm/spectre.h | ||
142 | new file mode 100644 | ||
143 | index 0000000000000..d1fa5607d3aa3 | ||
144 | --- /dev/null | ||
145 | +++ b/arch/arm/include/asm/spectre.h | ||
146 | @@ -0,0 +1,32 @@ | ||
147 | +/* SPDX-License-Identifier: GPL-2.0-only */ | ||
148 | + | ||
149 | +#ifndef __ASM_SPECTRE_H | ||
150 | +#define __ASM_SPECTRE_H | ||
151 | + | ||
152 | +enum { | ||
153 | + SPECTRE_UNAFFECTED, | ||
154 | + SPECTRE_MITIGATED, | ||
155 | + SPECTRE_VULNERABLE, | ||
156 | +}; | ||
157 | + | ||
158 | +enum { | ||
159 | + __SPECTRE_V2_METHOD_BPIALL, | ||
160 | + __SPECTRE_V2_METHOD_ICIALLU, | ||
161 | + __SPECTRE_V2_METHOD_SMC, | ||
162 | + __SPECTRE_V2_METHOD_HVC, | ||
163 | + __SPECTRE_V2_METHOD_LOOP8, | ||
164 | +}; | ||
165 | + | ||
166 | +enum { | ||
167 | + SPECTRE_V2_METHOD_BPIALL = BIT(__SPECTRE_V2_METHOD_BPIALL), | ||
168 | + SPECTRE_V2_METHOD_ICIALLU = BIT(__SPECTRE_V2_METHOD_ICIALLU), | ||
169 | + SPECTRE_V2_METHOD_SMC = BIT(__SPECTRE_V2_METHOD_SMC), | ||
170 | + SPECTRE_V2_METHOD_HVC = BIT(__SPECTRE_V2_METHOD_HVC), | ||
171 | + SPECTRE_V2_METHOD_LOOP8 = BIT(__SPECTRE_V2_METHOD_LOOP8), | ||
172 | +}; | ||
173 | + | ||
174 | +void spectre_v2_update_state(unsigned int state, unsigned int methods); | ||
175 | + | ||
176 | +int spectre_bhb_update_vectors(unsigned int method); | ||
177 | + | ||
178 | +#endif | ||
179 | diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile | ||
180 | index 8b679e2ca3c3d..dc31426cae6d8 100644 | ||
181 | --- a/arch/arm/kernel/Makefile | ||
182 | +++ b/arch/arm/kernel/Makefile | ||
183 | @@ -106,4 +106,6 @@ endif | ||
184 | |||
185 | obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o | ||
186 | |||
187 | +obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o | ||
188 | + | ||
189 | extra-y := $(head-y) vmlinux.lds | ||
190 | diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S | ||
191 | index 4937d514318ec..94d25425b7bce 100644 | ||
192 | --- a/arch/arm/kernel/entry-armv.S | ||
193 | +++ b/arch/arm/kernel/entry-armv.S | ||
194 | @@ -1005,12 +1005,11 @@ vector_\name: | ||
195 | sub lr, lr, #\correction | ||
196 | .endif | ||
197 | |||
198 | - @ | ||
199 | - @ Save r0, lr_<exception> (parent PC) and spsr_<exception> | ||
200 | - @ (parent CPSR) | ||
201 | - @ | ||
202 | + @ Save r0, lr_<exception> (parent PC) | ||
203 | stmia sp, {r0, lr} @ save r0, lr | ||
204 | - mrs lr, spsr | ||
205 | + | ||
206 | + @ Save spsr_<exception> (parent CPSR) | ||
207 | +2: mrs lr, spsr | ||
208 | str lr, [sp, #8] @ save spsr | ||
209 | |||
210 | @ | ||
211 | @@ -1031,6 +1030,44 @@ vector_\name: | ||
212 | movs pc, lr @ branch to handler in SVC mode | ||
213 | ENDPROC(vector_\name) | ||
214 | |||
215 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY | ||
216 | + .subsection 1 | ||
217 | + .align 5 | ||
218 | +vector_bhb_loop8_\name: | ||
219 | + .if \correction | ||
220 | + sub lr, lr, #\correction | ||
221 | + .endif | ||
222 | + | ||
223 | + @ Save r0, lr_<exception> (parent PC) | ||
224 | + stmia sp, {r0, lr} | ||
225 | + | ||
226 | + @ bhb workaround | ||
227 | + mov r0, #8 | ||
228 | +1: b . + 4 | ||
229 | + subs r0, r0, #1 | ||
230 | + bne 1b | ||
231 | + dsb | ||
232 | + isb | ||
233 | + b 2b | ||
234 | +ENDPROC(vector_bhb_loop8_\name) | ||
235 | + | ||
236 | +vector_bhb_bpiall_\name: | ||
237 | + .if \correction | ||
238 | + sub lr, lr, #\correction | ||
239 | + .endif | ||
240 | + | ||
241 | + @ Save r0, lr_<exception> (parent PC) | ||
242 | + stmia sp, {r0, lr} | ||
243 | + | ||
244 | + @ bhb workaround | ||
245 | + mcr p15, 0, r0, c7, c5, 6 @ BPIALL | ||
246 | + @ isb not needed due to "movs pc, lr" in the vector stub | ||
247 | + @ which gives a "context synchronisation". | ||
248 | + b 2b | ||
249 | +ENDPROC(vector_bhb_bpiall_\name) | ||
250 | + .previous | ||
251 | +#endif | ||
252 | + | ||
253 | .align 2 | ||
254 | @ handler addresses follow this label | ||
255 | 1: | ||
256 | @@ -1039,6 +1076,10 @@ ENDPROC(vector_\name) | ||
257 | .section .stubs, "ax", %progbits | ||
258 | @ This must be the first word | ||
259 | .word vector_swi | ||
260 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY | ||
261 | + .word vector_bhb_loop8_swi | ||
262 | + .word vector_bhb_bpiall_swi | ||
263 | +#endif | ||
264 | |||
265 | vector_rst: | ||
266 | ARM( swi SYS_ERROR0 ) | ||
267 | @@ -1153,8 +1194,10 @@ vector_addrexcptn: | ||
268 | * FIQ "NMI" handler | ||
269 | *----------------------------------------------------------------------------- | ||
270 | * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86 | ||
271 | - * systems. | ||
272 | + * systems. This must be the last vector stub, so lets place it in its own | ||
273 | + * subsection. | ||
274 | */ | ||
275 | + .subsection 2 | ||
276 | vector_stub fiq, FIQ_MODE, 4 | ||
277 | |||
278 | .long __fiq_usr @ 0 (USR_26 / USR_32) | ||
279 | @@ -1187,6 +1230,30 @@ vector_addrexcptn: | ||
280 | W(b) vector_irq | ||
281 | W(b) vector_fiq | ||
282 | |||
283 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY | ||
284 | + .section .vectors.bhb.loop8, "ax", %progbits | ||
285 | +.L__vectors_bhb_loop8_start: | ||
286 | + W(b) vector_rst | ||
287 | + W(b) vector_bhb_loop8_und | ||
288 | + W(ldr) pc, .L__vectors_bhb_loop8_start + 0x1004 | ||
289 | + W(b) vector_bhb_loop8_pabt | ||
290 | + W(b) vector_bhb_loop8_dabt | ||
291 | + W(b) vector_addrexcptn | ||
292 | + W(b) vector_bhb_loop8_irq | ||
293 | + W(b) vector_bhb_loop8_fiq | ||
294 | + | ||
295 | + .section .vectors.bhb.bpiall, "ax", %progbits | ||
296 | +.L__vectors_bhb_bpiall_start: | ||
297 | + W(b) vector_rst | ||
298 | + W(b) vector_bhb_bpiall_und | ||
299 | + W(ldr) pc, .L__vectors_bhb_bpiall_start + 0x1008 | ||
300 | + W(b) vector_bhb_bpiall_pabt | ||
301 | + W(b) vector_bhb_bpiall_dabt | ||
302 | + W(b) vector_addrexcptn | ||
303 | + W(b) vector_bhb_bpiall_irq | ||
304 | + W(b) vector_bhb_bpiall_fiq | ||
305 | +#endif | ||
306 | + | ||
307 | .data | ||
308 | .align 2 | ||
309 | |||
310 | diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S | ||
311 | index 271cb8a1eba1e..bd619da73c84e 100644 | ||
312 | --- a/arch/arm/kernel/entry-common.S | ||
313 | +++ b/arch/arm/kernel/entry-common.S | ||
314 | @@ -162,6 +162,29 @@ ENDPROC(ret_from_fork) | ||
315 | *----------------------------------------------------------------------------- | ||
316 | */ | ||
317 | |||
318 | + .align 5 | ||
319 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY | ||
320 | +ENTRY(vector_bhb_loop8_swi) | ||
321 | + sub sp, sp, #PT_REGS_SIZE | ||
322 | + stmia sp, {r0 - r12} | ||
323 | + mov r8, #8 | ||
324 | +1: b 2f | ||
325 | +2: subs r8, r8, #1 | ||
326 | + bne 1b | ||
327 | + dsb | ||
328 | + isb | ||
329 | + b 3f | ||
330 | +ENDPROC(vector_bhb_loop8_swi) | ||
331 | + | ||
332 | + .align 5 | ||
333 | +ENTRY(vector_bhb_bpiall_swi) | ||
334 | + sub sp, sp, #PT_REGS_SIZE | ||
335 | + stmia sp, {r0 - r12} | ||
336 | + mcr p15, 0, r8, c7, c5, 6 @ BPIALL | ||
337 | + isb | ||
338 | + b 3f | ||
339 | +ENDPROC(vector_bhb_bpiall_swi) | ||
340 | +#endif | ||
341 | .align 5 | ||
342 | ENTRY(vector_swi) | ||
343 | #ifdef CONFIG_CPU_V7M | ||
344 | @@ -169,6 +192,7 @@ ENTRY(vector_swi) | ||
345 | #else | ||
346 | sub sp, sp, #PT_REGS_SIZE | ||
347 | stmia sp, {r0 - r12} @ Calling r0 - r12 | ||
348 | +3: | ||
349 | ARM( add r8, sp, #S_PC ) | ||
350 | ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr | ||
351 | THUMB( mov r8, sp ) | ||
352 | diff --git a/arch/arm/kernel/spectre.c b/arch/arm/kernel/spectre.c | ||
353 | new file mode 100644 | ||
354 | index 0000000000000..0dcefc36fb7a0 | ||
355 | --- /dev/null | ||
356 | +++ b/arch/arm/kernel/spectre.c | ||
357 | @@ -0,0 +1,71 @@ | ||
358 | +// SPDX-License-Identifier: GPL-2.0-only | ||
359 | +#include <linux/bpf.h> | ||
360 | +#include <linux/cpu.h> | ||
361 | +#include <linux/device.h> | ||
362 | + | ||
363 | +#include <asm/spectre.h> | ||
364 | + | ||
365 | +static bool _unprivileged_ebpf_enabled(void) | ||
366 | +{ | ||
367 | +#ifdef CONFIG_BPF_SYSCALL | ||
368 | + return !sysctl_unprivileged_bpf_disabled; | ||
369 | +#else | ||
370 | + return false; | ||
371 | +#endif | ||
372 | +} | ||
373 | + | ||
374 | +ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, | ||
375 | + char *buf) | ||
376 | +{ | ||
377 | + return sprintf(buf, "Mitigation: __user pointer sanitization\n"); | ||
378 | +} | ||
379 | + | ||
380 | +static unsigned int spectre_v2_state; | ||
381 | +static unsigned int spectre_v2_methods; | ||
382 | + | ||
383 | +void spectre_v2_update_state(unsigned int state, unsigned int method) | ||
384 | +{ | ||
385 | + if (state > spectre_v2_state) | ||
386 | + spectre_v2_state = state; | ||
387 | + spectre_v2_methods |= method; | ||
388 | +} | ||
389 | + | ||
390 | +ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, | ||
391 | + char *buf) | ||
392 | +{ | ||
393 | + const char *method; | ||
394 | + | ||
395 | + if (spectre_v2_state == SPECTRE_UNAFFECTED) | ||
396 | + return sprintf(buf, "%s\n", "Not affected"); | ||
397 | + | ||
398 | + if (spectre_v2_state != SPECTRE_MITIGATED) | ||
399 | + return sprintf(buf, "%s\n", "Vulnerable"); | ||
400 | + | ||
401 | + if (_unprivileged_ebpf_enabled()) | ||
402 | + return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); | ||
403 | + | ||
404 | + switch (spectre_v2_methods) { | ||
405 | + case SPECTRE_V2_METHOD_BPIALL: | ||
406 | + method = "Branch predictor hardening"; | ||
407 | + break; | ||
408 | + | ||
409 | + case SPECTRE_V2_METHOD_ICIALLU: | ||
410 | + method = "I-cache invalidation"; | ||
411 | + break; | ||
412 | + | ||
413 | + case SPECTRE_V2_METHOD_SMC: | ||
414 | + case SPECTRE_V2_METHOD_HVC: | ||
415 | + method = "Firmware call"; | ||
416 | + break; | ||
417 | + | ||
418 | + case SPECTRE_V2_METHOD_LOOP8: | ||
419 | + method = "History overwrite"; | ||
420 | + break; | ||
421 | + | ||
422 | + default: | ||
423 | + method = "Multiple mitigations"; | ||
424 | + break; | ||
425 | + } | ||
426 | + | ||
427 | + return sprintf(buf, "Mitigation: %s\n", method); | ||
428 | +} | ||
429 | diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c | ||
430 | index 97a512551b217..207ef9a797bd4 100644 | ||
431 | --- a/arch/arm/kernel/traps.c | ||
432 | +++ b/arch/arm/kernel/traps.c | ||
433 | @@ -30,6 +30,7 @@ | ||
434 | #include <linux/atomic.h> | ||
435 | #include <asm/cacheflush.h> | ||
436 | #include <asm/exception.h> | ||
437 | +#include <asm/spectre.h> | ||
438 | #include <asm/unistd.h> | ||
439 | #include <asm/traps.h> | ||
440 | #include <asm/ptrace.h> | ||
441 | @@ -799,10 +800,59 @@ static inline void __init kuser_init(void *vectors) | ||
442 | } | ||
443 | #endif | ||
444 | |||
445 | +#ifndef CONFIG_CPU_V7M | ||
446 | +static void copy_from_lma(void *vma, void *lma_start, void *lma_end) | ||
447 | +{ | ||
448 | + memcpy(vma, lma_start, lma_end - lma_start); | ||
449 | +} | ||
450 | + | ||
451 | +static void flush_vectors(void *vma, size_t offset, size_t size) | ||
452 | +{ | ||
453 | + unsigned long start = (unsigned long)vma + offset; | ||
454 | + unsigned long end = start + size; | ||
455 | + | ||
456 | + flush_icache_range(start, end); | ||
457 | +} | ||
458 | + | ||
459 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY | ||
460 | +int spectre_bhb_update_vectors(unsigned int method) | ||
461 | +{ | ||
462 | + extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[]; | ||
463 | + extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[]; | ||
464 | + void *vec_start, *vec_end; | ||
465 | + | ||
466 | + if (system_state > SYSTEM_SCHEDULING) { | ||
467 | + pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n", | ||
468 | + smp_processor_id()); | ||
469 | + return SPECTRE_VULNERABLE; | ||
470 | + } | ||
471 | + | ||
472 | + switch (method) { | ||
473 | + case SPECTRE_V2_METHOD_LOOP8: | ||
474 | + vec_start = __vectors_bhb_loop8_start; | ||
475 | + vec_end = __vectors_bhb_loop8_end; | ||
476 | + break; | ||
477 | + | ||
478 | + case SPECTRE_V2_METHOD_BPIALL: | ||
479 | + vec_start = __vectors_bhb_bpiall_start; | ||
480 | + vec_end = __vectors_bhb_bpiall_end; | ||
481 | + break; | ||
482 | + | ||
483 | + default: | ||
484 | + pr_err("CPU%u: unknown Spectre BHB state %d\n", | ||
485 | + smp_processor_id(), method); | ||
486 | + return SPECTRE_VULNERABLE; | ||
487 | + } | ||
488 | + | ||
489 | + copy_from_lma(vectors_page, vec_start, vec_end); | ||
490 | + flush_vectors(vectors_page, 0, vec_end - vec_start); | ||
491 | + | ||
492 | + return SPECTRE_MITIGATED; | ||
493 | +} | ||
494 | +#endif | ||
495 | + | ||
496 | void __init early_trap_init(void *vectors_base) | ||
497 | { | ||
498 | -#ifndef CONFIG_CPU_V7M | ||
499 | - unsigned long vectors = (unsigned long)vectors_base; | ||
500 | extern char __stubs_start[], __stubs_end[]; | ||
501 | extern char __vectors_start[], __vectors_end[]; | ||
502 | unsigned i; | ||
503 | @@ -823,17 +873,20 @@ void __init early_trap_init(void *vectors_base) | ||
504 | * into the vector page, mapped at 0xffff0000, and ensure these | ||
505 | * are visible to the instruction stream. | ||
506 | */ | ||
507 | - memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start); | ||
508 | - memcpy((void *)vectors + 0x1000, __stubs_start, __stubs_end - __stubs_start); | ||
509 | + copy_from_lma(vectors_base, __vectors_start, __vectors_end); | ||
510 | + copy_from_lma(vectors_base + 0x1000, __stubs_start, __stubs_end); | ||
511 | |||
512 | kuser_init(vectors_base); | ||
513 | |||
514 | - flush_icache_range(vectors, vectors + PAGE_SIZE * 2); | ||
515 | + flush_vectors(vectors_base, 0, PAGE_SIZE * 2); | ||
516 | +} | ||
517 | #else /* ifndef CONFIG_CPU_V7M */ | ||
518 | +void __init early_trap_init(void *vectors_base) | ||
519 | +{ | ||
520 | /* | ||
521 | * on V7-M there is no need to copy the vector table to a dedicated | ||
522 | * memory area. The address is configurable and so a table in the kernel | ||
523 | * image can be used. | ||
524 | */ | ||
525 | -#endif | ||
526 | } | ||
527 | +#endif | ||
528 | diff --git a/arch/arm/kernel/vmlinux.lds.h b/arch/arm/kernel/vmlinux.lds.h | ||
529 | index 8247bc15addc4..78d156e4f0088 100644 | ||
530 | --- a/arch/arm/kernel/vmlinux.lds.h | ||
531 | +++ b/arch/arm/kernel/vmlinux.lds.h | ||
532 | @@ -25,6 +25,19 @@ | ||
533 | #define ARM_MMU_DISCARD(x) x | ||
534 | #endif | ||
535 | |||
536 | +/* | ||
537 | + * ld.lld does not support NOCROSSREFS: | ||
538 | + * https://github.com/ClangBuiltLinux/linux/issues/1609 | ||
539 | + */ | ||
540 | +#ifdef CONFIG_LD_IS_LLD | ||
541 | +#define NOCROSSREFS | ||
542 | +#endif | ||
543 | + | ||
544 | +/* Set start/end symbol names to the LMA for the section */ | ||
545 | +#define ARM_LMA(sym, section) \ | ||
546 | + sym##_start = LOADADDR(section); \ | ||
547 | + sym##_end = LOADADDR(section) + SIZEOF(section) | ||
548 | + | ||
549 | #define PROC_INFO \ | ||
550 | . = ALIGN(4); \ | ||
551 | __proc_info_begin = .; \ | ||
552 | @@ -100,19 +113,31 @@ | ||
553 | * only thing that matters is their relative offsets | ||
554 | */ | ||
555 | #define ARM_VECTORS \ | ||
556 | - __vectors_start = .; \ | ||
557 | - .vectors 0xffff0000 : AT(__vectors_start) { \ | ||
558 | - *(.vectors) \ | ||
559 | + __vectors_lma = .; \ | ||
560 | + OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { \ | ||
561 | + .vectors { \ | ||
562 | + *(.vectors) \ | ||
563 | + } \ | ||
564 | + .vectors.bhb.loop8 { \ | ||
565 | + *(.vectors.bhb.loop8) \ | ||
566 | + } \ | ||
567 | + .vectors.bhb.bpiall { \ | ||
568 | + *(.vectors.bhb.bpiall) \ | ||
569 | + } \ | ||
570 | } \ | ||
571 | - . = __vectors_start + SIZEOF(.vectors); \ | ||
572 | - __vectors_end = .; \ | ||
573 | + ARM_LMA(__vectors, .vectors); \ | ||
574 | + ARM_LMA(__vectors_bhb_loop8, .vectors.bhb.loop8); \ | ||
575 | + ARM_LMA(__vectors_bhb_bpiall, .vectors.bhb.bpiall); \ | ||
576 | + . = __vectors_lma + SIZEOF(.vectors) + \ | ||
577 | + SIZEOF(.vectors.bhb.loop8) + \ | ||
578 | + SIZEOF(.vectors.bhb.bpiall); \ | ||
579 | \ | ||
580 | - __stubs_start = .; \ | ||
581 | - .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_start) { \ | ||
582 | + __stubs_lma = .; \ | ||
583 | + .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { \ | ||
584 | *(.stubs) \ | ||
585 | } \ | ||
586 | - . = __stubs_start + SIZEOF(.stubs); \ | ||
587 | - __stubs_end = .; \ | ||
588 | + ARM_LMA(__stubs, .stubs); \ | ||
589 | + . = __stubs_lma + SIZEOF(.stubs); \ | ||
590 | \ | ||
591 | PROVIDE(vector_fiq_offset = vector_fiq - ADDR(.vectors)); | ||
592 | |||
593 | diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig | ||
594 | index 64cce0c8560ab..00ffee644372e 100644 | ||
595 | --- a/arch/arm/mm/Kconfig | ||
596 | +++ b/arch/arm/mm/Kconfig | ||
597 | @@ -833,6 +833,7 @@ config CPU_BPREDICT_DISABLE | ||
598 | |||
599 | config CPU_SPECTRE | ||
600 | bool | ||
601 | + select GENERIC_CPU_VULNERABILITIES | ||
602 | |||
603 | config HARDEN_BRANCH_PREDICTOR | ||
604 | bool "Harden the branch predictor against aliasing attacks" if EXPERT | ||
605 | @@ -853,6 +854,16 @@ config HARDEN_BRANCH_PREDICTOR | ||
606 | |||
607 | If unsure, say Y. | ||
608 | |||
609 | +config HARDEN_BRANCH_HISTORY | ||
610 | + bool "Harden Spectre style attacks against branch history" if EXPERT | ||
611 | + depends on CPU_SPECTRE | ||
612 | + default y | ||
613 | + help | ||
614 | + Speculation attacks against some high-performance processors can | ||
615 | + make use of branch history to influence future speculation. When | ||
616 | + taking an exception, a sequence of branches overwrites the branch | ||
617 | + history, or branch history is invalidated. | ||
618 | + | ||
619 | config TLS_REG_EMUL | ||
620 | bool | ||
621 | select NEED_KUSER_HELPERS | ||
622 | diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c | ||
623 | index a6554fdb56c54..097ef85bb7f21 100644 | ||
624 | --- a/arch/arm/mm/proc-v7-bugs.c | ||
625 | +++ b/arch/arm/mm/proc-v7-bugs.c | ||
626 | @@ -7,8 +7,35 @@ | ||
627 | #include <asm/cp15.h> | ||
628 | #include <asm/cputype.h> | ||
629 | #include <asm/proc-fns.h> | ||
630 | +#include <asm/spectre.h> | ||
631 | #include <asm/system_misc.h> | ||
632 | |||
633 | +#ifdef CONFIG_ARM_PSCI | ||
634 | +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) | ||
635 | +{ | ||
636 | + struct arm_smccc_res res; | ||
637 | + | ||
638 | + arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, | ||
639 | + ARM_SMCCC_ARCH_WORKAROUND_1, &res); | ||
640 | + | ||
641 | + switch ((int)res.a0) { | ||
642 | + case SMCCC_RET_SUCCESS: | ||
643 | + return SPECTRE_MITIGATED; | ||
644 | + | ||
645 | + case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: | ||
646 | + return SPECTRE_UNAFFECTED; | ||
647 | + | ||
648 | + default: | ||
649 | + return SPECTRE_VULNERABLE; | ||
650 | + } | ||
651 | +} | ||
652 | +#else | ||
653 | +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) | ||
654 | +{ | ||
655 | + return SPECTRE_VULNERABLE; | ||
656 | +} | ||
657 | +#endif | ||
658 | + | ||
659 | #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR | ||
660 | DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); | ||
661 | |||
662 | @@ -37,13 +64,61 @@ static void __maybe_unused call_hvc_arch_workaround_1(void) | ||
663 | arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); | ||
664 | } | ||
665 | |||
666 | -static void cpu_v7_spectre_init(void) | ||
667 | +static unsigned int spectre_v2_install_workaround(unsigned int method) | ||
668 | { | ||
669 | const char *spectre_v2_method = NULL; | ||
670 | int cpu = smp_processor_id(); | ||
671 | |||
672 | if (per_cpu(harden_branch_predictor_fn, cpu)) | ||
673 | - return; | ||
674 | + return SPECTRE_MITIGATED; | ||
675 | + | ||
676 | + switch (method) { | ||
677 | + case SPECTRE_V2_METHOD_BPIALL: | ||
678 | + per_cpu(harden_branch_predictor_fn, cpu) = | ||
679 | + harden_branch_predictor_bpiall; | ||
680 | + spectre_v2_method = "BPIALL"; | ||
681 | + break; | ||
682 | + | ||
683 | + case SPECTRE_V2_METHOD_ICIALLU: | ||
684 | + per_cpu(harden_branch_predictor_fn, cpu) = | ||
685 | + harden_branch_predictor_iciallu; | ||
686 | + spectre_v2_method = "ICIALLU"; | ||
687 | + break; | ||
688 | + | ||
689 | + case SPECTRE_V2_METHOD_HVC: | ||
690 | + per_cpu(harden_branch_predictor_fn, cpu) = | ||
691 | + call_hvc_arch_workaround_1; | ||
692 | + cpu_do_switch_mm = cpu_v7_hvc_switch_mm; | ||
693 | + spectre_v2_method = "hypervisor"; | ||
694 | + break; | ||
695 | + | ||
696 | + case SPECTRE_V2_METHOD_SMC: | ||
697 | + per_cpu(harden_branch_predictor_fn, cpu) = | ||
698 | + call_smc_arch_workaround_1; | ||
699 | + cpu_do_switch_mm = cpu_v7_smc_switch_mm; | ||
700 | + spectre_v2_method = "firmware"; | ||
701 | + break; | ||
702 | + } | ||
703 | + | ||
704 | + if (spectre_v2_method) | ||
705 | + pr_info("CPU%u: Spectre v2: using %s workaround\n", | ||
706 | + smp_processor_id(), spectre_v2_method); | ||
707 | + | ||
708 | + return SPECTRE_MITIGATED; | ||
709 | +} | ||
710 | +#else | ||
711 | +static unsigned int spectre_v2_install_workaround(unsigned int method) | ||
712 | +{ | ||
713 | + pr_info("CPU%u: Spectre V2: workarounds disabled by configuration\n", | ||
714 | + smp_processor_id()); | ||
715 | + | ||
716 | + return SPECTRE_VULNERABLE; | ||
717 | +} | ||
718 | +#endif | ||
719 | + | ||
720 | +static void cpu_v7_spectre_v2_init(void) | ||
721 | +{ | ||
722 | + unsigned int state, method = 0; | ||
723 | |||
724 | switch (read_cpuid_part()) { | ||
725 | case ARM_CPU_PART_CORTEX_A8: | ||
726 | @@ -52,32 +127,37 @@ static void cpu_v7_spectre_init(void) | ||
727 | case ARM_CPU_PART_CORTEX_A17: | ||
728 | case ARM_CPU_PART_CORTEX_A73: | ||
729 | case ARM_CPU_PART_CORTEX_A75: | ||
730 | - per_cpu(harden_branch_predictor_fn, cpu) = | ||
731 | - harden_branch_predictor_bpiall; | ||
732 | - spectre_v2_method = "BPIALL"; | ||
733 | + state = SPECTRE_MITIGATED; | ||
734 | + method = SPECTRE_V2_METHOD_BPIALL; | ||
735 | break; | ||
736 | |||
737 | case ARM_CPU_PART_CORTEX_A15: | ||
738 | case ARM_CPU_PART_BRAHMA_B15: | ||
739 | - per_cpu(harden_branch_predictor_fn, cpu) = | ||
740 | - harden_branch_predictor_iciallu; | ||
741 | - spectre_v2_method = "ICIALLU"; | ||
742 | + state = SPECTRE_MITIGATED; | ||
743 | + method = SPECTRE_V2_METHOD_ICIALLU; | ||
744 | break; | ||
745 | |||
746 | -#ifdef CONFIG_ARM_PSCI | ||
747 | case ARM_CPU_PART_BRAHMA_B53: | ||
748 | /* Requires no workaround */ | ||
749 | + state = SPECTRE_UNAFFECTED; | ||
750 | break; | ||
751 | + | ||
752 | default: | ||
753 | /* Other ARM CPUs require no workaround */ | ||
754 | - if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) | ||
755 | + if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) { | ||
756 | + state = SPECTRE_UNAFFECTED; | ||
757 | break; | ||
758 | + } | ||
759 | /* fallthrough */ | ||
760 | - /* Cortex A57/A72 require firmware workaround */ | ||
761 | + /* Cortex A57/A72 require firmware workaround */ | ||
762 | case ARM_CPU_PART_CORTEX_A57: | ||
763 | case ARM_CPU_PART_CORTEX_A72: { | ||
764 | struct arm_smccc_res res; | ||
765 | |||
766 | + state = spectre_v2_get_cpu_fw_mitigation_state(); | ||
767 | + if (state != SPECTRE_MITIGATED) | ||
768 | + break; | ||
769 | + | ||
770 | if (psci_ops.smccc_version == SMCCC_VERSION_1_0) | ||
771 | break; | ||
772 | |||
773 | @@ -87,10 +167,7 @@ static void cpu_v7_spectre_init(void) | ||
774 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); | ||
775 | if ((int)res.a0 != 0) | ||
776 | break; | ||
777 | - per_cpu(harden_branch_predictor_fn, cpu) = | ||
778 | - call_hvc_arch_workaround_1; | ||
779 | - cpu_do_switch_mm = cpu_v7_hvc_switch_mm; | ||
780 | - spectre_v2_method = "hypervisor"; | ||
781 | + method = SPECTRE_V2_METHOD_HVC; | ||
782 | break; | ||
783 | |||
784 | case PSCI_CONDUIT_SMC: | ||
785 | @@ -98,29 +175,97 @@ static void cpu_v7_spectre_init(void) | ||
786 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); | ||
787 | if ((int)res.a0 != 0) | ||
788 | break; | ||
789 | - per_cpu(harden_branch_predictor_fn, cpu) = | ||
790 | - call_smc_arch_workaround_1; | ||
791 | - cpu_do_switch_mm = cpu_v7_smc_switch_mm; | ||
792 | - spectre_v2_method = "firmware"; | ||
793 | + method = SPECTRE_V2_METHOD_SMC; | ||
794 | break; | ||
795 | |||
796 | default: | ||
797 | + state = SPECTRE_VULNERABLE; | ||
798 | break; | ||
799 | } | ||
800 | } | ||
801 | -#endif | ||
802 | } | ||
803 | |||
804 | - if (spectre_v2_method) | ||
805 | - pr_info("CPU%u: Spectre v2: using %s workaround\n", | ||
806 | - smp_processor_id(), spectre_v2_method); | ||
807 | + if (state == SPECTRE_MITIGATED) | ||
808 | + state = spectre_v2_install_workaround(method); | ||
809 | + | ||
810 | + spectre_v2_update_state(state, method); | ||
811 | +} | ||
812 | + | ||
813 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY | ||
814 | +static int spectre_bhb_method; | ||
815 | + | ||
816 | +static const char *spectre_bhb_method_name(int method) | ||
817 | +{ | ||
818 | + switch (method) { | ||
819 | + case SPECTRE_V2_METHOD_LOOP8: | ||
820 | + return "loop"; | ||
821 | + | ||
822 | + case SPECTRE_V2_METHOD_BPIALL: | ||
823 | + return "BPIALL"; | ||
824 | + | ||
825 | + default: | ||
826 | + return "unknown"; | ||
827 | + } | ||
828 | +} | ||
829 | + | ||
830 | +static int spectre_bhb_install_workaround(int method) | ||
831 | +{ | ||
832 | + if (spectre_bhb_method != method) { | ||
833 | + if (spectre_bhb_method) { | ||
834 | + pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n", | ||
835 | + smp_processor_id()); | ||
836 | + | ||
837 | + return SPECTRE_VULNERABLE; | ||
838 | + } | ||
839 | + | ||
840 | + if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE) | ||
841 | + return SPECTRE_VULNERABLE; | ||
842 | + | ||
843 | + spectre_bhb_method = method; | ||
844 | + } | ||
845 | + | ||
846 | + pr_info("CPU%u: Spectre BHB: using %s workaround\n", | ||
847 | + smp_processor_id(), spectre_bhb_method_name(method)); | ||
848 | + | ||
849 | + return SPECTRE_MITIGATED; | ||
850 | } | ||
851 | #else | ||
852 | -static void cpu_v7_spectre_init(void) | ||
853 | +static int spectre_bhb_install_workaround(int method) | ||
854 | { | ||
855 | + return SPECTRE_VULNERABLE; | ||
856 | } | ||
857 | #endif | ||
858 | |||
859 | +static void cpu_v7_spectre_bhb_init(void) | ||
860 | +{ | ||
861 | + unsigned int state, method = 0; | ||
862 | + | ||
863 | + switch (read_cpuid_part()) { | ||
864 | + case ARM_CPU_PART_CORTEX_A15: | ||
865 | + case ARM_CPU_PART_BRAHMA_B15: | ||
866 | + case ARM_CPU_PART_CORTEX_A57: | ||
867 | + case ARM_CPU_PART_CORTEX_A72: | ||
868 | + state = SPECTRE_MITIGATED; | ||
869 | + method = SPECTRE_V2_METHOD_LOOP8; | ||
870 | + break; | ||
871 | + | ||
872 | + case ARM_CPU_PART_CORTEX_A73: | ||
873 | + case ARM_CPU_PART_CORTEX_A75: | ||
874 | + state = SPECTRE_MITIGATED; | ||
875 | + method = SPECTRE_V2_METHOD_BPIALL; | ||
876 | + break; | ||
877 | + | ||
878 | + default: | ||
879 | + state = SPECTRE_UNAFFECTED; | ||
880 | + break; | ||
881 | + } | ||
882 | + | ||
883 | + if (state == SPECTRE_MITIGATED) | ||
884 | + state = spectre_bhb_install_workaround(method); | ||
885 | + | ||
886 | + spectre_v2_update_state(state, method); | ||
887 | +} | ||
888 | + | ||
889 | static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, | ||
890 | u32 mask, const char *msg) | ||
891 | { | ||
892 | @@ -149,16 +294,17 @@ static bool check_spectre_auxcr(bool *warned, u32 bit) | ||
893 | void cpu_v7_ca8_ibe(void) | ||
894 | { | ||
895 | if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6))) | ||
896 | - cpu_v7_spectre_init(); | ||
897 | + cpu_v7_spectre_v2_init(); | ||
898 | } | ||
899 | |||
900 | void cpu_v7_ca15_ibe(void) | ||
901 | { | ||
902 | if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0))) | ||
903 | - cpu_v7_spectre_init(); | ||
904 | + cpu_v7_spectre_v2_init(); | ||
905 | } | ||
906 | |||
907 | void cpu_v7_bugs_init(void) | ||
908 | { | ||
909 | - cpu_v7_spectre_init(); | ||
910 | + cpu_v7_spectre_v2_init(); | ||
911 | + cpu_v7_spectre_bhb_init(); | ||
912 | } | ||
913 | diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h | ||
914 | index d912457f56a79..f48905f796e9d 100644 | ||
915 | --- a/arch/x86/include/asm/cpufeatures.h | ||
916 | +++ b/arch/x86/include/asm/cpufeatures.h | ||
917 | @@ -202,7 +202,7 @@ | ||
918 | #define X86_FEATURE_SME ( 7*32+10) /* AMD Secure Memory Encryption */ | ||
919 | #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */ | ||
920 | #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ | ||
921 | -#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */ | ||
922 | +#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCE for Spectre variant 2 */ | ||
923 | #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ | ||
924 | #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */ | ||
925 | #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ | ||
926 | diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h | ||
927 | index b222a35959467..956df82bbc2bc 100644 | ||
928 | --- a/arch/x86/include/asm/nospec-branch.h | ||
929 | +++ b/arch/x86/include/asm/nospec-branch.h | ||
930 | @@ -115,7 +115,7 @@ | ||
931 | ANNOTATE_NOSPEC_ALTERNATIVE | ||
932 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ | ||
933 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ | ||
934 | - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD | ||
935 | + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_LFENCE | ||
936 | #else | ||
937 | jmp *\reg | ||
938 | #endif | ||
939 | @@ -126,7 +126,7 @@ | ||
940 | ANNOTATE_NOSPEC_ALTERNATIVE | ||
941 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ | ||
942 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ | ||
943 | - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD | ||
944 | + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_LFENCE | ||
945 | #else | ||
946 | call *\reg | ||
947 | #endif | ||
948 | @@ -171,7 +171,7 @@ | ||
949 | "lfence;\n" \ | ||
950 | ANNOTATE_RETPOLINE_SAFE \ | ||
951 | "call *%[thunk_target]\n", \ | ||
952 | - X86_FEATURE_RETPOLINE_AMD) | ||
953 | + X86_FEATURE_RETPOLINE_LFENCE) | ||
954 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) | ||
955 | |||
956 | #else /* CONFIG_X86_32 */ | ||
957 | @@ -201,7 +201,7 @@ | ||
958 | "lfence;\n" \ | ||
959 | ANNOTATE_RETPOLINE_SAFE \ | ||
960 | "call *%[thunk_target]\n", \ | ||
961 | - X86_FEATURE_RETPOLINE_AMD) | ||
962 | + X86_FEATURE_RETPOLINE_LFENCE) | ||
963 | |||
964 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | ||
965 | #endif | ||
966 | @@ -213,9 +213,11 @@ | ||
967 | /* The Spectre V2 mitigation variants */ | ||
968 | enum spectre_v2_mitigation { | ||
969 | SPECTRE_V2_NONE, | ||
970 | - SPECTRE_V2_RETPOLINE_GENERIC, | ||
971 | - SPECTRE_V2_RETPOLINE_AMD, | ||
972 | - SPECTRE_V2_IBRS_ENHANCED, | ||
973 | + SPECTRE_V2_RETPOLINE, | ||
974 | + SPECTRE_V2_LFENCE, | ||
975 | + SPECTRE_V2_EIBRS, | ||
976 | + SPECTRE_V2_EIBRS_RETPOLINE, | ||
977 | + SPECTRE_V2_EIBRS_LFENCE, | ||
978 | }; | ||
979 | |||
980 | /* The indirect branch speculation control variants */ | ||
981 | diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c | ||
982 | index fcc4238ee95f8..e817aaeef254c 100644 | ||
983 | --- a/arch/x86/kernel/cpu/bugs.c | ||
984 | +++ b/arch/x86/kernel/cpu/bugs.c | ||
985 | @@ -31,6 +31,7 @@ | ||
986 | #include <asm/intel-family.h> | ||
987 | #include <asm/e820/api.h> | ||
988 | #include <asm/hypervisor.h> | ||
989 | +#include <linux/bpf.h> | ||
990 | |||
991 | #include "cpu.h" | ||
992 | |||
993 | @@ -607,6 +608,32 @@ static inline const char *spectre_v2_module_string(void) | ||
994 | static inline const char *spectre_v2_module_string(void) { return ""; } | ||
995 | #endif | ||
996 | |||
997 | +#define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" | ||
998 | +#define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" | ||
999 | +#define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" | ||
1000 | + | ||
1001 | +#ifdef CONFIG_BPF_SYSCALL | ||
1002 | +void unpriv_ebpf_notify(int new_state) | ||
1003 | +{ | ||
1004 | + if (new_state) | ||
1005 | + return; | ||
1006 | + | ||
1007 | + /* Unprivileged eBPF is enabled */ | ||
1008 | + | ||
1009 | + switch (spectre_v2_enabled) { | ||
1010 | + case SPECTRE_V2_EIBRS: | ||
1011 | + pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); | ||
1012 | + break; | ||
1013 | + case SPECTRE_V2_EIBRS_LFENCE: | ||
1014 | + if (sched_smt_active()) | ||
1015 | + pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); | ||
1016 | + break; | ||
1017 | + default: | ||
1018 | + break; | ||
1019 | + } | ||
1020 | +} | ||
1021 | +#endif | ||
1022 | + | ||
1023 | static inline bool match_option(const char *arg, int arglen, const char *opt) | ||
1024 | { | ||
1025 | int len = strlen(opt); | ||
1026 | @@ -621,7 +648,10 @@ enum spectre_v2_mitigation_cmd { | ||
1027 | SPECTRE_V2_CMD_FORCE, | ||
1028 | SPECTRE_V2_CMD_RETPOLINE, | ||
1029 | SPECTRE_V2_CMD_RETPOLINE_GENERIC, | ||
1030 | - SPECTRE_V2_CMD_RETPOLINE_AMD, | ||
1031 | + SPECTRE_V2_CMD_RETPOLINE_LFENCE, | ||
1032 | + SPECTRE_V2_CMD_EIBRS, | ||
1033 | + SPECTRE_V2_CMD_EIBRS_RETPOLINE, | ||
1034 | + SPECTRE_V2_CMD_EIBRS_LFENCE, | ||
1035 | }; | ||
1036 | |||
1037 | enum spectre_v2_user_cmd { | ||
1038 | @@ -694,6 +724,13 @@ spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd) | ||
1039 | return SPECTRE_V2_USER_CMD_AUTO; | ||
1040 | } | ||
1041 | |||
1042 | +static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode) | ||
1043 | +{ | ||
1044 | + return (mode == SPECTRE_V2_EIBRS || | ||
1045 | + mode == SPECTRE_V2_EIBRS_RETPOLINE || | ||
1046 | + mode == SPECTRE_V2_EIBRS_LFENCE); | ||
1047 | +} | ||
1048 | + | ||
1049 | static void __init | ||
1050 | spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) | ||
1051 | { | ||
1052 | @@ -756,10 +793,12 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) | ||
1053 | } | ||
1054 | |||
1055 | /* | ||
1056 | - * If enhanced IBRS is enabled or SMT impossible, STIBP is not | ||
1057 | + * If no STIBP, enhanced IBRS is enabled or SMT impossible, STIBP is not | ||
1058 | * required. | ||
1059 | */ | ||
1060 | - if (!smt_possible || spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) | ||
1061 | + if (!boot_cpu_has(X86_FEATURE_STIBP) || | ||
1062 | + !smt_possible || | ||
1063 | + spectre_v2_in_eibrs_mode(spectre_v2_enabled)) | ||
1064 | return; | ||
1065 | |||
1066 | /* | ||
1067 | @@ -771,12 +810,6 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) | ||
1068 | boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) | ||
1069 | mode = SPECTRE_V2_USER_STRICT_PREFERRED; | ||
1070 | |||
1071 | - /* | ||
1072 | - * If STIBP is not available, clear the STIBP mode. | ||
1073 | - */ | ||
1074 | - if (!boot_cpu_has(X86_FEATURE_STIBP)) | ||
1075 | - mode = SPECTRE_V2_USER_NONE; | ||
1076 | - | ||
1077 | spectre_v2_user_stibp = mode; | ||
1078 | |||
1079 | set_mode: | ||
1080 | @@ -785,9 +818,11 @@ set_mode: | ||
1081 | |||
1082 | static const char * const spectre_v2_strings[] = { | ||
1083 | [SPECTRE_V2_NONE] = "Vulnerable", | ||
1084 | - [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", | ||
1085 | - [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", | ||
1086 | - [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", | ||
1087 | + [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", | ||
1088 | + [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", | ||
1089 | + [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS", | ||
1090 | + [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE", | ||
1091 | + [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines", | ||
1092 | }; | ||
1093 | |||
1094 | static const struct { | ||
1095 | @@ -798,8 +833,12 @@ static const struct { | ||
1096 | { "off", SPECTRE_V2_CMD_NONE, false }, | ||
1097 | { "on", SPECTRE_V2_CMD_FORCE, true }, | ||
1098 | { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, | ||
1099 | - { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false }, | ||
1100 | + { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, | ||
1101 | + { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, | ||
1102 | { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, | ||
1103 | + { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, | ||
1104 | + { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, | ||
1105 | + { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, | ||
1106 | { "auto", SPECTRE_V2_CMD_AUTO, false }, | ||
1107 | }; | ||
1108 | |||
1109 | @@ -836,17 +875,30 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) | ||
1110 | } | ||
1111 | |||
1112 | if ((cmd == SPECTRE_V2_CMD_RETPOLINE || | ||
1113 | - cmd == SPECTRE_V2_CMD_RETPOLINE_AMD || | ||
1114 | - cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && | ||
1115 | + cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || | ||
1116 | + cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || | ||
1117 | + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || | ||
1118 | + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && | ||
1119 | !IS_ENABLED(CONFIG_RETPOLINE)) { | ||
1120 | - pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); | ||
1121 | + pr_err("%s selected but not compiled in. Switching to AUTO select\n", | ||
1122 | + mitigation_options[i].option); | ||
1123 | + return SPECTRE_V2_CMD_AUTO; | ||
1124 | + } | ||
1125 | + | ||
1126 | + if ((cmd == SPECTRE_V2_CMD_EIBRS || | ||
1127 | + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || | ||
1128 | + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && | ||
1129 | + !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { | ||
1130 | + pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n", | ||
1131 | + mitigation_options[i].option); | ||
1132 | return SPECTRE_V2_CMD_AUTO; | ||
1133 | } | ||
1134 | |||
1135 | - if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD && | ||
1136 | - boot_cpu_data.x86_vendor != X86_VENDOR_HYGON && | ||
1137 | - boot_cpu_data.x86_vendor != X86_VENDOR_AMD) { | ||
1138 | - pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n"); | ||
1139 | + if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || | ||
1140 | + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && | ||
1141 | + !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { | ||
1142 | + pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", | ||
1143 | + mitigation_options[i].option); | ||
1144 | return SPECTRE_V2_CMD_AUTO; | ||
1145 | } | ||
1146 | |||
1147 | @@ -855,6 +907,16 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) | ||
1148 | return cmd; | ||
1149 | } | ||
1150 | |||
1151 | +static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) | ||
1152 | +{ | ||
1153 | + if (!IS_ENABLED(CONFIG_RETPOLINE)) { | ||
1154 | + pr_err("Kernel not compiled with retpoline; no mitigation available!"); | ||
1155 | + return SPECTRE_V2_NONE; | ||
1156 | + } | ||
1157 | + | ||
1158 | + return SPECTRE_V2_RETPOLINE; | ||
1159 | +} | ||
1160 | + | ||
1161 | static void __init spectre_v2_select_mitigation(void) | ||
1162 | { | ||
1163 | enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); | ||
1164 | @@ -875,49 +937,64 @@ static void __init spectre_v2_select_mitigation(void) | ||
1165 | case SPECTRE_V2_CMD_FORCE: | ||
1166 | case SPECTRE_V2_CMD_AUTO: | ||
1167 | if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { | ||
1168 | - mode = SPECTRE_V2_IBRS_ENHANCED; | ||
1169 | - /* Force it so VMEXIT will restore correctly */ | ||
1170 | - x86_spec_ctrl_base |= SPEC_CTRL_IBRS; | ||
1171 | - wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); | ||
1172 | - goto specv2_set_mode; | ||
1173 | + mode = SPECTRE_V2_EIBRS; | ||
1174 | + break; | ||
1175 | } | ||
1176 | - if (IS_ENABLED(CONFIG_RETPOLINE)) | ||
1177 | - goto retpoline_auto; | ||
1178 | + | ||
1179 | + mode = spectre_v2_select_retpoline(); | ||
1180 | break; | ||
1181 | - case SPECTRE_V2_CMD_RETPOLINE_AMD: | ||
1182 | - if (IS_ENABLED(CONFIG_RETPOLINE)) | ||
1183 | - goto retpoline_amd; | ||
1184 | + | ||
1185 | + case SPECTRE_V2_CMD_RETPOLINE_LFENCE: | ||
1186 | + pr_err(SPECTRE_V2_LFENCE_MSG); | ||
1187 | + mode = SPECTRE_V2_LFENCE; | ||
1188 | break; | ||
1189 | + | ||
1190 | case SPECTRE_V2_CMD_RETPOLINE_GENERIC: | ||
1191 | - if (IS_ENABLED(CONFIG_RETPOLINE)) | ||
1192 | - goto retpoline_generic; | ||
1193 | + mode = SPECTRE_V2_RETPOLINE; | ||
1194 | break; | ||
1195 | + | ||
1196 | case SPECTRE_V2_CMD_RETPOLINE: | ||
1197 | - if (IS_ENABLED(CONFIG_RETPOLINE)) | ||
1198 | - goto retpoline_auto; | ||
1199 | + mode = spectre_v2_select_retpoline(); | ||
1200 | + break; | ||
1201 | + | ||
1202 | + case SPECTRE_V2_CMD_EIBRS: | ||
1203 | + mode = SPECTRE_V2_EIBRS; | ||
1204 | + break; | ||
1205 | + | ||
1206 | + case SPECTRE_V2_CMD_EIBRS_LFENCE: | ||
1207 | + mode = SPECTRE_V2_EIBRS_LFENCE; | ||
1208 | + break; | ||
1209 | + | ||
1210 | + case SPECTRE_V2_CMD_EIBRS_RETPOLINE: | ||
1211 | + mode = SPECTRE_V2_EIBRS_RETPOLINE; | ||
1212 | break; | ||
1213 | } | ||
1214 | - pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!"); | ||
1215 | - return; | ||
1216 | |||
1217 | -retpoline_auto: | ||
1218 | - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || | ||
1219 | - boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { | ||
1220 | - retpoline_amd: | ||
1221 | - if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { | ||
1222 | - pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); | ||
1223 | - goto retpoline_generic; | ||
1224 | - } | ||
1225 | - mode = SPECTRE_V2_RETPOLINE_AMD; | ||
1226 | - setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); | ||
1227 | - setup_force_cpu_cap(X86_FEATURE_RETPOLINE); | ||
1228 | - } else { | ||
1229 | - retpoline_generic: | ||
1230 | - mode = SPECTRE_V2_RETPOLINE_GENERIC; | ||
1231 | + if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) | ||
1232 | + pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); | ||
1233 | + | ||
1234 | + if (spectre_v2_in_eibrs_mode(mode)) { | ||
1235 | + /* Force it so VMEXIT will restore correctly */ | ||
1236 | + x86_spec_ctrl_base |= SPEC_CTRL_IBRS; | ||
1237 | + wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); | ||
1238 | + } | ||
1239 | + | ||
1240 | + switch (mode) { | ||
1241 | + case SPECTRE_V2_NONE: | ||
1242 | + case SPECTRE_V2_EIBRS: | ||
1243 | + break; | ||
1244 | + | ||
1245 | + case SPECTRE_V2_LFENCE: | ||
1246 | + case SPECTRE_V2_EIBRS_LFENCE: | ||
1247 | + setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); | ||
1248 | + fallthrough; | ||
1249 | + | ||
1250 | + case SPECTRE_V2_RETPOLINE: | ||
1251 | + case SPECTRE_V2_EIBRS_RETPOLINE: | ||
1252 | setup_force_cpu_cap(X86_FEATURE_RETPOLINE); | ||
1253 | + break; | ||
1254 | } | ||
1255 | |||
1256 | -specv2_set_mode: | ||
1257 | spectre_v2_enabled = mode; | ||
1258 | pr_info("%s\n", spectre_v2_strings[mode]); | ||
1259 | |||
1260 | @@ -943,7 +1020,7 @@ specv2_set_mode: | ||
1261 | * the CPU supports Enhanced IBRS, kernel might un-intentionally not | ||
1262 | * enable IBRS around firmware calls. | ||
1263 | */ | ||
1264 | - if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) { | ||
1265 | + if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) { | ||
1266 | setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); | ||
1267 | pr_info("Enabling Restricted Speculation for firmware calls\n"); | ||
1268 | } | ||
1269 | @@ -1013,6 +1090,10 @@ void cpu_bugs_smt_update(void) | ||
1270 | { | ||
1271 | mutex_lock(&spec_ctrl_mutex); | ||
1272 | |||
1273 | + if (sched_smt_active() && unprivileged_ebpf_enabled() && | ||
1274 | + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) | ||
1275 | + pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); | ||
1276 | + | ||
1277 | switch (spectre_v2_user_stibp) { | ||
1278 | case SPECTRE_V2_USER_NONE: | ||
1279 | break; | ||
1280 | @@ -1267,7 +1348,6 @@ static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) | ||
1281 | if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && | ||
1282 | spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) | ||
1283 | return 0; | ||
1284 | - | ||
1285 | /* | ||
1286 | * With strict mode for both IBPB and STIBP, the instruction | ||
1287 | * code paths avoid checking this task flag and instead, | ||
1288 | @@ -1614,7 +1694,7 @@ static ssize_t tsx_async_abort_show_state(char *buf) | ||
1289 | |||
1290 | static char *stibp_state(void) | ||
1291 | { | ||
1292 | - if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) | ||
1293 | + if (spectre_v2_in_eibrs_mode(spectre_v2_enabled)) | ||
1294 | return ""; | ||
1295 | |||
1296 | switch (spectre_v2_user_stibp) { | ||
1297 | @@ -1644,6 +1724,27 @@ static char *ibpb_state(void) | ||
1298 | return ""; | ||
1299 | } | ||
1300 | |||
1301 | +static ssize_t spectre_v2_show_state(char *buf) | ||
1302 | +{ | ||
1303 | + if (spectre_v2_enabled == SPECTRE_V2_LFENCE) | ||
1304 | + return sprintf(buf, "Vulnerable: LFENCE\n"); | ||
1305 | + | ||
1306 | + if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) | ||
1307 | + return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); | ||
1308 | + | ||
1309 | + if (sched_smt_active() && unprivileged_ebpf_enabled() && | ||
1310 | + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) | ||
1311 | + return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); | ||
1312 | + | ||
1313 | + return sprintf(buf, "%s%s%s%s%s%s\n", | ||
1314 | + spectre_v2_strings[spectre_v2_enabled], | ||
1315 | + ibpb_state(), | ||
1316 | + boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", | ||
1317 | + stibp_state(), | ||
1318 | + boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", | ||
1319 | + spectre_v2_module_string()); | ||
1320 | +} | ||
1321 | + | ||
1322 | static ssize_t srbds_show_state(char *buf) | ||
1323 | { | ||
1324 | return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]); | ||
1325 | @@ -1669,12 +1770,7 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr | ||
1326 | return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); | ||
1327 | |||
1328 | case X86_BUG_SPECTRE_V2: | ||
1329 | - return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], | ||
1330 | - ibpb_state(), | ||
1331 | - boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", | ||
1332 | - stibp_state(), | ||
1333 | - boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", | ||
1334 | - spectre_v2_module_string()); | ||
1335 | + return spectre_v2_show_state(buf); | ||
1336 | |||
1337 | case X86_BUG_SPEC_STORE_BYPASS: | ||
1338 | return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); | ||
1339 | diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c | ||
1340 | index ce9a570f217ad..e5b92958c299e 100644 | ||
1341 | --- a/drivers/acpi/ec.c | ||
1342 | +++ b/drivers/acpi/ec.c | ||
1343 | @@ -2002,16 +2002,6 @@ bool acpi_ec_dispatch_gpe(void) | ||
1344 | if (acpi_any_gpe_status_set(first_ec->gpe)) | ||
1345 | return true; | ||
1346 | |||
1347 | - /* | ||
1348 | - * Cancel the SCI wakeup and process all pending events in case there | ||
1349 | - * are any wakeup ones in there. | ||
1350 | - * | ||
1351 | - * Note that if any non-EC GPEs are active at this point, the SCI will | ||
1352 | - * retrigger after the rearming in acpi_s2idle_wake(), so no events | ||
1353 | - * should be missed by canceling the wakeup here. | ||
1354 | - */ | ||
1355 | - pm_system_cancel_wakeup(); | ||
1356 | - | ||
1357 | /* | ||
1358 | * Dispatch the EC GPE in-band, but do not report wakeup in any case | ||
1359 | * to allow the caller to process events properly after that. | ||
1360 | diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c | ||
1361 | index cd590b4793e09..b0e23e3fe0d56 100644 | ||
1362 | --- a/drivers/acpi/sleep.c | ||
1363 | +++ b/drivers/acpi/sleep.c | ||
1364 | @@ -1003,13 +1003,19 @@ static bool acpi_s2idle_wake(void) | ||
1365 | if (acpi_check_wakeup_handlers()) | ||
1366 | return true; | ||
1367 | |||
1368 | - /* | ||
1369 | - * Check non-EC GPE wakeups and if there are none, cancel the | ||
1370 | - * SCI-related wakeup and dispatch the EC GPE. | ||
1371 | - */ | ||
1372 | + /* Check non-EC GPE wakeups and dispatch the EC GPE. */ | ||
1373 | if (acpi_ec_dispatch_gpe()) | ||
1374 | return true; | ||
1375 | |||
1376 | + /* | ||
1377 | + * Cancel the SCI wakeup and process all pending events in case | ||
1378 | + * there are any wakeup ones in there. | ||
1379 | + * | ||
1380 | + * Note that if any non-EC GPEs are active at this point, the | ||
1381 | + * SCI will retrigger after the rearming below, so no events | ||
1382 | + * should be missed by canceling the wakeup here. | ||
1383 | + */ | ||
1384 | + pm_system_cancel_wakeup(); | ||
1385 | acpi_os_wait_events_complete(); | ||
1386 | |||
1387 | /* | ||
1388 | diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c | ||
1389 | index 774af5ce70dad..3731066f2c1ca 100644 | ||
1390 | --- a/drivers/block/xen-blkfront.c | ||
1391 | +++ b/drivers/block/xen-blkfront.c | ||
1392 | @@ -1344,7 +1344,8 @@ free_shadow: | ||
1393 | rinfo->ring_ref[i] = GRANT_INVALID_REF; | ||
1394 | } | ||
1395 | } | ||
1396 | - free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * XEN_PAGE_SIZE)); | ||
1397 | + free_pages_exact(rinfo->ring.sring, | ||
1398 | + info->nr_ring_pages * XEN_PAGE_SIZE); | ||
1399 | rinfo->ring.sring = NULL; | ||
1400 | |||
1401 | if (rinfo->irq) | ||
1402 | @@ -1428,9 +1429,15 @@ static int blkif_get_final_status(enum blk_req_status s1, | ||
1403 | return BLKIF_RSP_OKAY; | ||
1404 | } | ||
1405 | |||
1406 | -static bool blkif_completion(unsigned long *id, | ||
1407 | - struct blkfront_ring_info *rinfo, | ||
1408 | - struct blkif_response *bret) | ||
1409 | +/* | ||
1410 | + * Return values: | ||
1411 | + * 1 response processed. | ||
1412 | + * 0 missing further responses. | ||
1413 | + * -1 error while processing. | ||
1414 | + */ | ||
1415 | +static int blkif_completion(unsigned long *id, | ||
1416 | + struct blkfront_ring_info *rinfo, | ||
1417 | + struct blkif_response *bret) | ||
1418 | { | ||
1419 | int i = 0; | ||
1420 | struct scatterlist *sg; | ||
1421 | @@ -1453,7 +1460,7 @@ static bool blkif_completion(unsigned long *id, | ||
1422 | |||
1423 | /* Wait the second response if not yet here. */ | ||
1424 | if (s2->status < REQ_DONE) | ||
1425 | - return false; | ||
1426 | + return 0; | ||
1427 | |||
1428 | bret->status = blkif_get_final_status(s->status, | ||
1429 | s2->status); | ||
1430 | @@ -1504,42 +1511,43 @@ static bool blkif_completion(unsigned long *id, | ||
1431 | } | ||
1432 | /* Add the persistent grant into the list of free grants */ | ||
1433 | for (i = 0; i < num_grant; i++) { | ||
1434 | - if (gnttab_query_foreign_access(s->grants_used[i]->gref)) { | ||
1435 | + if (!gnttab_try_end_foreign_access(s->grants_used[i]->gref)) { | ||
1436 | /* | ||
1437 | * If the grant is still mapped by the backend (the | ||
1438 | * backend has chosen to make this grant persistent) | ||
1439 | * we add it at the head of the list, so it will be | ||
1440 | * reused first. | ||
1441 | */ | ||
1442 | - if (!info->feature_persistent) | ||
1443 | - pr_alert_ratelimited("backed has not unmapped grant: %u\n", | ||
1444 | - s->grants_used[i]->gref); | ||
1445 | + if (!info->feature_persistent) { | ||
1446 | + pr_alert("backed has not unmapped grant: %u\n", | ||
1447 | + s->grants_used[i]->gref); | ||
1448 | + return -1; | ||
1449 | + } | ||
1450 | list_add(&s->grants_used[i]->node, &rinfo->grants); | ||
1451 | rinfo->persistent_gnts_c++; | ||
1452 | } else { | ||
1453 | /* | ||
1454 | - * If the grant is not mapped by the backend we end the | ||
1455 | - * foreign access and add it to the tail of the list, | ||
1456 | - * so it will not be picked again unless we run out of | ||
1457 | - * persistent grants. | ||
1458 | + * If the grant is not mapped by the backend we add it | ||
1459 | + * to the tail of the list, so it will not be picked | ||
1460 | + * again unless we run out of persistent grants. | ||
1461 | */ | ||
1462 | - gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL); | ||
1463 | s->grants_used[i]->gref = GRANT_INVALID_REF; | ||
1464 | list_add_tail(&s->grants_used[i]->node, &rinfo->grants); | ||
1465 | } | ||
1466 | } | ||
1467 | if (s->req.operation == BLKIF_OP_INDIRECT) { | ||
1468 | for (i = 0; i < INDIRECT_GREFS(num_grant); i++) { | ||
1469 | - if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) { | ||
1470 | - if (!info->feature_persistent) | ||
1471 | - pr_alert_ratelimited("backed has not unmapped grant: %u\n", | ||
1472 | - s->indirect_grants[i]->gref); | ||
1473 | + if (!gnttab_try_end_foreign_access(s->indirect_grants[i]->gref)) { | ||
1474 | + if (!info->feature_persistent) { | ||
1475 | + pr_alert("backed has not unmapped grant: %u\n", | ||
1476 | + s->indirect_grants[i]->gref); | ||
1477 | + return -1; | ||
1478 | + } | ||
1479 | list_add(&s->indirect_grants[i]->node, &rinfo->grants); | ||
1480 | rinfo->persistent_gnts_c++; | ||
1481 | } else { | ||
1482 | struct page *indirect_page; | ||
1483 | |||
1484 | - gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL); | ||
1485 | /* | ||
1486 | * Add the used indirect page back to the list of | ||
1487 | * available pages for indirect grefs. | ||
1488 | @@ -1554,7 +1562,7 @@ static bool blkif_completion(unsigned long *id, | ||
1489 | } | ||
1490 | } | ||
1491 | |||
1492 | - return true; | ||
1493 | + return 1; | ||
1494 | } | ||
1495 | |||
1496 | static irqreturn_t blkif_interrupt(int irq, void *dev_id) | ||
1497 | @@ -1620,12 +1628,17 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) | ||
1498 | } | ||
1499 | |||
1500 | if (bret.operation != BLKIF_OP_DISCARD) { | ||
1501 | + int ret; | ||
1502 | + | ||
1503 | /* | ||
1504 | * We may need to wait for an extra response if the | ||
1505 | * I/O request is split in 2 | ||
1506 | */ | ||
1507 | - if (!blkif_completion(&id, rinfo, &bret)) | ||
1508 | + ret = blkif_completion(&id, rinfo, &bret); | ||
1509 | + if (!ret) | ||
1510 | continue; | ||
1511 | + if (unlikely(ret < 0)) | ||
1512 | + goto err; | ||
1513 | } | ||
1514 | |||
1515 | if (add_id_to_freelist(rinfo, id)) { | ||
1516 | @@ -1731,8 +1744,7 @@ static int setup_blkring(struct xenbus_device *dev, | ||
1517 | for (i = 0; i < info->nr_ring_pages; i++) | ||
1518 | rinfo->ring_ref[i] = GRANT_INVALID_REF; | ||
1519 | |||
1520 | - sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH, | ||
1521 | - get_order(ring_size)); | ||
1522 | + sring = alloc_pages_exact(ring_size, GFP_NOIO); | ||
1523 | if (!sring) { | ||
1524 | xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring"); | ||
1525 | return -ENOMEM; | ||
1526 | @@ -1742,7 +1754,7 @@ static int setup_blkring(struct xenbus_device *dev, | ||
1527 | |||
1528 | err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref); | ||
1529 | if (err < 0) { | ||
1530 | - free_pages((unsigned long)sring, get_order(ring_size)); | ||
1531 | + free_pages_exact(sring, ring_size); | ||
1532 | rinfo->ring.sring = NULL; | ||
1533 | goto fail; | ||
1534 | } | ||
1535 | @@ -2720,11 +2732,10 @@ static void purge_persistent_grants(struct blkfront_info *info) | ||
1536 | list_for_each_entry_safe(gnt_list_entry, tmp, &rinfo->grants, | ||
1537 | node) { | ||
1538 | if (gnt_list_entry->gref == GRANT_INVALID_REF || | ||
1539 | - gnttab_query_foreign_access(gnt_list_entry->gref)) | ||
1540 | + !gnttab_try_end_foreign_access(gnt_list_entry->gref)) | ||
1541 | continue; | ||
1542 | |||
1543 | list_del(&gnt_list_entry->node); | ||
1544 | - gnttab_end_foreign_access(gnt_list_entry->gref, 0, 0UL); | ||
1545 | rinfo->persistent_gnts_c--; | ||
1546 | gnt_list_entry->gref = GRANT_INVALID_REF; | ||
1547 | list_add_tail(&gnt_list_entry->node, &rinfo->grants); | ||
1548 | diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c | ||
1549 | index 84f4ff351c629..eb797081d1596 100644 | ||
1550 | --- a/drivers/firmware/psci/psci.c | ||
1551 | +++ b/drivers/firmware/psci/psci.c | ||
1552 | @@ -57,6 +57,21 @@ struct psci_operations psci_ops = { | ||
1553 | .smccc_version = SMCCC_VERSION_1_0, | ||
1554 | }; | ||
1555 | |||
1556 | +enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void) | ||
1557 | +{ | ||
1558 | + if (psci_ops.smccc_version < SMCCC_VERSION_1_1) | ||
1559 | + return SMCCC_CONDUIT_NONE; | ||
1560 | + | ||
1561 | + switch (psci_ops.conduit) { | ||
1562 | + case PSCI_CONDUIT_SMC: | ||
1563 | + return SMCCC_CONDUIT_SMC; | ||
1564 | + case PSCI_CONDUIT_HVC: | ||
1565 | + return SMCCC_CONDUIT_HVC; | ||
1566 | + default: | ||
1567 | + return SMCCC_CONDUIT_NONE; | ||
1568 | + } | ||
1569 | +} | ||
1570 | + | ||
1571 | typedef unsigned long (psci_fn)(unsigned long, unsigned long, | ||
1572 | unsigned long, unsigned long); | ||
1573 | static psci_fn *invoke_psci_fn; | ||
1574 | diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c | ||
1575 | index d45d83968e769..94dd6edd18006 100644 | ||
1576 | --- a/drivers/net/xen-netfront.c | ||
1577 | +++ b/drivers/net/xen-netfront.c | ||
1578 | @@ -412,14 +412,12 @@ static bool xennet_tx_buf_gc(struct netfront_queue *queue) | ||
1579 | queue->tx_link[id] = TX_LINK_NONE; | ||
1580 | skb = queue->tx_skbs[id]; | ||
1581 | queue->tx_skbs[id] = NULL; | ||
1582 | - if (unlikely(gnttab_query_foreign_access( | ||
1583 | - queue->grant_tx_ref[id]) != 0)) { | ||
1584 | + if (unlikely(!gnttab_end_foreign_access_ref( | ||
1585 | + queue->grant_tx_ref[id], GNTMAP_readonly))) { | ||
1586 | dev_alert(dev, | ||
1587 | "Grant still in use by backend domain\n"); | ||
1588 | goto err; | ||
1589 | } | ||
1590 | - gnttab_end_foreign_access_ref( | ||
1591 | - queue->grant_tx_ref[id], GNTMAP_readonly); | ||
1592 | gnttab_release_grant_reference( | ||
1593 | &queue->gref_tx_head, queue->grant_tx_ref[id]); | ||
1594 | queue->grant_tx_ref[id] = GRANT_INVALID_REF; | ||
1595 | @@ -861,7 +859,6 @@ static int xennet_get_responses(struct netfront_queue *queue, | ||
1596 | int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD); | ||
1597 | int slots = 1; | ||
1598 | int err = 0; | ||
1599 | - unsigned long ret; | ||
1600 | |||
1601 | if (rx->flags & XEN_NETRXF_extra_info) { | ||
1602 | err = xennet_get_extras(queue, extras, rp); | ||
1603 | @@ -892,8 +889,13 @@ static int xennet_get_responses(struct netfront_queue *queue, | ||
1604 | goto next; | ||
1605 | } | ||
1606 | |||
1607 | - ret = gnttab_end_foreign_access_ref(ref, 0); | ||
1608 | - BUG_ON(!ret); | ||
1609 | + if (!gnttab_end_foreign_access_ref(ref, 0)) { | ||
1610 | + dev_alert(dev, | ||
1611 | + "Grant still in use by backend domain\n"); | ||
1612 | + queue->info->broken = true; | ||
1613 | + dev_alert(dev, "Disabled for further use\n"); | ||
1614 | + return -EINVAL; | ||
1615 | + } | ||
1616 | |||
1617 | gnttab_release_grant_reference(&queue->gref_rx_head, ref); | ||
1618 | |||
1619 | @@ -1097,6 +1099,10 @@ static int xennet_poll(struct napi_struct *napi, int budget) | ||
1620 | err = xennet_get_responses(queue, &rinfo, rp, &tmpq); | ||
1621 | |||
1622 | if (unlikely(err)) { | ||
1623 | + if (queue->info->broken) { | ||
1624 | + spin_unlock(&queue->rx_lock); | ||
1625 | + return 0; | ||
1626 | + } | ||
1627 | err: | ||
1628 | while ((skb = __skb_dequeue(&tmpq))) | ||
1629 | __skb_queue_tail(&errq, skb); | ||
1630 | @@ -1675,7 +1681,7 @@ static int setup_netfront(struct xenbus_device *dev, | ||
1631 | struct netfront_queue *queue, unsigned int feature_split_evtchn) | ||
1632 | { | ||
1633 | struct xen_netif_tx_sring *txs; | ||
1634 | - struct xen_netif_rx_sring *rxs; | ||
1635 | + struct xen_netif_rx_sring *rxs = NULL; | ||
1636 | grant_ref_t gref; | ||
1637 | int err; | ||
1638 | |||
1639 | @@ -1695,21 +1701,21 @@ static int setup_netfront(struct xenbus_device *dev, | ||
1640 | |||
1641 | err = xenbus_grant_ring(dev, txs, 1, &gref); | ||
1642 | if (err < 0) | ||
1643 | - goto grant_tx_ring_fail; | ||
1644 | + goto fail; | ||
1645 | queue->tx_ring_ref = gref; | ||
1646 | |||
1647 | rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); | ||
1648 | if (!rxs) { | ||
1649 | err = -ENOMEM; | ||
1650 | xenbus_dev_fatal(dev, err, "allocating rx ring page"); | ||
1651 | - goto alloc_rx_ring_fail; | ||
1652 | + goto fail; | ||
1653 | } | ||
1654 | SHARED_RING_INIT(rxs); | ||
1655 | FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); | ||
1656 | |||
1657 | err = xenbus_grant_ring(dev, rxs, 1, &gref); | ||
1658 | if (err < 0) | ||
1659 | - goto grant_rx_ring_fail; | ||
1660 | + goto fail; | ||
1661 | queue->rx_ring_ref = gref; | ||
1662 | |||
1663 | if (feature_split_evtchn) | ||
1664 | @@ -1722,22 +1728,28 @@ static int setup_netfront(struct xenbus_device *dev, | ||
1665 | err = setup_netfront_single(queue); | ||
1666 | |||
1667 | if (err) | ||
1668 | - goto alloc_evtchn_fail; | ||
1669 | + goto fail; | ||
1670 | |||
1671 | return 0; | ||
1672 | |||
1673 | /* If we fail to setup netfront, it is safe to just revoke access to | ||
1674 | * granted pages because backend is not accessing it at this point. | ||
1675 | */ | ||
1676 | -alloc_evtchn_fail: | ||
1677 | - gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0); | ||
1678 | -grant_rx_ring_fail: | ||
1679 | - free_page((unsigned long)rxs); | ||
1680 | -alloc_rx_ring_fail: | ||
1681 | - gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0); | ||
1682 | -grant_tx_ring_fail: | ||
1683 | - free_page((unsigned long)txs); | ||
1684 | -fail: | ||
1685 | + fail: | ||
1686 | + if (queue->rx_ring_ref != GRANT_INVALID_REF) { | ||
1687 | + gnttab_end_foreign_access(queue->rx_ring_ref, 0, | ||
1688 | + (unsigned long)rxs); | ||
1689 | + queue->rx_ring_ref = GRANT_INVALID_REF; | ||
1690 | + } else { | ||
1691 | + free_page((unsigned long)rxs); | ||
1692 | + } | ||
1693 | + if (queue->tx_ring_ref != GRANT_INVALID_REF) { | ||
1694 | + gnttab_end_foreign_access(queue->tx_ring_ref, 0, | ||
1695 | + (unsigned long)txs); | ||
1696 | + queue->tx_ring_ref = GRANT_INVALID_REF; | ||
1697 | + } else { | ||
1698 | + free_page((unsigned long)txs); | ||
1699 | + } | ||
1700 | return err; | ||
1701 | } | ||
1702 | |||
1703 | diff --git a/drivers/scsi/xen-scsifront.c b/drivers/scsi/xen-scsifront.c | ||
1704 | index f0068e96a177f..39e39869a1ad9 100644 | ||
1705 | --- a/drivers/scsi/xen-scsifront.c | ||
1706 | +++ b/drivers/scsi/xen-scsifront.c | ||
1707 | @@ -233,12 +233,11 @@ static void scsifront_gnttab_done(struct vscsifrnt_info *info, | ||
1708 | return; | ||
1709 | |||
1710 | for (i = 0; i < shadow->nr_grants; i++) { | ||
1711 | - if (unlikely(gnttab_query_foreign_access(shadow->gref[i]))) { | ||
1712 | + if (unlikely(!gnttab_try_end_foreign_access(shadow->gref[i]))) { | ||
1713 | shost_printk(KERN_ALERT, info->host, KBUILD_MODNAME | ||
1714 | "grant still in use by backend\n"); | ||
1715 | BUG(); | ||
1716 | } | ||
1717 | - gnttab_end_foreign_access(shadow->gref[i], 0, 0UL); | ||
1718 | } | ||
1719 | |||
1720 | kfree(shadow->sg); | ||
1721 | diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c | ||
1722 | index 3fa40c723e8e9..edb0acd0b8323 100644 | ||
1723 | --- a/drivers/xen/gntalloc.c | ||
1724 | +++ b/drivers/xen/gntalloc.c | ||
1725 | @@ -169,20 +169,14 @@ undo: | ||
1726 | __del_gref(gref); | ||
1727 | } | ||
1728 | |||
1729 | - /* It's possible for the target domain to map the just-allocated grant | ||
1730 | - * references by blindly guessing their IDs; if this is done, then | ||
1731 | - * __del_gref will leave them in the queue_gref list. They need to be | ||
1732 | - * added to the global list so that we can free them when they are no | ||
1733 | - * longer referenced. | ||
1734 | - */ | ||
1735 | - if (unlikely(!list_empty(&queue_gref))) | ||
1736 | - list_splice_tail(&queue_gref, &gref_list); | ||
1737 | mutex_unlock(&gref_mutex); | ||
1738 | return rc; | ||
1739 | } | ||
1740 | |||
1741 | static void __del_gref(struct gntalloc_gref *gref) | ||
1742 | { | ||
1743 | + unsigned long addr; | ||
1744 | + | ||
1745 | if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { | ||
1746 | uint8_t *tmp = kmap(gref->page); | ||
1747 | tmp[gref->notify.pgoff] = 0; | ||
1748 | @@ -196,21 +190,16 @@ static void __del_gref(struct gntalloc_gref *gref) | ||
1749 | gref->notify.flags = 0; | ||
1750 | |||
1751 | if (gref->gref_id) { | ||
1752 | - if (gnttab_query_foreign_access(gref->gref_id)) | ||
1753 | - return; | ||
1754 | - | ||
1755 | - if (!gnttab_end_foreign_access_ref(gref->gref_id, 0)) | ||
1756 | - return; | ||
1757 | - | ||
1758 | - gnttab_free_grant_reference(gref->gref_id); | ||
1759 | + if (gref->page) { | ||
1760 | + addr = (unsigned long)page_to_virt(gref->page); | ||
1761 | + gnttab_end_foreign_access(gref->gref_id, 0, addr); | ||
1762 | + } else | ||
1763 | + gnttab_free_grant_reference(gref->gref_id); | ||
1764 | } | ||
1765 | |||
1766 | gref_size--; | ||
1767 | list_del(&gref->next_gref); | ||
1768 | |||
1769 | - if (gref->page) | ||
1770 | - __free_page(gref->page); | ||
1771 | - | ||
1772 | kfree(gref); | ||
1773 | } | ||
1774 | |||
1775 | diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c | ||
1776 | index 49b381e104efa..c75dc17d1a617 100644 | ||
1777 | --- a/drivers/xen/grant-table.c | ||
1778 | +++ b/drivers/xen/grant-table.c | ||
1779 | @@ -135,12 +135,9 @@ struct gnttab_ops { | ||
1780 | */ | ||
1781 | unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref); | ||
1782 | /* | ||
1783 | - * Query the status of a grant entry. Ref parameter is reference of | ||
1784 | - * queried grant entry, return value is the status of queried entry. | ||
1785 | - * Detailed status(writing/reading) can be gotten from the return value | ||
1786 | - * by bit operations. | ||
1787 | + * Read the frame number related to a given grant reference. | ||
1788 | */ | ||
1789 | - int (*query_foreign_access)(grant_ref_t ref); | ||
1790 | + unsigned long (*read_frame)(grant_ref_t ref); | ||
1791 | }; | ||
1792 | |||
1793 | struct unmap_refs_callback_data { | ||
1794 | @@ -285,22 +282,6 @@ int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, | ||
1795 | } | ||
1796 | EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access); | ||
1797 | |||
1798 | -static int gnttab_query_foreign_access_v1(grant_ref_t ref) | ||
1799 | -{ | ||
1800 | - return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing); | ||
1801 | -} | ||
1802 | - | ||
1803 | -static int gnttab_query_foreign_access_v2(grant_ref_t ref) | ||
1804 | -{ | ||
1805 | - return grstatus[ref] & (GTF_reading|GTF_writing); | ||
1806 | -} | ||
1807 | - | ||
1808 | -int gnttab_query_foreign_access(grant_ref_t ref) | ||
1809 | -{ | ||
1810 | - return gnttab_interface->query_foreign_access(ref); | ||
1811 | -} | ||
1812 | -EXPORT_SYMBOL_GPL(gnttab_query_foreign_access); | ||
1813 | - | ||
1814 | static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly) | ||
1815 | { | ||
1816 | u16 flags, nflags; | ||
1817 | @@ -354,6 +335,16 @@ int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) | ||
1818 | } | ||
1819 | EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref); | ||
1820 | |||
1821 | +static unsigned long gnttab_read_frame_v1(grant_ref_t ref) | ||
1822 | +{ | ||
1823 | + return gnttab_shared.v1[ref].frame; | ||
1824 | +} | ||
1825 | + | ||
1826 | +static unsigned long gnttab_read_frame_v2(grant_ref_t ref) | ||
1827 | +{ | ||
1828 | + return gnttab_shared.v2[ref].full_page.frame; | ||
1829 | +} | ||
1830 | + | ||
1831 | struct deferred_entry { | ||
1832 | struct list_head list; | ||
1833 | grant_ref_t ref; | ||
1834 | @@ -383,12 +374,9 @@ static void gnttab_handle_deferred(struct timer_list *unused) | ||
1835 | spin_unlock_irqrestore(&gnttab_list_lock, flags); | ||
1836 | if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) { | ||
1837 | put_free_entry(entry->ref); | ||
1838 | - if (entry->page) { | ||
1839 | - pr_debug("freeing g.e. %#x (pfn %#lx)\n", | ||
1840 | - entry->ref, page_to_pfn(entry->page)); | ||
1841 | - put_page(entry->page); | ||
1842 | - } else | ||
1843 | - pr_info("freeing g.e. %#x\n", entry->ref); | ||
1844 | + pr_debug("freeing g.e. %#x (pfn %#lx)\n", | ||
1845 | + entry->ref, page_to_pfn(entry->page)); | ||
1846 | + put_page(entry->page); | ||
1847 | kfree(entry); | ||
1848 | entry = NULL; | ||
1849 | } else { | ||
1850 | @@ -413,9 +401,18 @@ static void gnttab_handle_deferred(struct timer_list *unused) | ||
1851 | static void gnttab_add_deferred(grant_ref_t ref, bool readonly, | ||
1852 | struct page *page) | ||
1853 | { | ||
1854 | - struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC); | ||
1855 | + struct deferred_entry *entry; | ||
1856 | + gfp_t gfp = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL; | ||
1857 | const char *what = KERN_WARNING "leaking"; | ||
1858 | |||
1859 | + entry = kmalloc(sizeof(*entry), gfp); | ||
1860 | + if (!page) { | ||
1861 | + unsigned long gfn = gnttab_interface->read_frame(ref); | ||
1862 | + | ||
1863 | + page = pfn_to_page(gfn_to_pfn(gfn)); | ||
1864 | + get_page(page); | ||
1865 | + } | ||
1866 | + | ||
1867 | if (entry) { | ||
1868 | unsigned long flags; | ||
1869 | |||
1870 | @@ -436,11 +433,21 @@ static void gnttab_add_deferred(grant_ref_t ref, bool readonly, | ||
1871 | what, ref, page ? page_to_pfn(page) : -1); | ||
1872 | } | ||
1873 | |||
1874 | +int gnttab_try_end_foreign_access(grant_ref_t ref) | ||
1875 | +{ | ||
1876 | + int ret = _gnttab_end_foreign_access_ref(ref, 0); | ||
1877 | + | ||
1878 | + if (ret) | ||
1879 | + put_free_entry(ref); | ||
1880 | + | ||
1881 | + return ret; | ||
1882 | +} | ||
1883 | +EXPORT_SYMBOL_GPL(gnttab_try_end_foreign_access); | ||
1884 | + | ||
1885 | void gnttab_end_foreign_access(grant_ref_t ref, int readonly, | ||
1886 | unsigned long page) | ||
1887 | { | ||
1888 | - if (gnttab_end_foreign_access_ref(ref, readonly)) { | ||
1889 | - put_free_entry(ref); | ||
1890 | + if (gnttab_try_end_foreign_access(ref)) { | ||
1891 | if (page != 0) | ||
1892 | put_page(virt_to_page(page)); | ||
1893 | } else | ||
1894 | @@ -1297,7 +1304,7 @@ static const struct gnttab_ops gnttab_v1_ops = { | ||
1895 | .update_entry = gnttab_update_entry_v1, | ||
1896 | .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1, | ||
1897 | .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1, | ||
1898 | - .query_foreign_access = gnttab_query_foreign_access_v1, | ||
1899 | + .read_frame = gnttab_read_frame_v1, | ||
1900 | }; | ||
1901 | |||
1902 | static const struct gnttab_ops gnttab_v2_ops = { | ||
1903 | @@ -1309,7 +1316,7 @@ static const struct gnttab_ops gnttab_v2_ops = { | ||
1904 | .update_entry = gnttab_update_entry_v2, | ||
1905 | .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2, | ||
1906 | .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2, | ||
1907 | - .query_foreign_access = gnttab_query_foreign_access_v2, | ||
1908 | + .read_frame = gnttab_read_frame_v2, | ||
1909 | }; | ||
1910 | |||
1911 | static bool gnttab_need_v2(void) | ||
1912 | diff --git a/drivers/xen/pvcalls-front.c b/drivers/xen/pvcalls-front.c | ||
1913 | index 57592a6b5c9e3..91e52e05555eb 100644 | ||
1914 | --- a/drivers/xen/pvcalls-front.c | ||
1915 | +++ b/drivers/xen/pvcalls-front.c | ||
1916 | @@ -337,8 +337,8 @@ static void free_active_ring(struct sock_mapping *map) | ||
1917 | if (!map->active.ring) | ||
1918 | return; | ||
1919 | |||
1920 | - free_pages((unsigned long)map->active.data.in, | ||
1921 | - map->active.ring->ring_order); | ||
1922 | + free_pages_exact(map->active.data.in, | ||
1923 | + PAGE_SIZE << map->active.ring->ring_order); | ||
1924 | free_page((unsigned long)map->active.ring); | ||
1925 | } | ||
1926 | |||
1927 | @@ -352,8 +352,8 @@ static int alloc_active_ring(struct sock_mapping *map) | ||
1928 | goto out; | ||
1929 | |||
1930 | map->active.ring->ring_order = PVCALLS_RING_ORDER; | ||
1931 | - bytes = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, | ||
1932 | - PVCALLS_RING_ORDER); | ||
1933 | + bytes = alloc_pages_exact(PAGE_SIZE << PVCALLS_RING_ORDER, | ||
1934 | + GFP_KERNEL | __GFP_ZERO); | ||
1935 | if (!bytes) | ||
1936 | goto out; | ||
1937 | |||
1938 | diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c | ||
1939 | index 81eddb8529ffc..8739dd0ee870d 100644 | ||
1940 | --- a/drivers/xen/xenbus/xenbus_client.c | ||
1941 | +++ b/drivers/xen/xenbus/xenbus_client.c | ||
1942 | @@ -366,7 +366,14 @@ int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr, | ||
1943 | unsigned int nr_pages, grant_ref_t *grefs) | ||
1944 | { | ||
1945 | int err; | ||
1946 | - int i, j; | ||
1947 | + unsigned int i; | ||
1948 | + grant_ref_t gref_head; | ||
1949 | + | ||
1950 | + err = gnttab_alloc_grant_references(nr_pages, &gref_head); | ||
1951 | + if (err) { | ||
1952 | + xenbus_dev_fatal(dev, err, "granting access to ring page"); | ||
1953 | + return err; | ||
1954 | + } | ||
1955 | |||
1956 | for (i = 0; i < nr_pages; i++) { | ||
1957 | unsigned long gfn; | ||
1958 | @@ -376,23 +383,14 @@ int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr, | ||
1959 | else | ||
1960 | gfn = virt_to_gfn(vaddr); | ||
1961 | |||
1962 | - err = gnttab_grant_foreign_access(dev->otherend_id, gfn, 0); | ||
1963 | - if (err < 0) { | ||
1964 | - xenbus_dev_fatal(dev, err, | ||
1965 | - "granting access to ring page"); | ||
1966 | - goto fail; | ||
1967 | - } | ||
1968 | - grefs[i] = err; | ||
1969 | + grefs[i] = gnttab_claim_grant_reference(&gref_head); | ||
1970 | + gnttab_grant_foreign_access_ref(grefs[i], dev->otherend_id, | ||
1971 | + gfn, 0); | ||
1972 | |||
1973 | vaddr = vaddr + XEN_PAGE_SIZE; | ||
1974 | } | ||
1975 | |||
1976 | return 0; | ||
1977 | - | ||
1978 | -fail: | ||
1979 | - for (j = 0; j < i; j++) | ||
1980 | - gnttab_end_foreign_access_ref(grefs[j], 0); | ||
1981 | - return err; | ||
1982 | } | ||
1983 | EXPORT_SYMBOL_GPL(xenbus_grant_ring); | ||
1984 | |||
1985 | diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h | ||
1986 | index 157e4a6a83f6d..4e97ba64dbb42 100644 | ||
1987 | --- a/include/linux/arm-smccc.h | ||
1988 | +++ b/include/linux/arm-smccc.h | ||
1989 | @@ -82,6 +82,22 @@ | ||
1990 | |||
1991 | #include <linux/linkage.h> | ||
1992 | #include <linux/types.h> | ||
1993 | + | ||
1994 | +enum arm_smccc_conduit { | ||
1995 | + SMCCC_CONDUIT_NONE, | ||
1996 | + SMCCC_CONDUIT_SMC, | ||
1997 | + SMCCC_CONDUIT_HVC, | ||
1998 | +}; | ||
1999 | + | ||
2000 | +/** | ||
2001 | + * arm_smccc_1_1_get_conduit() | ||
2002 | + * | ||
2003 | + * Returns the conduit to be used for SMCCCv1.1 or later. | ||
2004 | + * | ||
2005 | + * When SMCCCv1.1 is not present, returns SMCCC_CONDUIT_NONE. | ||
2006 | + */ | ||
2007 | +enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void); | ||
2008 | + | ||
2009 | /** | ||
2010 | * struct arm_smccc_res - Result from SMC/HVC call | ||
2011 | * @a0-a3 result values from registers 0 to 3 | ||
2012 | @@ -304,5 +320,63 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, | ||
2013 | #define SMCCC_RET_NOT_SUPPORTED -1 | ||
2014 | #define SMCCC_RET_NOT_REQUIRED -2 | ||
2015 | |||
2016 | +/* | ||
2017 | + * Like arm_smccc_1_1* but always returns SMCCC_RET_NOT_SUPPORTED. | ||
2018 | + * Used when the SMCCC conduit is not defined. The empty asm statement | ||
2019 | + * avoids compiler warnings about unused variables. | ||
2020 | + */ | ||
2021 | +#define __fail_smccc_1_1(...) \ | ||
2022 | + do { \ | ||
2023 | + __declare_args(__count_args(__VA_ARGS__), __VA_ARGS__); \ | ||
2024 | + asm ("" __constraints(__count_args(__VA_ARGS__))); \ | ||
2025 | + if (___res) \ | ||
2026 | + ___res->a0 = SMCCC_RET_NOT_SUPPORTED; \ | ||
2027 | + } while (0) | ||
2028 | + | ||
2029 | +/* | ||
2030 | + * arm_smccc_1_1_invoke() - make an SMCCC v1.1 compliant call | ||
2031 | + * | ||
2032 | + * This is a variadic macro taking one to eight source arguments, and | ||
2033 | + * an optional return structure. | ||
2034 | + * | ||
2035 | + * @a0-a7: arguments passed in registers 0 to 7 | ||
2036 | + * @res: result values from registers 0 to 3 | ||
2037 | + * | ||
2038 | + * This macro will make either an HVC call or an SMC call depending on the | ||
2039 | + * current SMCCC conduit. If no valid conduit is available then -1 | ||
2040 | + * (SMCCC_RET_NOT_SUPPORTED) is returned in @res.a0 (if supplied). | ||
2041 | + * | ||
2042 | + * The return value also provides the conduit that was used. | ||
2043 | + */ | ||
2044 | +#define arm_smccc_1_1_invoke(...) ({ \ | ||
2045 | + int method = arm_smccc_1_1_get_conduit(); \ | ||
2046 | + switch (method) { \ | ||
2047 | + case SMCCC_CONDUIT_HVC: \ | ||
2048 | + arm_smccc_1_1_hvc(__VA_ARGS__); \ | ||
2049 | + break; \ | ||
2050 | + case SMCCC_CONDUIT_SMC: \ | ||
2051 | + arm_smccc_1_1_smc(__VA_ARGS__); \ | ||
2052 | + break; \ | ||
2053 | + default: \ | ||
2054 | + __fail_smccc_1_1(__VA_ARGS__); \ | ||
2055 | + method = SMCCC_CONDUIT_NONE; \ | ||
2056 | + break; \ | ||
2057 | + } \ | ||
2058 | + method; \ | ||
2059 | + }) | ||
2060 | + | ||
2061 | +/* Paravirtualised time calls (defined by ARM DEN0057A) */ | ||
2062 | +#define ARM_SMCCC_HV_PV_TIME_FEATURES \ | ||
2063 | + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ | ||
2064 | + ARM_SMCCC_SMC_64, \ | ||
2065 | + ARM_SMCCC_OWNER_STANDARD_HYP, \ | ||
2066 | + 0x20) | ||
2067 | + | ||
2068 | +#define ARM_SMCCC_HV_PV_TIME_ST \ | ||
2069 | + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ | ||
2070 | + ARM_SMCCC_SMC_64, \ | ||
2071 | + ARM_SMCCC_OWNER_STANDARD_HYP, \ | ||
2072 | + 0x21) | ||
2073 | + | ||
2074 | #endif /*__ASSEMBLY__*/ | ||
2075 | #endif /*__LINUX_ARM_SMCCC_H*/ | ||
2076 | diff --git a/include/linux/bpf.h b/include/linux/bpf.h | ||
2077 | index 66590ae89c97c..a73ca7c9c7d0e 100644 | ||
2078 | --- a/include/linux/bpf.h | ||
2079 | +++ b/include/linux/bpf.h | ||
2080 | @@ -751,6 +751,12 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, | ||
2081 | int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, | ||
2082 | const union bpf_attr *kattr, | ||
2083 | union bpf_attr __user *uattr); | ||
2084 | + | ||
2085 | +static inline bool unprivileged_ebpf_enabled(void) | ||
2086 | +{ | ||
2087 | + return !sysctl_unprivileged_bpf_disabled; | ||
2088 | +} | ||
2089 | + | ||
2090 | #else /* !CONFIG_BPF_SYSCALL */ | ||
2091 | static inline struct bpf_prog *bpf_prog_get(u32 ufd) | ||
2092 | { | ||
2093 | @@ -881,6 +887,12 @@ static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, | ||
2094 | { | ||
2095 | return -ENOTSUPP; | ||
2096 | } | ||
2097 | + | ||
2098 | +static inline bool unprivileged_ebpf_enabled(void) | ||
2099 | +{ | ||
2100 | + return false; | ||
2101 | +} | ||
2102 | + | ||
2103 | #endif /* CONFIG_BPF_SYSCALL */ | ||
2104 | |||
2105 | static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, | ||
2106 | diff --git a/include/xen/grant_table.h b/include/xen/grant_table.h | ||
2107 | index a9978350b45b0..a58a89cc0e97d 100644 | ||
2108 | --- a/include/xen/grant_table.h | ||
2109 | +++ b/include/xen/grant_table.h | ||
2110 | @@ -97,17 +97,32 @@ int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly); | ||
2111 | * access has been ended, free the given page too. Access will be ended | ||
2112 | * immediately iff the grant entry is not in use, otherwise it will happen | ||
2113 | * some time later. page may be 0, in which case no freeing will occur. | ||
2114 | + * Note that the granted page might still be accessed (read or write) by the | ||
2115 | + * other side after gnttab_end_foreign_access() returns, so even if page was | ||
2116 | + * specified as 0 it is not allowed to just reuse the page for other | ||
2117 | + * purposes immediately. gnttab_end_foreign_access() will take an additional | ||
2118 | + * reference to the granted page in this case, which is dropped only after | ||
2119 | + * the grant is no longer in use. | ||
2120 | + * This requires that multi page allocations for areas subject to | ||
2121 | + * gnttab_end_foreign_access() are done via alloc_pages_exact() (and freeing | ||
2122 | + * via free_pages_exact()) in order to avoid high order pages. | ||
2123 | */ | ||
2124 | void gnttab_end_foreign_access(grant_ref_t ref, int readonly, | ||
2125 | unsigned long page); | ||
2126 | |||
2127 | +/* | ||
2128 | + * End access through the given grant reference, iff the grant entry is | ||
2129 | + * no longer in use. In case of success ending foreign access, the | ||
2130 | + * grant reference is deallocated. | ||
2131 | + * Return 1 if the grant entry was freed, 0 if it is still in use. | ||
2132 | + */ | ||
2133 | +int gnttab_try_end_foreign_access(grant_ref_t ref); | ||
2134 | + | ||
2135 | int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn); | ||
2136 | |||
2137 | unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref); | ||
2138 | unsigned long gnttab_end_foreign_transfer(grant_ref_t ref); | ||
2139 | |||
2140 | -int gnttab_query_foreign_access(grant_ref_t ref); | ||
2141 | - | ||
2142 | /* | ||
2143 | * operations on reserved batches of grant references | ||
2144 | */ | ||
2145 | diff --git a/kernel/sysctl.c b/kernel/sysctl.c | ||
2146 | index 8494d5a706bb5..0457d36540e38 100644 | ||
2147 | --- a/kernel/sysctl.c | ||
2148 | +++ b/kernel/sysctl.c | ||
2149 | @@ -251,6 +251,11 @@ static int sysrq_sysctl_handler(struct ctl_table *table, int write, | ||
2150 | #endif | ||
2151 | |||
2152 | #ifdef CONFIG_BPF_SYSCALL | ||
2153 | + | ||
2154 | +void __weak unpriv_ebpf_notify(int new_state) | ||
2155 | +{ | ||
2156 | +} | ||
2157 | + | ||
2158 | static int bpf_unpriv_handler(struct ctl_table *table, int write, | ||
2159 | void *buffer, size_t *lenp, loff_t *ppos) | ||
2160 | { | ||
2161 | @@ -268,6 +273,9 @@ static int bpf_unpriv_handler(struct ctl_table *table, int write, | ||
2162 | return -EPERM; | ||
2163 | *(int *)table->data = unpriv_enable; | ||
2164 | } | ||
2165 | + | ||
2166 | + unpriv_ebpf_notify(unpriv_enable); | ||
2167 | + | ||
2168 | return ret; | ||
2169 | } | ||
2170 | #endif | ||
2171 | diff --git a/net/9p/trans_xen.c b/net/9p/trans_xen.c | ||
2172 | index 44e6c74ed4288..2779ec1053a02 100644 | ||
2173 | --- a/net/9p/trans_xen.c | ||
2174 | +++ b/net/9p/trans_xen.c | ||
2175 | @@ -301,9 +301,9 @@ static void xen_9pfs_front_free(struct xen_9pfs_front_priv *priv) | ||
2176 | ref = priv->rings[i].intf->ref[j]; | ||
2177 | gnttab_end_foreign_access(ref, 0, 0); | ||
2178 | } | ||
2179 | - free_pages((unsigned long)priv->rings[i].data.in, | ||
2180 | - XEN_9PFS_RING_ORDER - | ||
2181 | - (PAGE_SHIFT - XEN_PAGE_SHIFT)); | ||
2182 | + free_pages_exact(priv->rings[i].data.in, | ||
2183 | + 1UL << (XEN_9PFS_RING_ORDER + | ||
2184 | + XEN_PAGE_SHIFT)); | ||
2185 | } | ||
2186 | gnttab_end_foreign_access(priv->rings[i].ref, 0, 0); | ||
2187 | free_page((unsigned long)priv->rings[i].intf); | ||
2188 | @@ -341,8 +341,8 @@ static int xen_9pfs_front_alloc_dataring(struct xenbus_device *dev, | ||
2189 | if (ret < 0) | ||
2190 | goto out; | ||
2191 | ring->ref = ret; | ||
2192 | - bytes = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, | ||
2193 | - XEN_9PFS_RING_ORDER - (PAGE_SHIFT - XEN_PAGE_SHIFT)); | ||
2194 | + bytes = alloc_pages_exact(1UL << (XEN_9PFS_RING_ORDER + XEN_PAGE_SHIFT), | ||
2195 | + GFP_KERNEL | __GFP_ZERO); | ||
2196 | if (!bytes) { | ||
2197 | ret = -ENOMEM; | ||
2198 | goto out; | ||
2199 | @@ -373,9 +373,7 @@ out: | ||
2200 | if (bytes) { | ||
2201 | for (i--; i >= 0; i--) | ||
2202 | gnttab_end_foreign_access(ring->intf->ref[i], 0, 0); | ||
2203 | - free_pages((unsigned long)bytes, | ||
2204 | - XEN_9PFS_RING_ORDER - | ||
2205 | - (PAGE_SHIFT - XEN_PAGE_SHIFT)); | ||
2206 | + free_pages_exact(bytes, 1UL << (XEN_9PFS_RING_ORDER + XEN_PAGE_SHIFT)); | ||
2207 | } | ||
2208 | gnttab_end_foreign_access(ring->ref, 0, 0); | ||
2209 | free_page((unsigned long)ring->intf); | ||
2210 | diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h | ||
2211 | index 0652d3eed9bda..4133c721af6ed 100644 | ||
2212 | --- a/tools/arch/x86/include/asm/cpufeatures.h | ||
2213 | +++ b/tools/arch/x86/include/asm/cpufeatures.h | ||
2214 | @@ -202,7 +202,7 @@ | ||
2215 | #define X86_FEATURE_SME ( 7*32+10) /* AMD Secure Memory Encryption */ | ||
2216 | #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */ | ||
2217 | #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ | ||
2218 | -#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */ | ||
2219 | +#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCEs for Spectre variant 2 */ | ||
2220 | #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ | ||
2221 | #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */ | ||
2222 | #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ |