Contents of /trunk/kernel-alx-legacy/patches-4.9/0405-4.9.306-all-fixes.patch
Parent Directory | Revision Log
Revision 3707 -
(show annotations)
(download)
Mon Oct 24 14:08:20 2022 UTC (18 months, 3 weeks ago) by niro
File size: 102080 byte(s)
Mon Oct 24 14:08:20 2022 UTC (18 months, 3 weeks ago) by niro
File size: 102080 byte(s)
-linux-4.9.306
1 | diff --git a/Documentation/hw-vuln/index.rst b/Documentation/hw-vuln/index.rst |
2 | index b5fbc6ae9d5fd..74466ba801678 100644 |
3 | --- a/Documentation/hw-vuln/index.rst |
4 | +++ b/Documentation/hw-vuln/index.rst |
5 | @@ -9,6 +9,7 @@ are configurable at compile, boot or run time. |
6 | .. toctree:: |
7 | :maxdepth: 1 |
8 | |
9 | + spectre |
10 | l1tf |
11 | mds |
12 | tsx_async_abort |
13 | diff --git a/Documentation/hw-vuln/spectre.rst b/Documentation/hw-vuln/spectre.rst |
14 | new file mode 100644 |
15 | index 0000000000000..c6c43ac2ba43d |
16 | --- /dev/null |
17 | +++ b/Documentation/hw-vuln/spectre.rst |
18 | @@ -0,0 +1,785 @@ |
19 | +.. SPDX-License-Identifier: GPL-2.0 |
20 | + |
21 | +Spectre Side Channels |
22 | +===================== |
23 | + |
24 | +Spectre is a class of side channel attacks that exploit branch prediction |
25 | +and speculative execution on modern CPUs to read memory, possibly |
26 | +bypassing access controls. Speculative execution side channel exploits |
27 | +do not modify memory but attempt to infer privileged data in the memory. |
28 | + |
29 | +This document covers Spectre variant 1 and Spectre variant 2. |
30 | + |
31 | +Affected processors |
32 | +------------------- |
33 | + |
34 | +Speculative execution side channel methods affect a wide range of modern |
35 | +high performance processors, since most modern high speed processors |
36 | +use branch prediction and speculative execution. |
37 | + |
38 | +The following CPUs are vulnerable: |
39 | + |
40 | + - Intel Core, Atom, Pentium, and Xeon processors |
41 | + |
42 | + - AMD Phenom, EPYC, and Zen processors |
43 | + |
44 | + - IBM POWER and zSeries processors |
45 | + |
46 | + - Higher end ARM processors |
47 | + |
48 | + - Apple CPUs |
49 | + |
50 | + - Higher end MIPS CPUs |
51 | + |
52 | + - Likely most other high performance CPUs. Contact your CPU vendor for details. |
53 | + |
54 | +Whether a processor is affected or not can be read out from the Spectre |
55 | +vulnerability files in sysfs. See :ref:`spectre_sys_info`. |
56 | + |
57 | +Related CVEs |
58 | +------------ |
59 | + |
60 | +The following CVE entries describe Spectre variants: |
61 | + |
62 | + ============= ======================= ========================== |
63 | + CVE-2017-5753 Bounds check bypass Spectre variant 1 |
64 | + CVE-2017-5715 Branch target injection Spectre variant 2 |
65 | + CVE-2019-1125 Spectre v1 swapgs Spectre variant 1 (swapgs) |
66 | + ============= ======================= ========================== |
67 | + |
68 | +Problem |
69 | +------- |
70 | + |
71 | +CPUs use speculative operations to improve performance. That may leave |
72 | +traces of memory accesses or computations in the processor's caches, |
73 | +buffers, and branch predictors. Malicious software may be able to |
74 | +influence the speculative execution paths, and then use the side effects |
75 | +of the speculative execution in the CPUs' caches and buffers to infer |
76 | +privileged data touched during the speculative execution. |
77 | + |
78 | +Spectre variant 1 attacks take advantage of speculative execution of |
79 | +conditional branches, while Spectre variant 2 attacks use speculative |
80 | +execution of indirect branches to leak privileged memory. |
81 | +See :ref:`[1] <spec_ref1>` :ref:`[5] <spec_ref5>` :ref:`[6] <spec_ref6>` |
82 | +:ref:`[7] <spec_ref7>` :ref:`[10] <spec_ref10>` :ref:`[11] <spec_ref11>`. |
83 | + |
84 | +Spectre variant 1 (Bounds Check Bypass) |
85 | +--------------------------------------- |
86 | + |
87 | +The bounds check bypass attack :ref:`[2] <spec_ref2>` takes advantage |
88 | +of speculative execution that bypasses conditional branch instructions |
89 | +used for memory access bounds check (e.g. checking if the index of an |
90 | +array results in memory access within a valid range). This results in |
91 | +memory accesses to invalid memory (with out-of-bound index) that are |
92 | +done speculatively before validation checks resolve. Such speculative |
93 | +memory accesses can leave side effects, creating side channels which |
94 | +leak information to the attacker. |
95 | + |
96 | +There are some extensions of Spectre variant 1 attacks for reading data |
97 | +over the network, see :ref:`[12] <spec_ref12>`. However such attacks |
98 | +are difficult, low bandwidth, fragile, and are considered low risk. |
99 | + |
100 | +Note that, despite "Bounds Check Bypass" name, Spectre variant 1 is not |
101 | +only about user-controlled array bounds checks. It can affect any |
102 | +conditional checks. The kernel entry code interrupt, exception, and NMI |
103 | +handlers all have conditional swapgs checks. Those may be problematic |
104 | +in the context of Spectre v1, as kernel code can speculatively run with |
105 | +a user GS. |
106 | + |
107 | +Spectre variant 2 (Branch Target Injection) |
108 | +------------------------------------------- |
109 | + |
110 | +The branch target injection attack takes advantage of speculative |
111 | +execution of indirect branches :ref:`[3] <spec_ref3>`. The indirect |
112 | +branch predictors inside the processor used to guess the target of |
113 | +indirect branches can be influenced by an attacker, causing gadget code |
114 | +to be speculatively executed, thus exposing sensitive data touched by |
115 | +the victim. The side effects left in the CPU's caches during speculative |
116 | +execution can be measured to infer data values. |
117 | + |
118 | +.. _poison_btb: |
119 | + |
120 | +In Spectre variant 2 attacks, the attacker can steer speculative indirect |
121 | +branches in the victim to gadget code by poisoning the branch target |
122 | +buffer of a CPU used for predicting indirect branch addresses. Such |
123 | +poisoning could be done by indirect branching into existing code, |
124 | +with the address offset of the indirect branch under the attacker's |
125 | +control. Since the branch prediction on impacted hardware does not |
126 | +fully disambiguate branch address and uses the offset for prediction, |
127 | +this could cause privileged code's indirect branch to jump to a gadget |
128 | +code with the same offset. |
129 | + |
130 | +The most useful gadgets take an attacker-controlled input parameter (such |
131 | +as a register value) so that the memory read can be controlled. Gadgets |
132 | +without input parameters might be possible, but the attacker would have |
133 | +very little control over what memory can be read, reducing the risk of |
134 | +the attack revealing useful data. |
135 | + |
136 | +One other variant 2 attack vector is for the attacker to poison the |
137 | +return stack buffer (RSB) :ref:`[13] <spec_ref13>` to cause speculative |
138 | +subroutine return instruction execution to go to a gadget. An attacker's |
139 | +imbalanced subroutine call instructions might "poison" entries in the |
140 | +return stack buffer which are later consumed by a victim's subroutine |
141 | +return instructions. This attack can be mitigated by flushing the return |
142 | +stack buffer on context switch, or virtual machine (VM) exit. |
143 | + |
144 | +On systems with simultaneous multi-threading (SMT), attacks are possible |
145 | +from the sibling thread, as level 1 cache and branch target buffer |
146 | +(BTB) may be shared between hardware threads in a CPU core. A malicious |
147 | +program running on the sibling thread may influence its peer's BTB to |
148 | +steer its indirect branch speculations to gadget code, and measure the |
149 | +speculative execution's side effects left in level 1 cache to infer the |
150 | +victim's data. |
151 | + |
152 | +Yet another variant 2 attack vector is for the attacker to poison the |
153 | +Branch History Buffer (BHB) to speculatively steer an indirect branch |
154 | +to a specific Branch Target Buffer (BTB) entry, even if the entry isn't |
155 | +associated with the source address of the indirect branch. Specifically, |
156 | +the BHB might be shared across privilege levels even in the presence of |
157 | +Enhanced IBRS. |
158 | + |
159 | +Currently the only known real-world BHB attack vector is via |
160 | +unprivileged eBPF. Therefore, it's highly recommended to not enable |
161 | +unprivileged eBPF, especially when eIBRS is used (without retpolines). |
162 | +For a full mitigation against BHB attacks, it's recommended to use |
163 | +retpolines (or eIBRS combined with retpolines). |
164 | + |
165 | +Attack scenarios |
166 | +---------------- |
167 | + |
168 | +The following list of attack scenarios have been anticipated, but may |
169 | +not cover all possible attack vectors. |
170 | + |
171 | +1. A user process attacking the kernel |
172 | +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
173 | + |
174 | +Spectre variant 1 |
175 | +~~~~~~~~~~~~~~~~~ |
176 | + |
177 | + The attacker passes a parameter to the kernel via a register or |
178 | + via a known address in memory during a syscall. Such parameter may |
179 | + be used later by the kernel as an index to an array or to derive |
180 | + a pointer for a Spectre variant 1 attack. The index or pointer |
181 | + is invalid, but bound checks are bypassed in the code branch taken |
182 | + for speculative execution. This could cause privileged memory to be |
183 | + accessed and leaked. |
184 | + |
185 | + For kernel code that has been identified where data pointers could |
186 | + potentially be influenced for Spectre attacks, new "nospec" accessor |
187 | + macros are used to prevent speculative loading of data. |
188 | + |
189 | +Spectre variant 1 (swapgs) |
190 | +~~~~~~~~~~~~~~~~~~~~~~~~~~ |
191 | + |
192 | + An attacker can train the branch predictor to speculatively skip the |
193 | + swapgs path for an interrupt or exception. If they initialize |
194 | + the GS register to a user-space value, if the swapgs is speculatively |
195 | + skipped, subsequent GS-related percpu accesses in the speculation |
196 | + window will be done with the attacker-controlled GS value. This |
197 | + could cause privileged memory to be accessed and leaked. |
198 | + |
199 | + For example: |
200 | + |
201 | + :: |
202 | + |
203 | + if (coming from user space) |
204 | + swapgs |
205 | + mov %gs:<percpu_offset>, %reg |
206 | + mov (%reg), %reg1 |
207 | + |
208 | + When coming from user space, the CPU can speculatively skip the |
209 | + swapgs, and then do a speculative percpu load using the user GS |
210 | + value. So the user can speculatively force a read of any kernel |
211 | + value. If a gadget exists which uses the percpu value as an address |
212 | + in another load/store, then the contents of the kernel value may |
213 | + become visible via an L1 side channel attack. |
214 | + |
215 | + A similar attack exists when coming from kernel space. The CPU can |
216 | + speculatively do the swapgs, causing the user GS to get used for the |
217 | + rest of the speculative window. |
218 | + |
219 | +Spectre variant 2 |
220 | +~~~~~~~~~~~~~~~~~ |
221 | + |
222 | + A spectre variant 2 attacker can :ref:`poison <poison_btb>` the branch |
223 | + target buffer (BTB) before issuing syscall to launch an attack. |
224 | + After entering the kernel, the kernel could use the poisoned branch |
225 | + target buffer on indirect jump and jump to gadget code in speculative |
226 | + execution. |
227 | + |
228 | + If an attacker tries to control the memory addresses leaked during |
229 | + speculative execution, he would also need to pass a parameter to the |
230 | + gadget, either through a register or a known address in memory. After |
231 | + the gadget has executed, he can measure the side effect. |
232 | + |
233 | + The kernel can protect itself against consuming poisoned branch |
234 | + target buffer entries by using return trampolines (also known as |
235 | + "retpoline") :ref:`[3] <spec_ref3>` :ref:`[9] <spec_ref9>` for all |
236 | + indirect branches. Return trampolines trap speculative execution paths |
237 | + to prevent jumping to gadget code during speculative execution. |
238 | + x86 CPUs with Enhanced Indirect Branch Restricted Speculation |
239 | + (Enhanced IBRS) available in hardware should use the feature to |
240 | + mitigate Spectre variant 2 instead of retpoline. Enhanced IBRS is |
241 | + more efficient than retpoline. |
242 | + |
243 | + There may be gadget code in firmware which could be exploited with |
244 | + Spectre variant 2 attack by a rogue user process. To mitigate such |
245 | + attacks on x86, Indirect Branch Restricted Speculation (IBRS) feature |
246 | + is turned on before the kernel invokes any firmware code. |
247 | + |
248 | +2. A user process attacking another user process |
249 | +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
250 | + |
251 | + A malicious user process can try to attack another user process, |
252 | + either via a context switch on the same hardware thread, or from the |
253 | + sibling hyperthread sharing a physical processor core on simultaneous |
254 | + multi-threading (SMT) system. |
255 | + |
256 | + Spectre variant 1 attacks generally require passing parameters |
257 | + between the processes, which needs a data passing relationship, such |
258 | + as remote procedure calls (RPC). Those parameters are used in gadget |
259 | + code to derive invalid data pointers accessing privileged memory in |
260 | + the attacked process. |
261 | + |
262 | + Spectre variant 2 attacks can be launched from a rogue process by |
263 | + :ref:`poisoning <poison_btb>` the branch target buffer. This can |
264 | + influence the indirect branch targets for a victim process that either |
265 | + runs later on the same hardware thread, or running concurrently on |
266 | + a sibling hardware thread sharing the same physical core. |
267 | + |
268 | + A user process can protect itself against Spectre variant 2 attacks |
269 | + by using the prctl() syscall to disable indirect branch speculation |
270 | + for itself. An administrator can also cordon off an unsafe process |
271 | + from polluting the branch target buffer by disabling the process's |
272 | + indirect branch speculation. This comes with a performance cost |
273 | + from not using indirect branch speculation and clearing the branch |
274 | + target buffer. When SMT is enabled on x86, for a process that has |
275 | + indirect branch speculation disabled, Single Threaded Indirect Branch |
276 | + Predictors (STIBP) :ref:`[4] <spec_ref4>` are turned on to prevent the |
277 | + sibling thread from controlling branch target buffer. In addition, |
278 | + the Indirect Branch Prediction Barrier (IBPB) is issued to clear the |
279 | + branch target buffer when context switching to and from such process. |
280 | + |
281 | + On x86, the return stack buffer is stuffed on context switch. |
282 | + This prevents the branch target buffer from being used for branch |
283 | + prediction when the return stack buffer underflows while switching to |
284 | + a deeper call stack. Any poisoned entries in the return stack buffer |
285 | + left by the previous process will also be cleared. |
286 | + |
287 | + User programs should use address space randomization to make attacks |
288 | + more difficult (Set /proc/sys/kernel/randomize_va_space = 1 or 2). |
289 | + |
290 | +3. A virtualized guest attacking the host |
291 | +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
292 | + |
293 | + The attack mechanism is similar to how user processes attack the |
294 | + kernel. The kernel is entered via hyper-calls or other virtualization |
295 | + exit paths. |
296 | + |
297 | + For Spectre variant 1 attacks, rogue guests can pass parameters |
298 | + (e.g. in registers) via hyper-calls to derive invalid pointers to |
299 | + speculate into privileged memory after entering the kernel. For places |
300 | + where such kernel code has been identified, nospec accessor macros |
301 | + are used to stop speculative memory access. |
302 | + |
303 | + For Spectre variant 2 attacks, rogue guests can :ref:`poison |
304 | + <poison_btb>` the branch target buffer or return stack buffer, causing |
305 | + the kernel to jump to gadget code in the speculative execution paths. |
306 | + |
307 | + To mitigate variant 2, the host kernel can use return trampolines |
308 | + for indirect branches to bypass the poisoned branch target buffer, |
309 | + and flushing the return stack buffer on VM exit. This prevents rogue |
310 | + guests from affecting indirect branching in the host kernel. |
311 | + |
312 | + To protect host processes from rogue guests, host processes can have |
313 | + indirect branch speculation disabled via prctl(). The branch target |
314 | + buffer is cleared before context switching to such processes. |
315 | + |
316 | +4. A virtualized guest attacking other guest |
317 | +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
318 | + |
319 | + A rogue guest may attack another guest to get data accessible by the |
320 | + other guest. |
321 | + |
322 | + Spectre variant 1 attacks are possible if parameters can be passed |
323 | + between guests. This may be done via mechanisms such as shared memory |
324 | + or message passing. Such parameters could be used to derive data |
325 | + pointers to privileged data in guest. The privileged data could be |
326 | + accessed by gadget code in the victim's speculation paths. |
327 | + |
328 | + Spectre variant 2 attacks can be launched from a rogue guest by |
329 | + :ref:`poisoning <poison_btb>` the branch target buffer or the return |
330 | + stack buffer. Such poisoned entries could be used to influence |
331 | + speculation execution paths in the victim guest. |
332 | + |
333 | + Linux kernel mitigates attacks to other guests running in the same |
334 | + CPU hardware thread by flushing the return stack buffer on VM exit, |
335 | + and clearing the branch target buffer before switching to a new guest. |
336 | + |
337 | + If SMT is used, Spectre variant 2 attacks from an untrusted guest |
338 | + in the sibling hyperthread can be mitigated by the administrator, |
339 | + by turning off the unsafe guest's indirect branch speculation via |
340 | + prctl(). A guest can also protect itself by turning on microcode |
341 | + based mitigations (such as IBPB or STIBP on x86) within the guest. |
342 | + |
343 | +.. _spectre_sys_info: |
344 | + |
345 | +Spectre system information |
346 | +-------------------------- |
347 | + |
348 | +The Linux kernel provides a sysfs interface to enumerate the current |
349 | +mitigation status of the system for Spectre: whether the system is |
350 | +vulnerable, and which mitigations are active. |
351 | + |
352 | +The sysfs file showing Spectre variant 1 mitigation status is: |
353 | + |
354 | + /sys/devices/system/cpu/vulnerabilities/spectre_v1 |
355 | + |
356 | +The possible values in this file are: |
357 | + |
358 | + .. list-table:: |
359 | + |
360 | + * - 'Not affected' |
361 | + - The processor is not vulnerable. |
362 | + * - 'Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers' |
363 | + - The swapgs protections are disabled; otherwise it has |
364 | + protection in the kernel on a case by case base with explicit |
365 | + pointer sanitation and usercopy LFENCE barriers. |
366 | + * - 'Mitigation: usercopy/swapgs barriers and __user pointer sanitization' |
367 | + - Protection in the kernel on a case by case base with explicit |
368 | + pointer sanitation, usercopy LFENCE barriers, and swapgs LFENCE |
369 | + barriers. |
370 | + |
371 | +However, the protections are put in place on a case by case basis, |
372 | +and there is no guarantee that all possible attack vectors for Spectre |
373 | +variant 1 are covered. |
374 | + |
375 | +The spectre_v2 kernel file reports if the kernel has been compiled with |
376 | +retpoline mitigation or if the CPU has hardware mitigation, and if the |
377 | +CPU has support for additional process-specific mitigation. |
378 | + |
379 | +This file also reports CPU features enabled by microcode to mitigate |
380 | +attack between user processes: |
381 | + |
382 | +1. Indirect Branch Prediction Barrier (IBPB) to add additional |
383 | + isolation between processes of different users. |
384 | +2. Single Thread Indirect Branch Predictors (STIBP) to add additional |
385 | + isolation between CPU threads running on the same core. |
386 | + |
387 | +These CPU features may impact performance when used and can be enabled |
388 | +per process on a case-by-case base. |
389 | + |
390 | +The sysfs file showing Spectre variant 2 mitigation status is: |
391 | + |
392 | + /sys/devices/system/cpu/vulnerabilities/spectre_v2 |
393 | + |
394 | +The possible values in this file are: |
395 | + |
396 | + - Kernel status: |
397 | + |
398 | + ======================================== ================================= |
399 | + 'Not affected' The processor is not vulnerable |
400 | + 'Mitigation: None' Vulnerable, no mitigation |
401 | + 'Mitigation: Retpolines' Use Retpoline thunks |
402 | + 'Mitigation: LFENCE' Use LFENCE instructions |
403 | + 'Mitigation: Enhanced IBRS' Hardware-focused mitigation |
404 | + 'Mitigation: Enhanced IBRS + Retpolines' Hardware-focused + Retpolines |
405 | + 'Mitigation: Enhanced IBRS + LFENCE' Hardware-focused + LFENCE |
406 | + ======================================== ================================= |
407 | + |
408 | + - Firmware status: Show if Indirect Branch Restricted Speculation (IBRS) is |
409 | + used to protect against Spectre variant 2 attacks when calling firmware (x86 only). |
410 | + |
411 | + ========== ============================================================= |
412 | + 'IBRS_FW' Protection against user program attacks when calling firmware |
413 | + ========== ============================================================= |
414 | + |
415 | + - Indirect branch prediction barrier (IBPB) status for protection between |
416 | + processes of different users. This feature can be controlled through |
417 | + prctl() per process, or through kernel command line options. This is |
418 | + an x86 only feature. For more details see below. |
419 | + |
420 | + =================== ======================================================== |
421 | + 'IBPB: disabled' IBPB unused |
422 | + 'IBPB: always-on' Use IBPB on all tasks |
423 | + 'IBPB: conditional' Use IBPB on SECCOMP or indirect branch restricted tasks |
424 | + =================== ======================================================== |
425 | + |
426 | + - Single threaded indirect branch prediction (STIBP) status for protection |
427 | + between different hyper threads. This feature can be controlled through |
428 | + prctl per process, or through kernel command line options. This is x86 |
429 | + only feature. For more details see below. |
430 | + |
431 | + ==================== ======================================================== |
432 | + 'STIBP: disabled' STIBP unused |
433 | + 'STIBP: forced' Use STIBP on all tasks |
434 | + 'STIBP: conditional' Use STIBP on SECCOMP or indirect branch restricted tasks |
435 | + ==================== ======================================================== |
436 | + |
437 | + - Return stack buffer (RSB) protection status: |
438 | + |
439 | + ============= =========================================== |
440 | + 'RSB filling' Protection of RSB on context switch enabled |
441 | + ============= =========================================== |
442 | + |
443 | +Full mitigation might require a microcode update from the CPU |
444 | +vendor. When the necessary microcode is not available, the kernel will |
445 | +report vulnerability. |
446 | + |
447 | +Turning on mitigation for Spectre variant 1 and Spectre variant 2 |
448 | +----------------------------------------------------------------- |
449 | + |
450 | +1. Kernel mitigation |
451 | +^^^^^^^^^^^^^^^^^^^^ |
452 | + |
453 | +Spectre variant 1 |
454 | +~~~~~~~~~~~~~~~~~ |
455 | + |
456 | + For the Spectre variant 1, vulnerable kernel code (as determined |
457 | + by code audit or scanning tools) is annotated on a case by case |
458 | + basis to use nospec accessor macros for bounds clipping :ref:`[2] |
459 | + <spec_ref2>` to avoid any usable disclosure gadgets. However, it may |
460 | + not cover all attack vectors for Spectre variant 1. |
461 | + |
462 | + Copy-from-user code has an LFENCE barrier to prevent the access_ok() |
463 | + check from being mis-speculated. The barrier is done by the |
464 | + barrier_nospec() macro. |
465 | + |
466 | + For the swapgs variant of Spectre variant 1, LFENCE barriers are |
467 | + added to interrupt, exception and NMI entry where needed. These |
468 | + barriers are done by the FENCE_SWAPGS_KERNEL_ENTRY and |
469 | + FENCE_SWAPGS_USER_ENTRY macros. |
470 | + |
471 | +Spectre variant 2 |
472 | +~~~~~~~~~~~~~~~~~ |
473 | + |
474 | + For Spectre variant 2 mitigation, the compiler turns indirect calls or |
475 | + jumps in the kernel into equivalent return trampolines (retpolines) |
476 | + :ref:`[3] <spec_ref3>` :ref:`[9] <spec_ref9>` to go to the target |
477 | + addresses. Speculative execution paths under retpolines are trapped |
478 | + in an infinite loop to prevent any speculative execution jumping to |
479 | + a gadget. |
480 | + |
481 | + To turn on retpoline mitigation on a vulnerable CPU, the kernel |
482 | + needs to be compiled with a gcc compiler that supports the |
483 | + -mindirect-branch=thunk-extern -mindirect-branch-register options. |
484 | + If the kernel is compiled with a Clang compiler, the compiler needs |
485 | + to support -mretpoline-external-thunk option. The kernel config |
486 | + CONFIG_RETPOLINE needs to be turned on, and the CPU needs to run with |
487 | + the latest updated microcode. |
488 | + |
489 | + On Intel Skylake-era systems the mitigation covers most, but not all, |
490 | + cases. See :ref:`[3] <spec_ref3>` for more details. |
491 | + |
492 | + On CPUs with hardware mitigation for Spectre variant 2 (e.g. Enhanced |
493 | + IBRS on x86), retpoline is automatically disabled at run time. |
494 | + |
495 | + The retpoline mitigation is turned on by default on vulnerable |
496 | + CPUs. It can be forced on or off by the administrator |
497 | + via the kernel command line and sysfs control files. See |
498 | + :ref:`spectre_mitigation_control_command_line`. |
499 | + |
500 | + On x86, indirect branch restricted speculation is turned on by default |
501 | + before invoking any firmware code to prevent Spectre variant 2 exploits |
502 | + using the firmware. |
503 | + |
504 | + Using kernel address space randomization (CONFIG_RANDOMIZE_BASE=y |
505 | + and CONFIG_SLAB_FREELIST_RANDOM=y in the kernel configuration) makes |
506 | + attacks on the kernel generally more difficult. |
507 | + |
508 | +2. User program mitigation |
509 | +^^^^^^^^^^^^^^^^^^^^^^^^^^ |
510 | + |
511 | + User programs can mitigate Spectre variant 1 using LFENCE or "bounds |
512 | + clipping". For more details see :ref:`[2] <spec_ref2>`. |
513 | + |
514 | + For Spectre variant 2 mitigation, individual user programs |
515 | + can be compiled with return trampolines for indirect branches. |
516 | + This protects them from consuming poisoned entries in the branch |
517 | + target buffer left by malicious software. Alternatively, the |
518 | + programs can disable their indirect branch speculation via prctl() |
519 | + (See Documentation/spec_ctrl.txt). |
520 | + On x86, this will turn on STIBP to guard against attacks from the |
521 | + sibling thread when the user program is running, and use IBPB to |
522 | + flush the branch target buffer when switching to/from the program. |
523 | + |
524 | + Restricting indirect branch speculation on a user program will |
525 | + also prevent the program from launching a variant 2 attack |
526 | + on x86. All sand-boxed SECCOMP programs have indirect branch |
527 | + speculation restricted by default. Administrators can change |
528 | + that behavior via the kernel command line and sysfs control files. |
529 | + See :ref:`spectre_mitigation_control_command_line`. |
530 | + |
531 | + Programs that disable their indirect branch speculation will have |
532 | + more overhead and run slower. |
533 | + |
534 | + User programs should use address space randomization |
535 | + (/proc/sys/kernel/randomize_va_space = 1 or 2) to make attacks more |
536 | + difficult. |
537 | + |
538 | +3. VM mitigation |
539 | +^^^^^^^^^^^^^^^^ |
540 | + |
541 | + Within the kernel, Spectre variant 1 attacks from rogue guests are |
542 | + mitigated on a case by case basis in VM exit paths. Vulnerable code |
543 | + uses nospec accessor macros for "bounds clipping", to avoid any |
544 | + usable disclosure gadgets. However, this may not cover all variant |
545 | + 1 attack vectors. |
546 | + |
547 | + For Spectre variant 2 attacks from rogue guests to the kernel, the |
548 | + Linux kernel uses retpoline or Enhanced IBRS to prevent consumption of |
549 | + poisoned entries in branch target buffer left by rogue guests. It also |
550 | + flushes the return stack buffer on every VM exit to prevent a return |
551 | + stack buffer underflow so poisoned branch target buffer could be used, |
552 | + or attacker guests leaving poisoned entries in the return stack buffer. |
553 | + |
554 | + To mitigate guest-to-guest attacks in the same CPU hardware thread, |
555 | + the branch target buffer is sanitized by flushing before switching |
556 | + to a new guest on a CPU. |
557 | + |
558 | + The above mitigations are turned on by default on vulnerable CPUs. |
559 | + |
560 | + To mitigate guest-to-guest attacks from sibling thread when SMT is |
561 | + in use, an untrusted guest running in the sibling thread can have |
562 | + its indirect branch speculation disabled by administrator via prctl(). |
563 | + |
564 | + The kernel also allows guests to use any microcode based mitigation |
565 | + they choose to use (such as IBPB or STIBP on x86) to protect themselves. |
566 | + |
567 | +.. _spectre_mitigation_control_command_line: |
568 | + |
569 | +Mitigation control on the kernel command line |
570 | +--------------------------------------------- |
571 | + |
572 | +Spectre variant 2 mitigation can be disabled or force enabled at the |
573 | +kernel command line. |
574 | + |
575 | + nospectre_v1 |
576 | + |
577 | + [X86,PPC] Disable mitigations for Spectre Variant 1 |
578 | + (bounds check bypass). With this option data leaks are |
579 | + possible in the system. |
580 | + |
581 | + nospectre_v2 |
582 | + |
583 | + [X86] Disable all mitigations for the Spectre variant 2 |
584 | + (indirect branch prediction) vulnerability. System may |
585 | + allow data leaks with this option, which is equivalent |
586 | + to spectre_v2=off. |
587 | + |
588 | + |
589 | + spectre_v2= |
590 | + |
591 | + [X86] Control mitigation of Spectre variant 2 |
592 | + (indirect branch speculation) vulnerability. |
593 | + The default operation protects the kernel from |
594 | + user space attacks. |
595 | + |
596 | + on |
597 | + unconditionally enable, implies |
598 | + spectre_v2_user=on |
599 | + off |
600 | + unconditionally disable, implies |
601 | + spectre_v2_user=off |
602 | + auto |
603 | + kernel detects whether your CPU model is |
604 | + vulnerable |
605 | + |
606 | + Selecting 'on' will, and 'auto' may, choose a |
607 | + mitigation method at run time according to the |
608 | + CPU, the available microcode, the setting of the |
609 | + CONFIG_RETPOLINE configuration option, and the |
610 | + compiler with which the kernel was built. |
611 | + |
612 | + Selecting 'on' will also enable the mitigation |
613 | + against user space to user space task attacks. |
614 | + |
615 | + Selecting 'off' will disable both the kernel and |
616 | + the user space protections. |
617 | + |
618 | + Specific mitigations can also be selected manually: |
619 | + |
620 | + retpoline auto pick between generic,lfence |
621 | + retpoline,generic Retpolines |
622 | + retpoline,lfence LFENCE; indirect branch |
623 | + retpoline,amd alias for retpoline,lfence |
624 | + eibrs enhanced IBRS |
625 | + eibrs,retpoline enhanced IBRS + Retpolines |
626 | + eibrs,lfence enhanced IBRS + LFENCE |
627 | + |
628 | + Not specifying this option is equivalent to |
629 | + spectre_v2=auto. |
630 | + |
631 | +For user space mitigation: |
632 | + |
633 | + spectre_v2_user= |
634 | + |
635 | + [X86] Control mitigation of Spectre variant 2 |
636 | + (indirect branch speculation) vulnerability between |
637 | + user space tasks |
638 | + |
639 | + on |
640 | + Unconditionally enable mitigations. Is |
641 | + enforced by spectre_v2=on |
642 | + |
643 | + off |
644 | + Unconditionally disable mitigations. Is |
645 | + enforced by spectre_v2=off |
646 | + |
647 | + prctl |
648 | + Indirect branch speculation is enabled, |
649 | + but mitigation can be enabled via prctl |
650 | + per thread. The mitigation control state |
651 | + is inherited on fork. |
652 | + |
653 | + prctl,ibpb |
654 | + Like "prctl" above, but only STIBP is |
655 | + controlled per thread. IBPB is issued |
656 | + always when switching between different user |
657 | + space processes. |
658 | + |
659 | + seccomp |
660 | + Same as "prctl" above, but all seccomp |
661 | + threads will enable the mitigation unless |
662 | + they explicitly opt out. |
663 | + |
664 | + seccomp,ibpb |
665 | + Like "seccomp" above, but only STIBP is |
666 | + controlled per thread. IBPB is issued |
667 | + always when switching between different |
668 | + user space processes. |
669 | + |
670 | + auto |
671 | + Kernel selects the mitigation depending on |
672 | + the available CPU features and vulnerability. |
673 | + |
674 | + Default mitigation: |
675 | + If CONFIG_SECCOMP=y then "seccomp", otherwise "prctl" |
676 | + |
677 | + Not specifying this option is equivalent to |
678 | + spectre_v2_user=auto. |
679 | + |
680 | + In general the kernel by default selects |
681 | + reasonable mitigations for the current CPU. To |
682 | + disable Spectre variant 2 mitigations, boot with |
683 | + spectre_v2=off. Spectre variant 1 mitigations |
684 | + cannot be disabled. |
685 | + |
686 | +Mitigation selection guide |
687 | +-------------------------- |
688 | + |
689 | +1. Trusted userspace |
690 | +^^^^^^^^^^^^^^^^^^^^ |
691 | + |
692 | + If all userspace applications are from trusted sources and do not |
693 | + execute externally supplied untrusted code, then the mitigations can |
694 | + be disabled. |
695 | + |
696 | +2. Protect sensitive programs |
697 | +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
698 | + |
699 | + For security-sensitive programs that have secrets (e.g. crypto |
700 | + keys), protection against Spectre variant 2 can be put in place by |
701 | + disabling indirect branch speculation when the program is running |
702 | + (See Documentation/spec_ctrl.txt). |
703 | + |
704 | +3. Sandbox untrusted programs |
705 | +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
706 | + |
707 | + Untrusted programs that could be a source of attacks can be cordoned |
708 | + off by disabling their indirect branch speculation when they are run |
709 | + (See Documentation/spec_ctrl.txt). |
710 | + This prevents untrusted programs from polluting the branch target |
711 | + buffer. All programs running in SECCOMP sandboxes have indirect |
712 | + branch speculation restricted by default. This behavior can be |
713 | + changed via the kernel command line and sysfs control files. See |
714 | + :ref:`spectre_mitigation_control_command_line`. |
715 | + |
716 | +3. High security mode |
717 | +^^^^^^^^^^^^^^^^^^^^^ |
718 | + |
719 | + All Spectre variant 2 mitigations can be forced on |
720 | + at boot time for all programs (See the "on" option in |
721 | + :ref:`spectre_mitigation_control_command_line`). This will add |
722 | + overhead as indirect branch speculations for all programs will be |
723 | + restricted. |
724 | + |
725 | + On x86, branch target buffer will be flushed with IBPB when switching |
726 | + to a new program. STIBP is left on all the time to protect programs |
727 | + against variant 2 attacks originating from programs running on |
728 | + sibling threads. |
729 | + |
730 | + Alternatively, STIBP can be used only when running programs |
731 | + whose indirect branch speculation is explicitly disabled, |
732 | + while IBPB is still used all the time when switching to a new |
733 | + program to clear the branch target buffer (See "ibpb" option in |
734 | + :ref:`spectre_mitigation_control_command_line`). This "ibpb" option |
735 | + has less performance cost than the "on" option, which leaves STIBP |
736 | + on all the time. |
737 | + |
738 | +References on Spectre |
739 | +--------------------- |
740 | + |
741 | +Intel white papers: |
742 | + |
743 | +.. _spec_ref1: |
744 | + |
745 | +[1] `Intel analysis of speculative execution side channels <https://newsroom.intel.com/wp-content/uploads/sites/11/2018/01/Intel-Analysis-of-Speculative-Execution-Side-Channels.pdf>`_. |
746 | + |
747 | +.. _spec_ref2: |
748 | + |
749 | +[2] `Bounds check bypass <https://software.intel.com/security-software-guidance/software-guidance/bounds-check-bypass>`_. |
750 | + |
751 | +.. _spec_ref3: |
752 | + |
753 | +[3] `Deep dive: Retpoline: A branch target injection mitigation <https://software.intel.com/security-software-guidance/insights/deep-dive-retpoline-branch-target-injection-mitigation>`_. |
754 | + |
755 | +.. _spec_ref4: |
756 | + |
757 | +[4] `Deep Dive: Single Thread Indirect Branch Predictors <https://software.intel.com/security-software-guidance/insights/deep-dive-single-thread-indirect-branch-predictors>`_. |
758 | + |
759 | +AMD white papers: |
760 | + |
761 | +.. _spec_ref5: |
762 | + |
763 | +[5] `AMD64 technology indirect branch control extension <https://developer.amd.com/wp-content/resources/Architecture_Guidelines_Update_Indirect_Branch_Control.pdf>`_. |
764 | + |
765 | +.. _spec_ref6: |
766 | + |
767 | +[6] `Software techniques for managing speculation on AMD processors <https://developer.amd.com/wp-content/resources/Managing-Speculation-on-AMD-Processors.pdf>`_. |
768 | + |
769 | +ARM white papers: |
770 | + |
771 | +.. _spec_ref7: |
772 | + |
773 | +[7] `Cache speculation side-channels <https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability/download-the-whitepaper>`_. |
774 | + |
775 | +.. _spec_ref8: |
776 | + |
777 | +[8] `Cache speculation issues update <https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability/latest-updates/cache-speculation-issues-update>`_. |
778 | + |
779 | +Google white paper: |
780 | + |
781 | +.. _spec_ref9: |
782 | + |
783 | +[9] `Retpoline: a software construct for preventing branch-target-injection <https://support.google.com/faqs/answer/7625886>`_. |
784 | + |
785 | +MIPS white paper: |
786 | + |
787 | +.. _spec_ref10: |
788 | + |
789 | +[10] `MIPS: response on speculative execution and side channel vulnerabilities <https://www.mips.com/blog/mips-response-on-speculative-execution-and-side-channel-vulnerabilities/>`_. |
790 | + |
791 | +Academic papers: |
792 | + |
793 | +.. _spec_ref11: |
794 | + |
795 | +[11] `Spectre Attacks: Exploiting Speculative Execution <https://spectreattack.com/spectre.pdf>`_. |
796 | + |
797 | +.. _spec_ref12: |
798 | + |
799 | +[12] `NetSpectre: Read Arbitrary Memory over Network <https://arxiv.org/abs/1807.10535>`_. |
800 | + |
801 | +.. _spec_ref13: |
802 | + |
803 | +[13] `Spectre Returns! Speculation Attacks using the Return Stack Buffer <https://www.usenix.org/system/files/conference/woot18/woot18-paper-koruyeh.pdf>`_. |
804 | diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt |
805 | index 713765521c451..6c0957c67d207 100644 |
806 | --- a/Documentation/kernel-parameters.txt |
807 | +++ b/Documentation/kernel-parameters.txt |
808 | @@ -4174,8 +4174,12 @@ bytes respectively. Such letter suffixes can also be entirely omitted. |
809 | Specific mitigations can also be selected manually: |
810 | |
811 | retpoline - replace indirect branches |
812 | - retpoline,generic - google's original retpoline |
813 | - retpoline,amd - AMD-specific minimal thunk |
814 | + retpoline,generic - Retpolines |
815 | + retpoline,lfence - LFENCE; indirect branch |
816 | + retpoline,amd - alias for retpoline,lfence |
817 | + eibrs - enhanced IBRS |
818 | + eibrs,retpoline - enhanced IBRS + Retpolines |
819 | + eibrs,lfence - enhanced IBRS + LFENCE |
820 | |
821 | Not specifying this option is equivalent to |
822 | spectre_v2=auto. |
823 | diff --git a/Makefile b/Makefile |
824 | index 308c848b01dc2..482b841188572 100644 |
825 | --- a/Makefile |
826 | +++ b/Makefile |
827 | @@ -1,6 +1,6 @@ |
828 | VERSION = 4 |
829 | PATCHLEVEL = 9 |
830 | -SUBLEVEL = 305 |
831 | +SUBLEVEL = 306 |
832 | EXTRAVERSION = |
833 | NAME = Roaring Lionus |
834 | |
835 | diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h |
836 | index 7d727506096f6..2fa3fd30a9d61 100644 |
837 | --- a/arch/arm/include/asm/assembler.h |
838 | +++ b/arch/arm/include/asm/assembler.h |
839 | @@ -108,6 +108,16 @@ |
840 | .endm |
841 | #endif |
842 | |
843 | +#if __LINUX_ARM_ARCH__ < 7 |
844 | + .macro dsb, args |
845 | + mcr p15, 0, r0, c7, c10, 4 |
846 | + .endm |
847 | + |
848 | + .macro isb, args |
849 | + mcr p15, 0, r0, c7, c5, 4 |
850 | + .endm |
851 | +#endif |
852 | + |
853 | .macro asm_trace_hardirqs_off, save=1 |
854 | #if defined(CONFIG_TRACE_IRQFLAGS) |
855 | .if \save |
856 | diff --git a/arch/arm/include/asm/spectre.h b/arch/arm/include/asm/spectre.h |
857 | new file mode 100644 |
858 | index 0000000000000..d1fa5607d3aa3 |
859 | --- /dev/null |
860 | +++ b/arch/arm/include/asm/spectre.h |
861 | @@ -0,0 +1,32 @@ |
862 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
863 | + |
864 | +#ifndef __ASM_SPECTRE_H |
865 | +#define __ASM_SPECTRE_H |
866 | + |
867 | +enum { |
868 | + SPECTRE_UNAFFECTED, |
869 | + SPECTRE_MITIGATED, |
870 | + SPECTRE_VULNERABLE, |
871 | +}; |
872 | + |
873 | +enum { |
874 | + __SPECTRE_V2_METHOD_BPIALL, |
875 | + __SPECTRE_V2_METHOD_ICIALLU, |
876 | + __SPECTRE_V2_METHOD_SMC, |
877 | + __SPECTRE_V2_METHOD_HVC, |
878 | + __SPECTRE_V2_METHOD_LOOP8, |
879 | +}; |
880 | + |
881 | +enum { |
882 | + SPECTRE_V2_METHOD_BPIALL = BIT(__SPECTRE_V2_METHOD_BPIALL), |
883 | + SPECTRE_V2_METHOD_ICIALLU = BIT(__SPECTRE_V2_METHOD_ICIALLU), |
884 | + SPECTRE_V2_METHOD_SMC = BIT(__SPECTRE_V2_METHOD_SMC), |
885 | + SPECTRE_V2_METHOD_HVC = BIT(__SPECTRE_V2_METHOD_HVC), |
886 | + SPECTRE_V2_METHOD_LOOP8 = BIT(__SPECTRE_V2_METHOD_LOOP8), |
887 | +}; |
888 | + |
889 | +void spectre_v2_update_state(unsigned int state, unsigned int methods); |
890 | + |
891 | +int spectre_bhb_update_vectors(unsigned int method); |
892 | + |
893 | +#endif |
894 | diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile |
895 | index 9bddd762880cf..1738d5b61eaa1 100644 |
896 | --- a/arch/arm/kernel/Makefile |
897 | +++ b/arch/arm/kernel/Makefile |
898 | @@ -100,4 +100,6 @@ endif |
899 | |
900 | obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o |
901 | |
902 | +obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o |
903 | + |
904 | extra-y := $(head-y) vmlinux.lds |
905 | diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S |
906 | index 2cac25a69a85d..1040efcb98db6 100644 |
907 | --- a/arch/arm/kernel/entry-armv.S |
908 | +++ b/arch/arm/kernel/entry-armv.S |
909 | @@ -1036,12 +1036,11 @@ vector_\name: |
910 | sub lr, lr, #\correction |
911 | .endif |
912 | |
913 | - @ |
914 | - @ Save r0, lr_<exception> (parent PC) and spsr_<exception> |
915 | - @ (parent CPSR) |
916 | - @ |
917 | + @ Save r0, lr_<exception> (parent PC) |
918 | stmia sp, {r0, lr} @ save r0, lr |
919 | - mrs lr, spsr |
920 | + |
921 | + @ Save spsr_<exception> (parent CPSR) |
922 | +2: mrs lr, spsr |
923 | str lr, [sp, #8] @ save spsr |
924 | |
925 | @ |
926 | @@ -1062,6 +1061,44 @@ vector_\name: |
927 | movs pc, lr @ branch to handler in SVC mode |
928 | ENDPROC(vector_\name) |
929 | |
930 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
931 | + .subsection 1 |
932 | + .align 5 |
933 | +vector_bhb_loop8_\name: |
934 | + .if \correction |
935 | + sub lr, lr, #\correction |
936 | + .endif |
937 | + |
938 | + @ Save r0, lr_<exception> (parent PC) |
939 | + stmia sp, {r0, lr} |
940 | + |
941 | + @ bhb workaround |
942 | + mov r0, #8 |
943 | +1: b . + 4 |
944 | + subs r0, r0, #1 |
945 | + bne 1b |
946 | + dsb |
947 | + isb |
948 | + b 2b |
949 | +ENDPROC(vector_bhb_loop8_\name) |
950 | + |
951 | +vector_bhb_bpiall_\name: |
952 | + .if \correction |
953 | + sub lr, lr, #\correction |
954 | + .endif |
955 | + |
956 | + @ Save r0, lr_<exception> (parent PC) |
957 | + stmia sp, {r0, lr} |
958 | + |
959 | + @ bhb workaround |
960 | + mcr p15, 0, r0, c7, c5, 6 @ BPIALL |
961 | + @ isb not needed due to "movs pc, lr" in the vector stub |
962 | + @ which gives a "context synchronisation". |
963 | + b 2b |
964 | +ENDPROC(vector_bhb_bpiall_\name) |
965 | + .previous |
966 | +#endif |
967 | + |
968 | .align 2 |
969 | @ handler addresses follow this label |
970 | 1: |
971 | @@ -1070,6 +1107,10 @@ ENDPROC(vector_\name) |
972 | .section .stubs, "ax", %progbits |
973 | @ This must be the first word |
974 | .word vector_swi |
975 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
976 | + .word vector_bhb_loop8_swi |
977 | + .word vector_bhb_bpiall_swi |
978 | +#endif |
979 | |
980 | vector_rst: |
981 | ARM( swi SYS_ERROR0 ) |
982 | @@ -1184,8 +1225,10 @@ vector_addrexcptn: |
983 | * FIQ "NMI" handler |
984 | *----------------------------------------------------------------------------- |
985 | * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86 |
986 | - * systems. |
987 | + * systems. This must be the last vector stub, so lets place it in its own |
988 | + * subsection. |
989 | */ |
990 | + .subsection 2 |
991 | vector_stub fiq, FIQ_MODE, 4 |
992 | |
993 | .long __fiq_usr @ 0 (USR_26 / USR_32) |
994 | @@ -1218,6 +1261,30 @@ vector_addrexcptn: |
995 | W(b) vector_irq |
996 | W(b) vector_fiq |
997 | |
998 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
999 | + .section .vectors.bhb.loop8, "ax", %progbits |
1000 | +.L__vectors_bhb_loop8_start: |
1001 | + W(b) vector_rst |
1002 | + W(b) vector_bhb_loop8_und |
1003 | + W(ldr) pc, .L__vectors_bhb_loop8_start + 0x1004 |
1004 | + W(b) vector_bhb_loop8_pabt |
1005 | + W(b) vector_bhb_loop8_dabt |
1006 | + W(b) vector_addrexcptn |
1007 | + W(b) vector_bhb_loop8_irq |
1008 | + W(b) vector_bhb_loop8_fiq |
1009 | + |
1010 | + .section .vectors.bhb.bpiall, "ax", %progbits |
1011 | +.L__vectors_bhb_bpiall_start: |
1012 | + W(b) vector_rst |
1013 | + W(b) vector_bhb_bpiall_und |
1014 | + W(ldr) pc, .L__vectors_bhb_bpiall_start + 0x1008 |
1015 | + W(b) vector_bhb_bpiall_pabt |
1016 | + W(b) vector_bhb_bpiall_dabt |
1017 | + W(b) vector_addrexcptn |
1018 | + W(b) vector_bhb_bpiall_irq |
1019 | + W(b) vector_bhb_bpiall_fiq |
1020 | +#endif |
1021 | + |
1022 | .data |
1023 | |
1024 | .globl cr_alignment |
1025 | diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S |
1026 | index 178a2a9606595..fb0f505c9924f 100644 |
1027 | --- a/arch/arm/kernel/entry-common.S |
1028 | +++ b/arch/arm/kernel/entry-common.S |
1029 | @@ -142,6 +142,29 @@ ENDPROC(ret_from_fork) |
1030 | *----------------------------------------------------------------------------- |
1031 | */ |
1032 | |
1033 | + .align 5 |
1034 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
1035 | +ENTRY(vector_bhb_loop8_swi) |
1036 | + sub sp, sp, #PT_REGS_SIZE |
1037 | + stmia sp, {r0 - r12} |
1038 | + mov r8, #8 |
1039 | +1: b 2f |
1040 | +2: subs r8, r8, #1 |
1041 | + bne 1b |
1042 | + dsb |
1043 | + isb |
1044 | + b 3f |
1045 | +ENDPROC(vector_bhb_loop8_swi) |
1046 | + |
1047 | + .align 5 |
1048 | +ENTRY(vector_bhb_bpiall_swi) |
1049 | + sub sp, sp, #PT_REGS_SIZE |
1050 | + stmia sp, {r0 - r12} |
1051 | + mcr p15, 0, r8, c7, c5, 6 @ BPIALL |
1052 | + isb |
1053 | + b 3f |
1054 | +ENDPROC(vector_bhb_bpiall_swi) |
1055 | +#endif |
1056 | .align 5 |
1057 | ENTRY(vector_swi) |
1058 | #ifdef CONFIG_CPU_V7M |
1059 | @@ -149,6 +172,7 @@ ENTRY(vector_swi) |
1060 | #else |
1061 | sub sp, sp, #PT_REGS_SIZE |
1062 | stmia sp, {r0 - r12} @ Calling r0 - r12 |
1063 | +3: |
1064 | ARM( add r8, sp, #S_PC ) |
1065 | ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr |
1066 | THUMB( mov r8, sp ) |
1067 | diff --git a/arch/arm/kernel/spectre.c b/arch/arm/kernel/spectre.c |
1068 | new file mode 100644 |
1069 | index 0000000000000..0dcefc36fb7a0 |
1070 | --- /dev/null |
1071 | +++ b/arch/arm/kernel/spectre.c |
1072 | @@ -0,0 +1,71 @@ |
1073 | +// SPDX-License-Identifier: GPL-2.0-only |
1074 | +#include <linux/bpf.h> |
1075 | +#include <linux/cpu.h> |
1076 | +#include <linux/device.h> |
1077 | + |
1078 | +#include <asm/spectre.h> |
1079 | + |
1080 | +static bool _unprivileged_ebpf_enabled(void) |
1081 | +{ |
1082 | +#ifdef CONFIG_BPF_SYSCALL |
1083 | + return !sysctl_unprivileged_bpf_disabled; |
1084 | +#else |
1085 | + return false; |
1086 | +#endif |
1087 | +} |
1088 | + |
1089 | +ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, |
1090 | + char *buf) |
1091 | +{ |
1092 | + return sprintf(buf, "Mitigation: __user pointer sanitization\n"); |
1093 | +} |
1094 | + |
1095 | +static unsigned int spectre_v2_state; |
1096 | +static unsigned int spectre_v2_methods; |
1097 | + |
1098 | +void spectre_v2_update_state(unsigned int state, unsigned int method) |
1099 | +{ |
1100 | + if (state > spectre_v2_state) |
1101 | + spectre_v2_state = state; |
1102 | + spectre_v2_methods |= method; |
1103 | +} |
1104 | + |
1105 | +ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, |
1106 | + char *buf) |
1107 | +{ |
1108 | + const char *method; |
1109 | + |
1110 | + if (spectre_v2_state == SPECTRE_UNAFFECTED) |
1111 | + return sprintf(buf, "%s\n", "Not affected"); |
1112 | + |
1113 | + if (spectre_v2_state != SPECTRE_MITIGATED) |
1114 | + return sprintf(buf, "%s\n", "Vulnerable"); |
1115 | + |
1116 | + if (_unprivileged_ebpf_enabled()) |
1117 | + return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); |
1118 | + |
1119 | + switch (spectre_v2_methods) { |
1120 | + case SPECTRE_V2_METHOD_BPIALL: |
1121 | + method = "Branch predictor hardening"; |
1122 | + break; |
1123 | + |
1124 | + case SPECTRE_V2_METHOD_ICIALLU: |
1125 | + method = "I-cache invalidation"; |
1126 | + break; |
1127 | + |
1128 | + case SPECTRE_V2_METHOD_SMC: |
1129 | + case SPECTRE_V2_METHOD_HVC: |
1130 | + method = "Firmware call"; |
1131 | + break; |
1132 | + |
1133 | + case SPECTRE_V2_METHOD_LOOP8: |
1134 | + method = "History overwrite"; |
1135 | + break; |
1136 | + |
1137 | + default: |
1138 | + method = "Multiple mitigations"; |
1139 | + break; |
1140 | + } |
1141 | + |
1142 | + return sprintf(buf, "Mitigation: %s\n", method); |
1143 | +} |
1144 | diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c |
1145 | index aa316a7562b1f..7fca7ece8f979 100644 |
1146 | --- a/arch/arm/kernel/traps.c |
1147 | +++ b/arch/arm/kernel/traps.c |
1148 | @@ -31,6 +31,7 @@ |
1149 | #include <linux/atomic.h> |
1150 | #include <asm/cacheflush.h> |
1151 | #include <asm/exception.h> |
1152 | +#include <asm/spectre.h> |
1153 | #include <asm/unistd.h> |
1154 | #include <asm/traps.h> |
1155 | #include <asm/ptrace.h> |
1156 | @@ -819,10 +820,59 @@ static inline void __init kuser_init(void *vectors) |
1157 | } |
1158 | #endif |
1159 | |
1160 | +#ifndef CONFIG_CPU_V7M |
1161 | +static void copy_from_lma(void *vma, void *lma_start, void *lma_end) |
1162 | +{ |
1163 | + memcpy(vma, lma_start, lma_end - lma_start); |
1164 | +} |
1165 | + |
1166 | +static void flush_vectors(void *vma, size_t offset, size_t size) |
1167 | +{ |
1168 | + unsigned long start = (unsigned long)vma + offset; |
1169 | + unsigned long end = start + size; |
1170 | + |
1171 | + flush_icache_range(start, end); |
1172 | +} |
1173 | + |
1174 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
1175 | +int spectre_bhb_update_vectors(unsigned int method) |
1176 | +{ |
1177 | + extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[]; |
1178 | + extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[]; |
1179 | + void *vec_start, *vec_end; |
1180 | + |
1181 | + if (system_state >= SYSTEM_RUNNING) { |
1182 | + pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n", |
1183 | + smp_processor_id()); |
1184 | + return SPECTRE_VULNERABLE; |
1185 | + } |
1186 | + |
1187 | + switch (method) { |
1188 | + case SPECTRE_V2_METHOD_LOOP8: |
1189 | + vec_start = __vectors_bhb_loop8_start; |
1190 | + vec_end = __vectors_bhb_loop8_end; |
1191 | + break; |
1192 | + |
1193 | + case SPECTRE_V2_METHOD_BPIALL: |
1194 | + vec_start = __vectors_bhb_bpiall_start; |
1195 | + vec_end = __vectors_bhb_bpiall_end; |
1196 | + break; |
1197 | + |
1198 | + default: |
1199 | + pr_err("CPU%u: unknown Spectre BHB state %d\n", |
1200 | + smp_processor_id(), method); |
1201 | + return SPECTRE_VULNERABLE; |
1202 | + } |
1203 | + |
1204 | + copy_from_lma(vectors_page, vec_start, vec_end); |
1205 | + flush_vectors(vectors_page, 0, vec_end - vec_start); |
1206 | + |
1207 | + return SPECTRE_MITIGATED; |
1208 | +} |
1209 | +#endif |
1210 | + |
1211 | void __init early_trap_init(void *vectors_base) |
1212 | { |
1213 | -#ifndef CONFIG_CPU_V7M |
1214 | - unsigned long vectors = (unsigned long)vectors_base; |
1215 | extern char __stubs_start[], __stubs_end[]; |
1216 | extern char __vectors_start[], __vectors_end[]; |
1217 | unsigned i; |
1218 | @@ -843,17 +893,20 @@ void __init early_trap_init(void *vectors_base) |
1219 | * into the vector page, mapped at 0xffff0000, and ensure these |
1220 | * are visible to the instruction stream. |
1221 | */ |
1222 | - memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start); |
1223 | - memcpy((void *)vectors + 0x1000, __stubs_start, __stubs_end - __stubs_start); |
1224 | + copy_from_lma(vectors_base, __vectors_start, __vectors_end); |
1225 | + copy_from_lma(vectors_base + 0x1000, __stubs_start, __stubs_end); |
1226 | |
1227 | kuser_init(vectors_base); |
1228 | |
1229 | - flush_icache_range(vectors, vectors + PAGE_SIZE * 2); |
1230 | + flush_vectors(vectors_base, 0, PAGE_SIZE * 2); |
1231 | +} |
1232 | #else /* ifndef CONFIG_CPU_V7M */ |
1233 | +void __init early_trap_init(void *vectors_base) |
1234 | +{ |
1235 | /* |
1236 | * on V7-M there is no need to copy the vector table to a dedicated |
1237 | * memory area. The address is configurable and so a table in the kernel |
1238 | * image can be used. |
1239 | */ |
1240 | -#endif |
1241 | } |
1242 | +#endif |
1243 | diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S |
1244 | index 37b2a11af3459..d80ef8c2bb461 100644 |
1245 | --- a/arch/arm/kernel/vmlinux-xip.lds.S |
1246 | +++ b/arch/arm/kernel/vmlinux-xip.lds.S |
1247 | @@ -12,6 +12,19 @@ |
1248 | #include <asm/memory.h> |
1249 | #include <asm/page.h> |
1250 | |
1251 | +/* |
1252 | + * ld.lld does not support NOCROSSREFS: |
1253 | + * https://github.com/ClangBuiltLinux/linux/issues/1609 |
1254 | + */ |
1255 | +#ifdef CONFIG_LD_IS_LLD |
1256 | +#define NOCROSSREFS |
1257 | +#endif |
1258 | + |
1259 | +/* Set start/end symbol names to the LMA for the section */ |
1260 | +#define ARM_LMA(sym, section) \ |
1261 | + sym##_start = LOADADDR(section); \ |
1262 | + sym##_end = LOADADDR(section) + SIZEOF(section) |
1263 | + |
1264 | #define PROC_INFO \ |
1265 | . = ALIGN(4); \ |
1266 | VMLINUX_SYMBOL(__proc_info_begin) = .; \ |
1267 | @@ -148,19 +161,31 @@ SECTIONS |
1268 | * The vectors and stubs are relocatable code, and the |
1269 | * only thing that matters is their relative offsets |
1270 | */ |
1271 | - __vectors_start = .; |
1272 | - .vectors 0xffff0000 : AT(__vectors_start) { |
1273 | - *(.vectors) |
1274 | + __vectors_lma = .; |
1275 | + OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { |
1276 | + .vectors { |
1277 | + *(.vectors) |
1278 | + } |
1279 | + .vectors.bhb.loop8 { |
1280 | + *(.vectors.bhb.loop8) |
1281 | + } |
1282 | + .vectors.bhb.bpiall { |
1283 | + *(.vectors.bhb.bpiall) |
1284 | + } |
1285 | } |
1286 | - . = __vectors_start + SIZEOF(.vectors); |
1287 | - __vectors_end = .; |
1288 | - |
1289 | - __stubs_start = .; |
1290 | - .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_start) { |
1291 | + ARM_LMA(__vectors, .vectors); |
1292 | + ARM_LMA(__vectors_bhb_loop8, .vectors.bhb.loop8); |
1293 | + ARM_LMA(__vectors_bhb_bpiall, .vectors.bhb.bpiall); |
1294 | + . = __vectors_lma + SIZEOF(.vectors) + |
1295 | + SIZEOF(.vectors.bhb.loop8) + |
1296 | + SIZEOF(.vectors.bhb.bpiall); |
1297 | + |
1298 | + __stubs_lma = .; |
1299 | + .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { |
1300 | *(.stubs) |
1301 | } |
1302 | - . = __stubs_start + SIZEOF(.stubs); |
1303 | - __stubs_end = .; |
1304 | + ARM_LMA(__stubs, .stubs); |
1305 | + . = __stubs_lma + SIZEOF(.stubs); |
1306 | |
1307 | PROVIDE(vector_fiq_offset = vector_fiq - ADDR(.vectors)); |
1308 | |
1309 | diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S |
1310 | index f7f55df0bf7b3..0d560a24408f0 100644 |
1311 | --- a/arch/arm/kernel/vmlinux.lds.S |
1312 | +++ b/arch/arm/kernel/vmlinux.lds.S |
1313 | @@ -14,6 +14,19 @@ |
1314 | #include <asm/page.h> |
1315 | #include <asm/pgtable.h> |
1316 | |
1317 | +/* |
1318 | + * ld.lld does not support NOCROSSREFS: |
1319 | + * https://github.com/ClangBuiltLinux/linux/issues/1609 |
1320 | + */ |
1321 | +#ifdef CONFIG_LD_IS_LLD |
1322 | +#define NOCROSSREFS |
1323 | +#endif |
1324 | + |
1325 | +/* Set start/end symbol names to the LMA for the section */ |
1326 | +#define ARM_LMA(sym, section) \ |
1327 | + sym##_start = LOADADDR(section); \ |
1328 | + sym##_end = LOADADDR(section) + SIZEOF(section) |
1329 | + |
1330 | #define PROC_INFO \ |
1331 | . = ALIGN(4); \ |
1332 | VMLINUX_SYMBOL(__proc_info_begin) = .; \ |
1333 | @@ -169,19 +182,31 @@ SECTIONS |
1334 | * The vectors and stubs are relocatable code, and the |
1335 | * only thing that matters is their relative offsets |
1336 | */ |
1337 | - __vectors_start = .; |
1338 | - .vectors 0xffff0000 : AT(__vectors_start) { |
1339 | - *(.vectors) |
1340 | + __vectors_lma = .; |
1341 | + OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { |
1342 | + .vectors { |
1343 | + *(.vectors) |
1344 | + } |
1345 | + .vectors.bhb.loop8 { |
1346 | + *(.vectors.bhb.loop8) |
1347 | + } |
1348 | + .vectors.bhb.bpiall { |
1349 | + *(.vectors.bhb.bpiall) |
1350 | + } |
1351 | } |
1352 | - . = __vectors_start + SIZEOF(.vectors); |
1353 | - __vectors_end = .; |
1354 | - |
1355 | - __stubs_start = .; |
1356 | - .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_start) { |
1357 | + ARM_LMA(__vectors, .vectors); |
1358 | + ARM_LMA(__vectors_bhb_loop8, .vectors.bhb.loop8); |
1359 | + ARM_LMA(__vectors_bhb_bpiall, .vectors.bhb.bpiall); |
1360 | + . = __vectors_lma + SIZEOF(.vectors) + |
1361 | + SIZEOF(.vectors.bhb.loop8) + |
1362 | + SIZEOF(.vectors.bhb.bpiall); |
1363 | + |
1364 | + __stubs_lma = .; |
1365 | + .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { |
1366 | *(.stubs) |
1367 | } |
1368 | - . = __stubs_start + SIZEOF(.stubs); |
1369 | - __stubs_end = .; |
1370 | + ARM_LMA(__stubs, .stubs); |
1371 | + . = __stubs_lma + SIZEOF(.stubs); |
1372 | |
1373 | PROVIDE(vector_fiq_offset = vector_fiq - ADDR(.vectors)); |
1374 | |
1375 | diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig |
1376 | index 93623627a0b68..5c98074010d25 100644 |
1377 | --- a/arch/arm/mm/Kconfig |
1378 | +++ b/arch/arm/mm/Kconfig |
1379 | @@ -803,6 +803,7 @@ config CPU_BPREDICT_DISABLE |
1380 | |
1381 | config CPU_SPECTRE |
1382 | bool |
1383 | + select GENERIC_CPU_VULNERABILITIES |
1384 | |
1385 | config HARDEN_BRANCH_PREDICTOR |
1386 | bool "Harden the branch predictor against aliasing attacks" if EXPERT |
1387 | @@ -823,6 +824,16 @@ config HARDEN_BRANCH_PREDICTOR |
1388 | |
1389 | If unsure, say Y. |
1390 | |
1391 | +config HARDEN_BRANCH_HISTORY |
1392 | + bool "Harden Spectre style attacks against branch history" if EXPERT |
1393 | + depends on CPU_SPECTRE |
1394 | + default y |
1395 | + help |
1396 | + Speculation attacks against some high-performance processors can |
1397 | + make use of branch history to influence future speculation. When |
1398 | + taking an exception, a sequence of branches overwrites the branch |
1399 | + history, or branch history is invalidated. |
1400 | + |
1401 | config TLS_REG_EMUL |
1402 | bool |
1403 | select NEED_KUSER_HELPERS |
1404 | diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c |
1405 | index 9a07916af8dd2..1b6e770bc1cd3 100644 |
1406 | --- a/arch/arm/mm/proc-v7-bugs.c |
1407 | +++ b/arch/arm/mm/proc-v7-bugs.c |
1408 | @@ -7,8 +7,36 @@ |
1409 | #include <asm/cp15.h> |
1410 | #include <asm/cputype.h> |
1411 | #include <asm/proc-fns.h> |
1412 | +#include <asm/spectre.h> |
1413 | #include <asm/system_misc.h> |
1414 | |
1415 | +#ifdef CONFIG_ARM_PSCI |
1416 | +#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED 1 |
1417 | +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) |
1418 | +{ |
1419 | + struct arm_smccc_res res; |
1420 | + |
1421 | + arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
1422 | + ARM_SMCCC_ARCH_WORKAROUND_1, &res); |
1423 | + |
1424 | + switch ((int)res.a0) { |
1425 | + case SMCCC_RET_SUCCESS: |
1426 | + return SPECTRE_MITIGATED; |
1427 | + |
1428 | + case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: |
1429 | + return SPECTRE_UNAFFECTED; |
1430 | + |
1431 | + default: |
1432 | + return SPECTRE_VULNERABLE; |
1433 | + } |
1434 | +} |
1435 | +#else |
1436 | +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) |
1437 | +{ |
1438 | + return SPECTRE_VULNERABLE; |
1439 | +} |
1440 | +#endif |
1441 | + |
1442 | #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
1443 | DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); |
1444 | |
1445 | @@ -37,13 +65,61 @@ static void __maybe_unused call_hvc_arch_workaround_1(void) |
1446 | arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); |
1447 | } |
1448 | |
1449 | -static void cpu_v7_spectre_init(void) |
1450 | +static unsigned int spectre_v2_install_workaround(unsigned int method) |
1451 | { |
1452 | const char *spectre_v2_method = NULL; |
1453 | int cpu = smp_processor_id(); |
1454 | |
1455 | if (per_cpu(harden_branch_predictor_fn, cpu)) |
1456 | - return; |
1457 | + return SPECTRE_MITIGATED; |
1458 | + |
1459 | + switch (method) { |
1460 | + case SPECTRE_V2_METHOD_BPIALL: |
1461 | + per_cpu(harden_branch_predictor_fn, cpu) = |
1462 | + harden_branch_predictor_bpiall; |
1463 | + spectre_v2_method = "BPIALL"; |
1464 | + break; |
1465 | + |
1466 | + case SPECTRE_V2_METHOD_ICIALLU: |
1467 | + per_cpu(harden_branch_predictor_fn, cpu) = |
1468 | + harden_branch_predictor_iciallu; |
1469 | + spectre_v2_method = "ICIALLU"; |
1470 | + break; |
1471 | + |
1472 | + case SPECTRE_V2_METHOD_HVC: |
1473 | + per_cpu(harden_branch_predictor_fn, cpu) = |
1474 | + call_hvc_arch_workaround_1; |
1475 | + cpu_do_switch_mm = cpu_v7_hvc_switch_mm; |
1476 | + spectre_v2_method = "hypervisor"; |
1477 | + break; |
1478 | + |
1479 | + case SPECTRE_V2_METHOD_SMC: |
1480 | + per_cpu(harden_branch_predictor_fn, cpu) = |
1481 | + call_smc_arch_workaround_1; |
1482 | + cpu_do_switch_mm = cpu_v7_smc_switch_mm; |
1483 | + spectre_v2_method = "firmware"; |
1484 | + break; |
1485 | + } |
1486 | + |
1487 | + if (spectre_v2_method) |
1488 | + pr_info("CPU%u: Spectre v2: using %s workaround\n", |
1489 | + smp_processor_id(), spectre_v2_method); |
1490 | + |
1491 | + return SPECTRE_MITIGATED; |
1492 | +} |
1493 | +#else |
1494 | +static unsigned int spectre_v2_install_workaround(unsigned int method) |
1495 | +{ |
1496 | + pr_info("CPU%u: Spectre V2: workarounds disabled by configuration\n", |
1497 | + smp_processor_id()); |
1498 | + |
1499 | + return SPECTRE_VULNERABLE; |
1500 | +} |
1501 | +#endif |
1502 | + |
1503 | +static void cpu_v7_spectre_v2_init(void) |
1504 | +{ |
1505 | + unsigned int state, method = 0; |
1506 | |
1507 | switch (read_cpuid_part()) { |
1508 | case ARM_CPU_PART_CORTEX_A8: |
1509 | @@ -52,29 +128,32 @@ static void cpu_v7_spectre_init(void) |
1510 | case ARM_CPU_PART_CORTEX_A17: |
1511 | case ARM_CPU_PART_CORTEX_A73: |
1512 | case ARM_CPU_PART_CORTEX_A75: |
1513 | - per_cpu(harden_branch_predictor_fn, cpu) = |
1514 | - harden_branch_predictor_bpiall; |
1515 | - spectre_v2_method = "BPIALL"; |
1516 | + state = SPECTRE_MITIGATED; |
1517 | + method = SPECTRE_V2_METHOD_BPIALL; |
1518 | break; |
1519 | |
1520 | case ARM_CPU_PART_CORTEX_A15: |
1521 | case ARM_CPU_PART_BRAHMA_B15: |
1522 | - per_cpu(harden_branch_predictor_fn, cpu) = |
1523 | - harden_branch_predictor_iciallu; |
1524 | - spectre_v2_method = "ICIALLU"; |
1525 | + state = SPECTRE_MITIGATED; |
1526 | + method = SPECTRE_V2_METHOD_ICIALLU; |
1527 | break; |
1528 | |
1529 | -#ifdef CONFIG_ARM_PSCI |
1530 | default: |
1531 | /* Other ARM CPUs require no workaround */ |
1532 | - if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) |
1533 | + if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) { |
1534 | + state = SPECTRE_UNAFFECTED; |
1535 | break; |
1536 | + } |
1537 | /* fallthrough */ |
1538 | - /* Cortex A57/A72 require firmware workaround */ |
1539 | + /* Cortex A57/A72 require firmware workaround */ |
1540 | case ARM_CPU_PART_CORTEX_A57: |
1541 | case ARM_CPU_PART_CORTEX_A72: { |
1542 | struct arm_smccc_res res; |
1543 | |
1544 | + state = spectre_v2_get_cpu_fw_mitigation_state(); |
1545 | + if (state != SPECTRE_MITIGATED) |
1546 | + break; |
1547 | + |
1548 | if (psci_ops.smccc_version == SMCCC_VERSION_1_0) |
1549 | break; |
1550 | |
1551 | @@ -84,10 +163,7 @@ static void cpu_v7_spectre_init(void) |
1552 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); |
1553 | if ((int)res.a0 != 0) |
1554 | break; |
1555 | - per_cpu(harden_branch_predictor_fn, cpu) = |
1556 | - call_hvc_arch_workaround_1; |
1557 | - cpu_do_switch_mm = cpu_v7_hvc_switch_mm; |
1558 | - spectre_v2_method = "hypervisor"; |
1559 | + method = SPECTRE_V2_METHOD_HVC; |
1560 | break; |
1561 | |
1562 | case PSCI_CONDUIT_SMC: |
1563 | @@ -95,29 +171,97 @@ static void cpu_v7_spectre_init(void) |
1564 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); |
1565 | if ((int)res.a0 != 0) |
1566 | break; |
1567 | - per_cpu(harden_branch_predictor_fn, cpu) = |
1568 | - call_smc_arch_workaround_1; |
1569 | - cpu_do_switch_mm = cpu_v7_smc_switch_mm; |
1570 | - spectre_v2_method = "firmware"; |
1571 | + method = SPECTRE_V2_METHOD_SMC; |
1572 | break; |
1573 | |
1574 | default: |
1575 | + state = SPECTRE_VULNERABLE; |
1576 | break; |
1577 | } |
1578 | } |
1579 | -#endif |
1580 | } |
1581 | |
1582 | - if (spectre_v2_method) |
1583 | - pr_info("CPU%u: Spectre v2: using %s workaround\n", |
1584 | - smp_processor_id(), spectre_v2_method); |
1585 | + if (state == SPECTRE_MITIGATED) |
1586 | + state = spectre_v2_install_workaround(method); |
1587 | + |
1588 | + spectre_v2_update_state(state, method); |
1589 | +} |
1590 | + |
1591 | +#ifdef CONFIG_HARDEN_BRANCH_HISTORY |
1592 | +static int spectre_bhb_method; |
1593 | + |
1594 | +static const char *spectre_bhb_method_name(int method) |
1595 | +{ |
1596 | + switch (method) { |
1597 | + case SPECTRE_V2_METHOD_LOOP8: |
1598 | + return "loop"; |
1599 | + |
1600 | + case SPECTRE_V2_METHOD_BPIALL: |
1601 | + return "BPIALL"; |
1602 | + |
1603 | + default: |
1604 | + return "unknown"; |
1605 | + } |
1606 | +} |
1607 | + |
1608 | +static int spectre_bhb_install_workaround(int method) |
1609 | +{ |
1610 | + if (spectre_bhb_method != method) { |
1611 | + if (spectre_bhb_method) { |
1612 | + pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n", |
1613 | + smp_processor_id()); |
1614 | + |
1615 | + return SPECTRE_VULNERABLE; |
1616 | + } |
1617 | + |
1618 | + if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE) |
1619 | + return SPECTRE_VULNERABLE; |
1620 | + |
1621 | + spectre_bhb_method = method; |
1622 | + } |
1623 | + |
1624 | + pr_info("CPU%u: Spectre BHB: using %s workaround\n", |
1625 | + smp_processor_id(), spectre_bhb_method_name(method)); |
1626 | + |
1627 | + return SPECTRE_MITIGATED; |
1628 | } |
1629 | #else |
1630 | -static void cpu_v7_spectre_init(void) |
1631 | +static int spectre_bhb_install_workaround(int method) |
1632 | { |
1633 | + return SPECTRE_VULNERABLE; |
1634 | } |
1635 | #endif |
1636 | |
1637 | +static void cpu_v7_spectre_bhb_init(void) |
1638 | +{ |
1639 | + unsigned int state, method = 0; |
1640 | + |
1641 | + switch (read_cpuid_part()) { |
1642 | + case ARM_CPU_PART_CORTEX_A15: |
1643 | + case ARM_CPU_PART_BRAHMA_B15: |
1644 | + case ARM_CPU_PART_CORTEX_A57: |
1645 | + case ARM_CPU_PART_CORTEX_A72: |
1646 | + state = SPECTRE_MITIGATED; |
1647 | + method = SPECTRE_V2_METHOD_LOOP8; |
1648 | + break; |
1649 | + |
1650 | + case ARM_CPU_PART_CORTEX_A73: |
1651 | + case ARM_CPU_PART_CORTEX_A75: |
1652 | + state = SPECTRE_MITIGATED; |
1653 | + method = SPECTRE_V2_METHOD_BPIALL; |
1654 | + break; |
1655 | + |
1656 | + default: |
1657 | + state = SPECTRE_UNAFFECTED; |
1658 | + break; |
1659 | + } |
1660 | + |
1661 | + if (state == SPECTRE_MITIGATED) |
1662 | + state = spectre_bhb_install_workaround(method); |
1663 | + |
1664 | + spectre_v2_update_state(state, method); |
1665 | +} |
1666 | + |
1667 | static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, |
1668 | u32 mask, const char *msg) |
1669 | { |
1670 | @@ -146,16 +290,17 @@ static bool check_spectre_auxcr(bool *warned, u32 bit) |
1671 | void cpu_v7_ca8_ibe(void) |
1672 | { |
1673 | if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6))) |
1674 | - cpu_v7_spectre_init(); |
1675 | + cpu_v7_spectre_v2_init(); |
1676 | } |
1677 | |
1678 | void cpu_v7_ca15_ibe(void) |
1679 | { |
1680 | if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0))) |
1681 | - cpu_v7_spectre_init(); |
1682 | + cpu_v7_spectre_v2_init(); |
1683 | } |
1684 | |
1685 | void cpu_v7_bugs_init(void) |
1686 | { |
1687 | - cpu_v7_spectre_init(); |
1688 | + cpu_v7_spectre_v2_init(); |
1689 | + cpu_v7_spectre_bhb_init(); |
1690 | } |
1691 | diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig |
1692 | index 3ce5b5bd1dc45..fa202cd53b619 100644 |
1693 | --- a/arch/x86/Kconfig |
1694 | +++ b/arch/x86/Kconfig |
1695 | @@ -418,10 +418,6 @@ config RETPOLINE |
1696 | branches. Requires a compiler with -mindirect-branch=thunk-extern |
1697 | support for full protection. The kernel may run slower. |
1698 | |
1699 | - Without compiler support, at least indirect branches in assembler |
1700 | - code are eliminated. Since this includes the syscall entry path, |
1701 | - it is not entirely pointless. |
1702 | - |
1703 | if X86_32 |
1704 | config X86_EXTENDED_PLATFORM |
1705 | bool "Support for extended (non-PC) x86 platforms" |
1706 | diff --git a/arch/x86/Makefile b/arch/x86/Makefile |
1707 | index 0bc35e3e6c5cd..a77737a979c8c 100644 |
1708 | --- a/arch/x86/Makefile |
1709 | +++ b/arch/x86/Makefile |
1710 | @@ -221,9 +221,7 @@ ifdef CONFIG_RETPOLINE |
1711 | RETPOLINE_CFLAGS_CLANG := -mretpoline-external-thunk |
1712 | |
1713 | RETPOLINE_CFLAGS += $(call cc-option,$(RETPOLINE_CFLAGS_GCC),$(call cc-option,$(RETPOLINE_CFLAGS_CLANG))) |
1714 | - ifneq ($(RETPOLINE_CFLAGS),) |
1715 | - KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE |
1716 | - endif |
1717 | + KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) |
1718 | endif |
1719 | |
1720 | archscripts: scripts_basic |
1721 | @@ -239,6 +237,13 @@ archprepare: |
1722 | ifeq ($(CONFIG_KEXEC_FILE),y) |
1723 | $(Q)$(MAKE) $(build)=arch/x86/purgatory arch/x86/purgatory/kexec-purgatory.c |
1724 | endif |
1725 | +ifdef CONFIG_RETPOLINE |
1726 | +ifeq ($(RETPOLINE_CFLAGS),) |
1727 | + @echo "You are building kernel with non-retpoline compiler." >&2 |
1728 | + @echo "Please update your compiler." >&2 |
1729 | + @false |
1730 | +endif |
1731 | +endif |
1732 | |
1733 | ### |
1734 | # Kernel objects |
1735 | diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h |
1736 | index 8ceb7a8a249c7..5b197248d5465 100644 |
1737 | --- a/arch/x86/include/asm/cpufeatures.h |
1738 | +++ b/arch/x86/include/asm/cpufeatures.h |
1739 | @@ -195,7 +195,7 @@ |
1740 | #define X86_FEATURE_FENCE_SWAPGS_USER ( 7*32+10) /* "" LFENCE in user entry SWAPGS path */ |
1741 | #define X86_FEATURE_FENCE_SWAPGS_KERNEL ( 7*32+11) /* "" LFENCE in kernel entry SWAPGS path */ |
1742 | #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ |
1743 | -#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */ |
1744 | +#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCE for Spectre variant 2 */ |
1745 | |
1746 | #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ |
1747 | #define X86_FEATURE_SSBD ( 7*32+17) /* Speculative Store Bypass Disable */ |
1748 | diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h |
1749 | index 204a5ce65afda..19829b00e4fed 100644 |
1750 | --- a/arch/x86/include/asm/nospec-branch.h |
1751 | +++ b/arch/x86/include/asm/nospec-branch.h |
1752 | @@ -119,7 +119,7 @@ |
1753 | ANNOTATE_NOSPEC_ALTERNATIVE |
1754 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ |
1755 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ |
1756 | - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD |
1757 | + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_LFENCE |
1758 | #else |
1759 | jmp *\reg |
1760 | #endif |
1761 | @@ -130,7 +130,7 @@ |
1762 | ANNOTATE_NOSPEC_ALTERNATIVE |
1763 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ |
1764 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ |
1765 | - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD |
1766 | + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_LFENCE |
1767 | #else |
1768 | call *\reg |
1769 | #endif |
1770 | @@ -164,29 +164,35 @@ |
1771 | _ASM_PTR " 999b\n\t" \ |
1772 | ".popsection\n\t" |
1773 | |
1774 | -#if defined(CONFIG_X86_64) && defined(RETPOLINE) |
1775 | +#ifdef CONFIG_RETPOLINE |
1776 | +#ifdef CONFIG_X86_64 |
1777 | |
1778 | /* |
1779 | - * Since the inline asm uses the %V modifier which is only in newer GCC, |
1780 | - * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. |
1781 | + * Inline asm uses the %V modifier which is only in newer GCC |
1782 | + * which is ensured when CONFIG_RETPOLINE is defined. |
1783 | */ |
1784 | # define CALL_NOSPEC \ |
1785 | ANNOTATE_NOSPEC_ALTERNATIVE \ |
1786 | - ALTERNATIVE( \ |
1787 | + ALTERNATIVE_2( \ |
1788 | ANNOTATE_RETPOLINE_SAFE \ |
1789 | "call *%[thunk_target]\n", \ |
1790 | "call __x86_indirect_thunk_%V[thunk_target]\n", \ |
1791 | - X86_FEATURE_RETPOLINE) |
1792 | + X86_FEATURE_RETPOLINE, \ |
1793 | + "lfence;\n" \ |
1794 | + ANNOTATE_RETPOLINE_SAFE \ |
1795 | + "call *%[thunk_target]\n", \ |
1796 | + X86_FEATURE_RETPOLINE_LFENCE) |
1797 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) |
1798 | |
1799 | -#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) |
1800 | +#else /* CONFIG_X86_32 */ |
1801 | /* |
1802 | * For i386 we use the original ret-equivalent retpoline, because |
1803 | * otherwise we'll run out of registers. We don't care about CET |
1804 | * here, anyway. |
1805 | */ |
1806 | # define CALL_NOSPEC \ |
1807 | - ALTERNATIVE( \ |
1808 | + ANNOTATE_NOSPEC_ALTERNATIVE \ |
1809 | + ALTERNATIVE_2( \ |
1810 | ANNOTATE_RETPOLINE_SAFE \ |
1811 | "call *%[thunk_target]\n", \ |
1812 | " jmp 904f;\n" \ |
1813 | @@ -201,9 +207,14 @@ |
1814 | " ret;\n" \ |
1815 | " .align 16\n" \ |
1816 | "904: call 901b;\n", \ |
1817 | - X86_FEATURE_RETPOLINE) |
1818 | + X86_FEATURE_RETPOLINE, \ |
1819 | + "lfence;\n" \ |
1820 | + ANNOTATE_RETPOLINE_SAFE \ |
1821 | + "call *%[thunk_target]\n", \ |
1822 | + X86_FEATURE_RETPOLINE_LFENCE) |
1823 | |
1824 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) |
1825 | +#endif |
1826 | #else /* No retpoline for C / inline asm */ |
1827 | # define CALL_NOSPEC "call *%[thunk_target]\n" |
1828 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) |
1829 | @@ -212,11 +223,11 @@ |
1830 | /* The Spectre V2 mitigation variants */ |
1831 | enum spectre_v2_mitigation { |
1832 | SPECTRE_V2_NONE, |
1833 | - SPECTRE_V2_RETPOLINE_MINIMAL, |
1834 | - SPECTRE_V2_RETPOLINE_MINIMAL_AMD, |
1835 | - SPECTRE_V2_RETPOLINE_GENERIC, |
1836 | - SPECTRE_V2_RETPOLINE_AMD, |
1837 | - SPECTRE_V2_IBRS_ENHANCED, |
1838 | + SPECTRE_V2_RETPOLINE, |
1839 | + SPECTRE_V2_LFENCE, |
1840 | + SPECTRE_V2_EIBRS, |
1841 | + SPECTRE_V2_EIBRS_RETPOLINE, |
1842 | + SPECTRE_V2_EIBRS_LFENCE, |
1843 | }; |
1844 | |
1845 | /* The indirect branch speculation control variants */ |
1846 | diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c |
1847 | index a884bb7e7b01d..94aa0206b1f98 100644 |
1848 | --- a/arch/x86/kernel/cpu/bugs.c |
1849 | +++ b/arch/x86/kernel/cpu/bugs.c |
1850 | @@ -30,6 +30,7 @@ |
1851 | #include <asm/cacheflush.h> |
1852 | #include <asm/intel-family.h> |
1853 | #include <asm/e820.h> |
1854 | +#include <linux/bpf.h> |
1855 | |
1856 | #include "cpu.h" |
1857 | |
1858 | @@ -585,7 +586,7 @@ static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = |
1859 | static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = |
1860 | SPECTRE_V2_USER_NONE; |
1861 | |
1862 | -#ifdef RETPOLINE |
1863 | +#ifdef CONFIG_RETPOLINE |
1864 | static bool spectre_v2_bad_module; |
1865 | |
1866 | bool retpoline_module_ok(bool has_retpoline) |
1867 | @@ -606,6 +607,32 @@ static inline const char *spectre_v2_module_string(void) |
1868 | static inline const char *spectre_v2_module_string(void) { return ""; } |
1869 | #endif |
1870 | |
1871 | +#define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" |
1872 | +#define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" |
1873 | +#define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" |
1874 | + |
1875 | +#ifdef CONFIG_BPF_SYSCALL |
1876 | +void unpriv_ebpf_notify(int new_state) |
1877 | +{ |
1878 | + if (new_state) |
1879 | + return; |
1880 | + |
1881 | + /* Unprivileged eBPF is enabled */ |
1882 | + |
1883 | + switch (spectre_v2_enabled) { |
1884 | + case SPECTRE_V2_EIBRS: |
1885 | + pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); |
1886 | + break; |
1887 | + case SPECTRE_V2_EIBRS_LFENCE: |
1888 | + if (sched_smt_active()) |
1889 | + pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); |
1890 | + break; |
1891 | + default: |
1892 | + break; |
1893 | + } |
1894 | +} |
1895 | +#endif |
1896 | + |
1897 | static inline bool match_option(const char *arg, int arglen, const char *opt) |
1898 | { |
1899 | int len = strlen(opt); |
1900 | @@ -620,7 +647,10 @@ enum spectre_v2_mitigation_cmd { |
1901 | SPECTRE_V2_CMD_FORCE, |
1902 | SPECTRE_V2_CMD_RETPOLINE, |
1903 | SPECTRE_V2_CMD_RETPOLINE_GENERIC, |
1904 | - SPECTRE_V2_CMD_RETPOLINE_AMD, |
1905 | + SPECTRE_V2_CMD_RETPOLINE_LFENCE, |
1906 | + SPECTRE_V2_CMD_EIBRS, |
1907 | + SPECTRE_V2_CMD_EIBRS_RETPOLINE, |
1908 | + SPECTRE_V2_CMD_EIBRS_LFENCE, |
1909 | }; |
1910 | |
1911 | enum spectre_v2_user_cmd { |
1912 | @@ -693,6 +723,13 @@ spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd) |
1913 | return SPECTRE_V2_USER_CMD_AUTO; |
1914 | } |
1915 | |
1916 | +static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode) |
1917 | +{ |
1918 | + return (mode == SPECTRE_V2_EIBRS || |
1919 | + mode == SPECTRE_V2_EIBRS_RETPOLINE || |
1920 | + mode == SPECTRE_V2_EIBRS_LFENCE); |
1921 | +} |
1922 | + |
1923 | static void __init |
1924 | spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) |
1925 | { |
1926 | @@ -755,10 +792,12 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) |
1927 | } |
1928 | |
1929 | /* |
1930 | - * If enhanced IBRS is enabled or SMT impossible, STIBP is not |
1931 | + * If no STIBP, enhanced IBRS is enabled or SMT impossible, STIBP is not |
1932 | * required. |
1933 | */ |
1934 | - if (!smt_possible || spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) |
1935 | + if (!boot_cpu_has(X86_FEATURE_STIBP) || |
1936 | + !smt_possible || |
1937 | + spectre_v2_in_eibrs_mode(spectre_v2_enabled)) |
1938 | return; |
1939 | |
1940 | /* |
1941 | @@ -770,12 +809,6 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) |
1942 | boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) |
1943 | mode = SPECTRE_V2_USER_STRICT_PREFERRED; |
1944 | |
1945 | - /* |
1946 | - * If STIBP is not available, clear the STIBP mode. |
1947 | - */ |
1948 | - if (!boot_cpu_has(X86_FEATURE_STIBP)) |
1949 | - mode = SPECTRE_V2_USER_NONE; |
1950 | - |
1951 | spectre_v2_user_stibp = mode; |
1952 | |
1953 | set_mode: |
1954 | @@ -784,11 +817,11 @@ set_mode: |
1955 | |
1956 | static const char * const spectre_v2_strings[] = { |
1957 | [SPECTRE_V2_NONE] = "Vulnerable", |
1958 | - [SPECTRE_V2_RETPOLINE_MINIMAL] = "Vulnerable: Minimal generic ASM retpoline", |
1959 | - [SPECTRE_V2_RETPOLINE_MINIMAL_AMD] = "Vulnerable: Minimal AMD ASM retpoline", |
1960 | - [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", |
1961 | - [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", |
1962 | - [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", |
1963 | + [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", |
1964 | + [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", |
1965 | + [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS", |
1966 | + [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE", |
1967 | + [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines", |
1968 | }; |
1969 | |
1970 | static const struct { |
1971 | @@ -799,8 +832,12 @@ static const struct { |
1972 | { "off", SPECTRE_V2_CMD_NONE, false }, |
1973 | { "on", SPECTRE_V2_CMD_FORCE, true }, |
1974 | { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, |
1975 | - { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false }, |
1976 | + { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, |
1977 | + { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, |
1978 | { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, |
1979 | + { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, |
1980 | + { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, |
1981 | + { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, |
1982 | { "auto", SPECTRE_V2_CMD_AUTO, false }, |
1983 | }; |
1984 | |
1985 | @@ -810,11 +847,6 @@ static void __init spec_v2_print_cond(const char *reason, bool secure) |
1986 | pr_info("%s selected on command line.\n", reason); |
1987 | } |
1988 | |
1989 | -static inline bool retp_compiler(void) |
1990 | -{ |
1991 | - return __is_defined(RETPOLINE); |
1992 | -} |
1993 | - |
1994 | static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) |
1995 | { |
1996 | enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; |
1997 | @@ -842,16 +874,30 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) |
1998 | } |
1999 | |
2000 | if ((cmd == SPECTRE_V2_CMD_RETPOLINE || |
2001 | - cmd == SPECTRE_V2_CMD_RETPOLINE_AMD || |
2002 | - cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && |
2003 | + cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || |
2004 | + cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || |
2005 | + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || |
2006 | + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && |
2007 | !IS_ENABLED(CONFIG_RETPOLINE)) { |
2008 | - pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); |
2009 | + pr_err("%s selected but not compiled in. Switching to AUTO select\n", |
2010 | + mitigation_options[i].option); |
2011 | + return SPECTRE_V2_CMD_AUTO; |
2012 | + } |
2013 | + |
2014 | + if ((cmd == SPECTRE_V2_CMD_EIBRS || |
2015 | + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || |
2016 | + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && |
2017 | + !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { |
2018 | + pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n", |
2019 | + mitigation_options[i].option); |
2020 | return SPECTRE_V2_CMD_AUTO; |
2021 | } |
2022 | |
2023 | - if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD && |
2024 | - boot_cpu_data.x86_vendor != X86_VENDOR_AMD) { |
2025 | - pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n"); |
2026 | + if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || |
2027 | + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && |
2028 | + !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { |
2029 | + pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", |
2030 | + mitigation_options[i].option); |
2031 | return SPECTRE_V2_CMD_AUTO; |
2032 | } |
2033 | |
2034 | @@ -860,6 +906,16 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) |
2035 | return cmd; |
2036 | } |
2037 | |
2038 | +static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) |
2039 | +{ |
2040 | + if (!IS_ENABLED(CONFIG_RETPOLINE)) { |
2041 | + pr_err("Kernel not compiled with retpoline; no mitigation available!"); |
2042 | + return SPECTRE_V2_NONE; |
2043 | + } |
2044 | + |
2045 | + return SPECTRE_V2_RETPOLINE; |
2046 | +} |
2047 | + |
2048 | static void __init spectre_v2_select_mitigation(void) |
2049 | { |
2050 | enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); |
2051 | @@ -880,50 +936,64 @@ static void __init spectre_v2_select_mitigation(void) |
2052 | case SPECTRE_V2_CMD_FORCE: |
2053 | case SPECTRE_V2_CMD_AUTO: |
2054 | if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { |
2055 | - mode = SPECTRE_V2_IBRS_ENHANCED; |
2056 | - /* Force it so VMEXIT will restore correctly */ |
2057 | - x86_spec_ctrl_base |= SPEC_CTRL_IBRS; |
2058 | - wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); |
2059 | - goto specv2_set_mode; |
2060 | + mode = SPECTRE_V2_EIBRS; |
2061 | + break; |
2062 | } |
2063 | - if (IS_ENABLED(CONFIG_RETPOLINE)) |
2064 | - goto retpoline_auto; |
2065 | + |
2066 | + mode = spectre_v2_select_retpoline(); |
2067 | break; |
2068 | - case SPECTRE_V2_CMD_RETPOLINE_AMD: |
2069 | - if (IS_ENABLED(CONFIG_RETPOLINE)) |
2070 | - goto retpoline_amd; |
2071 | + |
2072 | + case SPECTRE_V2_CMD_RETPOLINE_LFENCE: |
2073 | + pr_err(SPECTRE_V2_LFENCE_MSG); |
2074 | + mode = SPECTRE_V2_LFENCE; |
2075 | break; |
2076 | + |
2077 | case SPECTRE_V2_CMD_RETPOLINE_GENERIC: |
2078 | - if (IS_ENABLED(CONFIG_RETPOLINE)) |
2079 | - goto retpoline_generic; |
2080 | + mode = SPECTRE_V2_RETPOLINE; |
2081 | break; |
2082 | + |
2083 | case SPECTRE_V2_CMD_RETPOLINE: |
2084 | - if (IS_ENABLED(CONFIG_RETPOLINE)) |
2085 | - goto retpoline_auto; |
2086 | + mode = spectre_v2_select_retpoline(); |
2087 | + break; |
2088 | + |
2089 | + case SPECTRE_V2_CMD_EIBRS: |
2090 | + mode = SPECTRE_V2_EIBRS; |
2091 | + break; |
2092 | + |
2093 | + case SPECTRE_V2_CMD_EIBRS_LFENCE: |
2094 | + mode = SPECTRE_V2_EIBRS_LFENCE; |
2095 | + break; |
2096 | + |
2097 | + case SPECTRE_V2_CMD_EIBRS_RETPOLINE: |
2098 | + mode = SPECTRE_V2_EIBRS_RETPOLINE; |
2099 | break; |
2100 | } |
2101 | - pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!"); |
2102 | - return; |
2103 | |
2104 | -retpoline_auto: |
2105 | - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { |
2106 | - retpoline_amd: |
2107 | - if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { |
2108 | - pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); |
2109 | - goto retpoline_generic; |
2110 | - } |
2111 | - mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD : |
2112 | - SPECTRE_V2_RETPOLINE_MINIMAL_AMD; |
2113 | - setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); |
2114 | - setup_force_cpu_cap(X86_FEATURE_RETPOLINE); |
2115 | - } else { |
2116 | - retpoline_generic: |
2117 | - mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC : |
2118 | - SPECTRE_V2_RETPOLINE_MINIMAL; |
2119 | + if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) |
2120 | + pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); |
2121 | + |
2122 | + if (spectre_v2_in_eibrs_mode(mode)) { |
2123 | + /* Force it so VMEXIT will restore correctly */ |
2124 | + x86_spec_ctrl_base |= SPEC_CTRL_IBRS; |
2125 | + wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); |
2126 | + } |
2127 | + |
2128 | + switch (mode) { |
2129 | + case SPECTRE_V2_NONE: |
2130 | + case SPECTRE_V2_EIBRS: |
2131 | + break; |
2132 | + |
2133 | + case SPECTRE_V2_LFENCE: |
2134 | + case SPECTRE_V2_EIBRS_LFENCE: |
2135 | + setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); |
2136 | + /* fallthrough */ |
2137 | + |
2138 | + case SPECTRE_V2_RETPOLINE: |
2139 | + case SPECTRE_V2_EIBRS_RETPOLINE: |
2140 | setup_force_cpu_cap(X86_FEATURE_RETPOLINE); |
2141 | + break; |
2142 | } |
2143 | |
2144 | -specv2_set_mode: |
2145 | spectre_v2_enabled = mode; |
2146 | pr_info("%s\n", spectre_v2_strings[mode]); |
2147 | |
2148 | @@ -949,7 +1019,7 @@ specv2_set_mode: |
2149 | * the CPU supports Enhanced IBRS, kernel might un-intentionally not |
2150 | * enable IBRS around firmware calls. |
2151 | */ |
2152 | - if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) { |
2153 | + if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) { |
2154 | setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); |
2155 | pr_info("Enabling Restricted Speculation for firmware calls\n"); |
2156 | } |
2157 | @@ -1019,6 +1089,10 @@ void arch_smt_update(void) |
2158 | { |
2159 | mutex_lock(&spec_ctrl_mutex); |
2160 | |
2161 | + if (sched_smt_active() && unprivileged_ebpf_enabled() && |
2162 | + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) |
2163 | + pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); |
2164 | + |
2165 | switch (spectre_v2_user_stibp) { |
2166 | case SPECTRE_V2_USER_NONE: |
2167 | break; |
2168 | @@ -1263,7 +1337,6 @@ static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) |
2169 | if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && |
2170 | spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) |
2171 | return 0; |
2172 | - |
2173 | /* |
2174 | * With strict mode for both IBPB and STIBP, the instruction |
2175 | * code paths avoid checking this task flag and instead, |
2176 | @@ -1610,7 +1683,7 @@ static ssize_t tsx_async_abort_show_state(char *buf) |
2177 | |
2178 | static char *stibp_state(void) |
2179 | { |
2180 | - if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) |
2181 | + if (spectre_v2_in_eibrs_mode(spectre_v2_enabled)) |
2182 | return ""; |
2183 | |
2184 | switch (spectre_v2_user_stibp) { |
2185 | @@ -1640,6 +1713,27 @@ static char *ibpb_state(void) |
2186 | return ""; |
2187 | } |
2188 | |
2189 | +static ssize_t spectre_v2_show_state(char *buf) |
2190 | +{ |
2191 | + if (spectre_v2_enabled == SPECTRE_V2_LFENCE) |
2192 | + return sprintf(buf, "Vulnerable: LFENCE\n"); |
2193 | + |
2194 | + if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) |
2195 | + return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); |
2196 | + |
2197 | + if (sched_smt_active() && unprivileged_ebpf_enabled() && |
2198 | + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) |
2199 | + return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); |
2200 | + |
2201 | + return sprintf(buf, "%s%s%s%s%s%s\n", |
2202 | + spectre_v2_strings[spectre_v2_enabled], |
2203 | + ibpb_state(), |
2204 | + boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", |
2205 | + stibp_state(), |
2206 | + boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", |
2207 | + spectre_v2_module_string()); |
2208 | +} |
2209 | + |
2210 | static ssize_t srbds_show_state(char *buf) |
2211 | { |
2212 | return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]); |
2213 | @@ -1662,12 +1756,7 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr |
2214 | return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); |
2215 | |
2216 | case X86_BUG_SPECTRE_V2: |
2217 | - return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], |
2218 | - ibpb_state(), |
2219 | - boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", |
2220 | - stibp_state(), |
2221 | - boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", |
2222 | - spectre_v2_module_string()); |
2223 | + return spectre_v2_show_state(buf); |
2224 | |
2225 | case X86_BUG_SPEC_STORE_BYPASS: |
2226 | return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); |
2227 | diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c |
2228 | index d420597b0d2b4..17ea0ba50278d 100644 |
2229 | --- a/drivers/block/xen-blkfront.c |
2230 | +++ b/drivers/block/xen-blkfront.c |
2231 | @@ -1266,17 +1266,16 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo) |
2232 | list_for_each_entry_safe(persistent_gnt, n, |
2233 | &rinfo->grants, node) { |
2234 | list_del(&persistent_gnt->node); |
2235 | - if (persistent_gnt->gref != GRANT_INVALID_REF) { |
2236 | - gnttab_end_foreign_access(persistent_gnt->gref, |
2237 | - 0, 0UL); |
2238 | - rinfo->persistent_gnts_c--; |
2239 | - } |
2240 | + if (persistent_gnt->gref == GRANT_INVALID_REF || |
2241 | + !gnttab_try_end_foreign_access(persistent_gnt->gref)) |
2242 | + continue; |
2243 | + |
2244 | + rinfo->persistent_gnts_c--; |
2245 | if (info->feature_persistent) |
2246 | __free_page(persistent_gnt->page); |
2247 | kfree(persistent_gnt); |
2248 | } |
2249 | } |
2250 | - BUG_ON(rinfo->persistent_gnts_c != 0); |
2251 | |
2252 | for (i = 0; i < BLK_RING_SIZE(info); i++) { |
2253 | /* |
2254 | @@ -1333,7 +1332,8 @@ free_shadow: |
2255 | rinfo->ring_ref[i] = GRANT_INVALID_REF; |
2256 | } |
2257 | } |
2258 | - free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * XEN_PAGE_SIZE)); |
2259 | + free_pages_exact(rinfo->ring.sring, |
2260 | + info->nr_ring_pages * XEN_PAGE_SIZE); |
2261 | rinfo->ring.sring = NULL; |
2262 | |
2263 | if (rinfo->irq) |
2264 | @@ -1417,9 +1417,15 @@ static int blkif_get_final_status(enum blk_req_status s1, |
2265 | return BLKIF_RSP_OKAY; |
2266 | } |
2267 | |
2268 | -static bool blkif_completion(unsigned long *id, |
2269 | - struct blkfront_ring_info *rinfo, |
2270 | - struct blkif_response *bret) |
2271 | +/* |
2272 | + * Return values: |
2273 | + * 1 response processed. |
2274 | + * 0 missing further responses. |
2275 | + * -1 error while processing. |
2276 | + */ |
2277 | +static int blkif_completion(unsigned long *id, |
2278 | + struct blkfront_ring_info *rinfo, |
2279 | + struct blkif_response *bret) |
2280 | { |
2281 | int i = 0; |
2282 | struct scatterlist *sg; |
2283 | @@ -1493,42 +1499,43 @@ static bool blkif_completion(unsigned long *id, |
2284 | } |
2285 | /* Add the persistent grant into the list of free grants */ |
2286 | for (i = 0; i < num_grant; i++) { |
2287 | - if (gnttab_query_foreign_access(s->grants_used[i]->gref)) { |
2288 | + if (!gnttab_try_end_foreign_access(s->grants_used[i]->gref)) { |
2289 | /* |
2290 | * If the grant is still mapped by the backend (the |
2291 | * backend has chosen to make this grant persistent) |
2292 | * we add it at the head of the list, so it will be |
2293 | * reused first. |
2294 | */ |
2295 | - if (!info->feature_persistent) |
2296 | - pr_alert_ratelimited("backed has not unmapped grant: %u\n", |
2297 | - s->grants_used[i]->gref); |
2298 | + if (!info->feature_persistent) { |
2299 | + pr_alert("backed has not unmapped grant: %u\n", |
2300 | + s->grants_used[i]->gref); |
2301 | + return -1; |
2302 | + } |
2303 | list_add(&s->grants_used[i]->node, &rinfo->grants); |
2304 | rinfo->persistent_gnts_c++; |
2305 | } else { |
2306 | /* |
2307 | - * If the grant is not mapped by the backend we end the |
2308 | - * foreign access and add it to the tail of the list, |
2309 | - * so it will not be picked again unless we run out of |
2310 | - * persistent grants. |
2311 | + * If the grant is not mapped by the backend we add it |
2312 | + * to the tail of the list, so it will not be picked |
2313 | + * again unless we run out of persistent grants. |
2314 | */ |
2315 | - gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL); |
2316 | s->grants_used[i]->gref = GRANT_INVALID_REF; |
2317 | list_add_tail(&s->grants_used[i]->node, &rinfo->grants); |
2318 | } |
2319 | } |
2320 | if (s->req.operation == BLKIF_OP_INDIRECT) { |
2321 | for (i = 0; i < INDIRECT_GREFS(num_grant); i++) { |
2322 | - if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) { |
2323 | - if (!info->feature_persistent) |
2324 | - pr_alert_ratelimited("backed has not unmapped grant: %u\n", |
2325 | - s->indirect_grants[i]->gref); |
2326 | + if (!gnttab_try_end_foreign_access(s->indirect_grants[i]->gref)) { |
2327 | + if (!info->feature_persistent) { |
2328 | + pr_alert("backed has not unmapped grant: %u\n", |
2329 | + s->indirect_grants[i]->gref); |
2330 | + return -1; |
2331 | + } |
2332 | list_add(&s->indirect_grants[i]->node, &rinfo->grants); |
2333 | rinfo->persistent_gnts_c++; |
2334 | } else { |
2335 | struct page *indirect_page; |
2336 | |
2337 | - gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL); |
2338 | /* |
2339 | * Add the used indirect page back to the list of |
2340 | * available pages for indirect grefs. |
2341 | @@ -1610,12 +1617,17 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) |
2342 | } |
2343 | |
2344 | if (bret.operation != BLKIF_OP_DISCARD) { |
2345 | + int ret; |
2346 | + |
2347 | /* |
2348 | * We may need to wait for an extra response if the |
2349 | * I/O request is split in 2 |
2350 | */ |
2351 | - if (!blkif_completion(&id, rinfo, &bret)) |
2352 | + ret = blkif_completion(&id, rinfo, &bret); |
2353 | + if (!ret) |
2354 | continue; |
2355 | + if (unlikely(ret < 0)) |
2356 | + goto err; |
2357 | } |
2358 | |
2359 | if (add_id_to_freelist(rinfo, id)) { |
2360 | @@ -1717,8 +1729,7 @@ static int setup_blkring(struct xenbus_device *dev, |
2361 | for (i = 0; i < info->nr_ring_pages; i++) |
2362 | rinfo->ring_ref[i] = GRANT_INVALID_REF; |
2363 | |
2364 | - sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH, |
2365 | - get_order(ring_size)); |
2366 | + sring = alloc_pages_exact(ring_size, GFP_NOIO); |
2367 | if (!sring) { |
2368 | xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring"); |
2369 | return -ENOMEM; |
2370 | @@ -1728,7 +1739,7 @@ static int setup_blkring(struct xenbus_device *dev, |
2371 | |
2372 | err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref); |
2373 | if (err < 0) { |
2374 | - free_pages((unsigned long)sring, get_order(ring_size)); |
2375 | + free_pages_exact(sring, ring_size); |
2376 | rinfo->ring.sring = NULL; |
2377 | goto fail; |
2378 | } |
2379 | diff --git a/drivers/firmware/psci.c b/drivers/firmware/psci.c |
2380 | index 79a48c37fb35b..2a6d9572d6397 100644 |
2381 | --- a/drivers/firmware/psci.c |
2382 | +++ b/drivers/firmware/psci.c |
2383 | @@ -64,6 +64,21 @@ struct psci_operations psci_ops = { |
2384 | .smccc_version = SMCCC_VERSION_1_0, |
2385 | }; |
2386 | |
2387 | +enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void) |
2388 | +{ |
2389 | + if (psci_ops.smccc_version < SMCCC_VERSION_1_1) |
2390 | + return SMCCC_CONDUIT_NONE; |
2391 | + |
2392 | + switch (psci_ops.conduit) { |
2393 | + case PSCI_CONDUIT_SMC: |
2394 | + return SMCCC_CONDUIT_SMC; |
2395 | + case PSCI_CONDUIT_HVC: |
2396 | + return SMCCC_CONDUIT_HVC; |
2397 | + default: |
2398 | + return SMCCC_CONDUIT_NONE; |
2399 | + } |
2400 | +} |
2401 | + |
2402 | typedef unsigned long (psci_fn)(unsigned long, unsigned long, |
2403 | unsigned long, unsigned long); |
2404 | static psci_fn *invoke_psci_fn; |
2405 | diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c |
2406 | index 65a50bc5661d2..82dcd44b3e5e2 100644 |
2407 | --- a/drivers/net/xen-netfront.c |
2408 | +++ b/drivers/net/xen-netfront.c |
2409 | @@ -413,14 +413,12 @@ static bool xennet_tx_buf_gc(struct netfront_queue *queue) |
2410 | queue->tx_link[id] = TX_LINK_NONE; |
2411 | skb = queue->tx_skbs[id]; |
2412 | queue->tx_skbs[id] = NULL; |
2413 | - if (unlikely(gnttab_query_foreign_access( |
2414 | - queue->grant_tx_ref[id]) != 0)) { |
2415 | + if (unlikely(!gnttab_end_foreign_access_ref( |
2416 | + queue->grant_tx_ref[id], GNTMAP_readonly))) { |
2417 | dev_alert(dev, |
2418 | "Grant still in use by backend domain\n"); |
2419 | goto err; |
2420 | } |
2421 | - gnttab_end_foreign_access_ref( |
2422 | - queue->grant_tx_ref[id], GNTMAP_readonly); |
2423 | gnttab_release_grant_reference( |
2424 | &queue->gref_tx_head, queue->grant_tx_ref[id]); |
2425 | queue->grant_tx_ref[id] = GRANT_INVALID_REF; |
2426 | @@ -840,7 +838,6 @@ static int xennet_get_responses(struct netfront_queue *queue, |
2427 | int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD); |
2428 | int slots = 1; |
2429 | int err = 0; |
2430 | - unsigned long ret; |
2431 | |
2432 | if (rx->flags & XEN_NETRXF_extra_info) { |
2433 | err = xennet_get_extras(queue, extras, rp); |
2434 | @@ -871,8 +868,13 @@ static int xennet_get_responses(struct netfront_queue *queue, |
2435 | goto next; |
2436 | } |
2437 | |
2438 | - ret = gnttab_end_foreign_access_ref(ref, 0); |
2439 | - BUG_ON(!ret); |
2440 | + if (!gnttab_end_foreign_access_ref(ref, 0)) { |
2441 | + dev_alert(dev, |
2442 | + "Grant still in use by backend domain\n"); |
2443 | + queue->info->broken = true; |
2444 | + dev_alert(dev, "Disabled for further use\n"); |
2445 | + return -EINVAL; |
2446 | + } |
2447 | |
2448 | gnttab_release_grant_reference(&queue->gref_rx_head, ref); |
2449 | |
2450 | @@ -1076,6 +1078,10 @@ static int xennet_poll(struct napi_struct *napi, int budget) |
2451 | err = xennet_get_responses(queue, &rinfo, rp, &tmpq); |
2452 | |
2453 | if (unlikely(err)) { |
2454 | + if (queue->info->broken) { |
2455 | + spin_unlock(&queue->rx_lock); |
2456 | + return 0; |
2457 | + } |
2458 | err: |
2459 | while ((skb = __skb_dequeue(&tmpq))) |
2460 | __skb_queue_tail(&errq, skb); |
2461 | @@ -1673,7 +1679,7 @@ static int setup_netfront(struct xenbus_device *dev, |
2462 | struct netfront_queue *queue, unsigned int feature_split_evtchn) |
2463 | { |
2464 | struct xen_netif_tx_sring *txs; |
2465 | - struct xen_netif_rx_sring *rxs; |
2466 | + struct xen_netif_rx_sring *rxs = NULL; |
2467 | grant_ref_t gref; |
2468 | int err; |
2469 | |
2470 | @@ -1693,21 +1699,21 @@ static int setup_netfront(struct xenbus_device *dev, |
2471 | |
2472 | err = xenbus_grant_ring(dev, txs, 1, &gref); |
2473 | if (err < 0) |
2474 | - goto grant_tx_ring_fail; |
2475 | + goto fail; |
2476 | queue->tx_ring_ref = gref; |
2477 | |
2478 | rxs = (struct xen_netif_rx_sring *)get_zeroed_page(GFP_NOIO | __GFP_HIGH); |
2479 | if (!rxs) { |
2480 | err = -ENOMEM; |
2481 | xenbus_dev_fatal(dev, err, "allocating rx ring page"); |
2482 | - goto alloc_rx_ring_fail; |
2483 | + goto fail; |
2484 | } |
2485 | SHARED_RING_INIT(rxs); |
2486 | FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); |
2487 | |
2488 | err = xenbus_grant_ring(dev, rxs, 1, &gref); |
2489 | if (err < 0) |
2490 | - goto grant_rx_ring_fail; |
2491 | + goto fail; |
2492 | queue->rx_ring_ref = gref; |
2493 | |
2494 | if (feature_split_evtchn) |
2495 | @@ -1720,22 +1726,28 @@ static int setup_netfront(struct xenbus_device *dev, |
2496 | err = setup_netfront_single(queue); |
2497 | |
2498 | if (err) |
2499 | - goto alloc_evtchn_fail; |
2500 | + goto fail; |
2501 | |
2502 | return 0; |
2503 | |
2504 | /* If we fail to setup netfront, it is safe to just revoke access to |
2505 | * granted pages because backend is not accessing it at this point. |
2506 | */ |
2507 | -alloc_evtchn_fail: |
2508 | - gnttab_end_foreign_access_ref(queue->rx_ring_ref, 0); |
2509 | -grant_rx_ring_fail: |
2510 | - free_page((unsigned long)rxs); |
2511 | -alloc_rx_ring_fail: |
2512 | - gnttab_end_foreign_access_ref(queue->tx_ring_ref, 0); |
2513 | -grant_tx_ring_fail: |
2514 | - free_page((unsigned long)txs); |
2515 | -fail: |
2516 | + fail: |
2517 | + if (queue->rx_ring_ref != GRANT_INVALID_REF) { |
2518 | + gnttab_end_foreign_access(queue->rx_ring_ref, 0, |
2519 | + (unsigned long)rxs); |
2520 | + queue->rx_ring_ref = GRANT_INVALID_REF; |
2521 | + } else { |
2522 | + free_page((unsigned long)rxs); |
2523 | + } |
2524 | + if (queue->tx_ring_ref != GRANT_INVALID_REF) { |
2525 | + gnttab_end_foreign_access(queue->tx_ring_ref, 0, |
2526 | + (unsigned long)txs); |
2527 | + queue->tx_ring_ref = GRANT_INVALID_REF; |
2528 | + } else { |
2529 | + free_page((unsigned long)txs); |
2530 | + } |
2531 | return err; |
2532 | } |
2533 | |
2534 | diff --git a/drivers/scsi/xen-scsifront.c b/drivers/scsi/xen-scsifront.c |
2535 | index e1b32ed0aa205..bdfe94c023dcd 100644 |
2536 | --- a/drivers/scsi/xen-scsifront.c |
2537 | +++ b/drivers/scsi/xen-scsifront.c |
2538 | @@ -210,12 +210,11 @@ static void scsifront_gnttab_done(struct vscsifrnt_info *info, uint32_t id) |
2539 | return; |
2540 | |
2541 | for (i = 0; i < s->nr_grants; i++) { |
2542 | - if (unlikely(gnttab_query_foreign_access(s->gref[i]) != 0)) { |
2543 | + if (unlikely(!gnttab_try_end_foreign_access(s->gref[i]))) { |
2544 | shost_printk(KERN_ALERT, info->host, KBUILD_MODNAME |
2545 | "grant still in use by backend\n"); |
2546 | BUG(); |
2547 | } |
2548 | - gnttab_end_foreign_access(s->gref[i], 0, 0UL); |
2549 | } |
2550 | |
2551 | kfree(s->sg); |
2552 | diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c |
2553 | index 7a47c4c9fb1bb..24f8900eccadd 100644 |
2554 | --- a/drivers/xen/gntalloc.c |
2555 | +++ b/drivers/xen/gntalloc.c |
2556 | @@ -166,20 +166,14 @@ undo: |
2557 | __del_gref(gref); |
2558 | } |
2559 | |
2560 | - /* It's possible for the target domain to map the just-allocated grant |
2561 | - * references by blindly guessing their IDs; if this is done, then |
2562 | - * __del_gref will leave them in the queue_gref list. They need to be |
2563 | - * added to the global list so that we can free them when they are no |
2564 | - * longer referenced. |
2565 | - */ |
2566 | - if (unlikely(!list_empty(&queue_gref))) |
2567 | - list_splice_tail(&queue_gref, &gref_list); |
2568 | mutex_unlock(&gref_mutex); |
2569 | return rc; |
2570 | } |
2571 | |
2572 | static void __del_gref(struct gntalloc_gref *gref) |
2573 | { |
2574 | + unsigned long addr; |
2575 | + |
2576 | if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { |
2577 | uint8_t *tmp = kmap(gref->page); |
2578 | tmp[gref->notify.pgoff] = 0; |
2579 | @@ -193,21 +187,16 @@ static void __del_gref(struct gntalloc_gref *gref) |
2580 | gref->notify.flags = 0; |
2581 | |
2582 | if (gref->gref_id) { |
2583 | - if (gnttab_query_foreign_access(gref->gref_id)) |
2584 | - return; |
2585 | - |
2586 | - if (!gnttab_end_foreign_access_ref(gref->gref_id, 0)) |
2587 | - return; |
2588 | - |
2589 | - gnttab_free_grant_reference(gref->gref_id); |
2590 | + if (gref->page) { |
2591 | + addr = (unsigned long)page_to_virt(gref->page); |
2592 | + gnttab_end_foreign_access(gref->gref_id, 0, addr); |
2593 | + } else |
2594 | + gnttab_free_grant_reference(gref->gref_id); |
2595 | } |
2596 | |
2597 | gref_size--; |
2598 | list_del(&gref->next_gref); |
2599 | |
2600 | - if (gref->page) |
2601 | - __free_page(gref->page); |
2602 | - |
2603 | kfree(gref); |
2604 | } |
2605 | |
2606 | diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c |
2607 | index 775d4195966c4..02754b4923e96 100644 |
2608 | --- a/drivers/xen/grant-table.c |
2609 | +++ b/drivers/xen/grant-table.c |
2610 | @@ -114,12 +114,9 @@ struct gnttab_ops { |
2611 | */ |
2612 | unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref); |
2613 | /* |
2614 | - * Query the status of a grant entry. Ref parameter is reference of |
2615 | - * queried grant entry, return value is the status of queried entry. |
2616 | - * Detailed status(writing/reading) can be gotten from the return value |
2617 | - * by bit operations. |
2618 | + * Read the frame number related to a given grant reference. |
2619 | */ |
2620 | - int (*query_foreign_access)(grant_ref_t ref); |
2621 | + unsigned long (*read_frame)(grant_ref_t ref); |
2622 | }; |
2623 | |
2624 | struct unmap_refs_callback_data { |
2625 | @@ -254,17 +251,6 @@ int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, |
2626 | } |
2627 | EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access); |
2628 | |
2629 | -static int gnttab_query_foreign_access_v1(grant_ref_t ref) |
2630 | -{ |
2631 | - return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing); |
2632 | -} |
2633 | - |
2634 | -int gnttab_query_foreign_access(grant_ref_t ref) |
2635 | -{ |
2636 | - return gnttab_interface->query_foreign_access(ref); |
2637 | -} |
2638 | -EXPORT_SYMBOL_GPL(gnttab_query_foreign_access); |
2639 | - |
2640 | static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly) |
2641 | { |
2642 | u16 flags, nflags; |
2643 | @@ -295,6 +281,11 @@ int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) |
2644 | } |
2645 | EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref); |
2646 | |
2647 | +static unsigned long gnttab_read_frame_v1(grant_ref_t ref) |
2648 | +{ |
2649 | + return gnttab_shared.v1[ref].frame; |
2650 | +} |
2651 | + |
2652 | struct deferred_entry { |
2653 | struct list_head list; |
2654 | grant_ref_t ref; |
2655 | @@ -324,12 +315,9 @@ static void gnttab_handle_deferred(unsigned long unused) |
2656 | spin_unlock_irqrestore(&gnttab_list_lock, flags); |
2657 | if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) { |
2658 | put_free_entry(entry->ref); |
2659 | - if (entry->page) { |
2660 | - pr_debug("freeing g.e. %#x (pfn %#lx)\n", |
2661 | - entry->ref, page_to_pfn(entry->page)); |
2662 | - put_page(entry->page); |
2663 | - } else |
2664 | - pr_info("freeing g.e. %#x\n", entry->ref); |
2665 | + pr_debug("freeing g.e. %#x (pfn %#lx)\n", |
2666 | + entry->ref, page_to_pfn(entry->page)); |
2667 | + put_page(entry->page); |
2668 | kfree(entry); |
2669 | entry = NULL; |
2670 | } else { |
2671 | @@ -354,9 +342,18 @@ static void gnttab_handle_deferred(unsigned long unused) |
2672 | static void gnttab_add_deferred(grant_ref_t ref, bool readonly, |
2673 | struct page *page) |
2674 | { |
2675 | - struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC); |
2676 | + struct deferred_entry *entry; |
2677 | + gfp_t gfp = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL; |
2678 | const char *what = KERN_WARNING "leaking"; |
2679 | |
2680 | + entry = kmalloc(sizeof(*entry), gfp); |
2681 | + if (!page) { |
2682 | + unsigned long gfn = gnttab_interface->read_frame(ref); |
2683 | + |
2684 | + page = pfn_to_page(gfn_to_pfn(gfn)); |
2685 | + get_page(page); |
2686 | + } |
2687 | + |
2688 | if (entry) { |
2689 | unsigned long flags; |
2690 | |
2691 | @@ -377,11 +374,21 @@ static void gnttab_add_deferred(grant_ref_t ref, bool readonly, |
2692 | what, ref, page ? page_to_pfn(page) : -1); |
2693 | } |
2694 | |
2695 | +int gnttab_try_end_foreign_access(grant_ref_t ref) |
2696 | +{ |
2697 | + int ret = _gnttab_end_foreign_access_ref(ref, 0); |
2698 | + |
2699 | + if (ret) |
2700 | + put_free_entry(ref); |
2701 | + |
2702 | + return ret; |
2703 | +} |
2704 | +EXPORT_SYMBOL_GPL(gnttab_try_end_foreign_access); |
2705 | + |
2706 | void gnttab_end_foreign_access(grant_ref_t ref, int readonly, |
2707 | unsigned long page) |
2708 | { |
2709 | - if (gnttab_end_foreign_access_ref(ref, readonly)) { |
2710 | - put_free_entry(ref); |
2711 | + if (gnttab_try_end_foreign_access(ref)) { |
2712 | if (page != 0) |
2713 | put_page(virt_to_page(page)); |
2714 | } else |
2715 | @@ -1018,7 +1025,7 @@ static const struct gnttab_ops gnttab_v1_ops = { |
2716 | .update_entry = gnttab_update_entry_v1, |
2717 | .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1, |
2718 | .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1, |
2719 | - .query_foreign_access = gnttab_query_foreign_access_v1, |
2720 | + .read_frame = gnttab_read_frame_v1, |
2721 | }; |
2722 | |
2723 | static void gnttab_request_version(void) |
2724 | diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c |
2725 | index 8bbd887ca422b..5ee38e939165c 100644 |
2726 | --- a/drivers/xen/xenbus/xenbus_client.c |
2727 | +++ b/drivers/xen/xenbus/xenbus_client.c |
2728 | @@ -387,7 +387,14 @@ int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr, |
2729 | unsigned int nr_pages, grant_ref_t *grefs) |
2730 | { |
2731 | int err; |
2732 | - int i, j; |
2733 | + unsigned int i; |
2734 | + grant_ref_t gref_head; |
2735 | + |
2736 | + err = gnttab_alloc_grant_references(nr_pages, &gref_head); |
2737 | + if (err) { |
2738 | + xenbus_dev_fatal(dev, err, "granting access to ring page"); |
2739 | + return err; |
2740 | + } |
2741 | |
2742 | for (i = 0; i < nr_pages; i++) { |
2743 | unsigned long gfn; |
2744 | @@ -397,23 +404,14 @@ int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr, |
2745 | else |
2746 | gfn = virt_to_gfn(vaddr); |
2747 | |
2748 | - err = gnttab_grant_foreign_access(dev->otherend_id, gfn, 0); |
2749 | - if (err < 0) { |
2750 | - xenbus_dev_fatal(dev, err, |
2751 | - "granting access to ring page"); |
2752 | - goto fail; |
2753 | - } |
2754 | - grefs[i] = err; |
2755 | + grefs[i] = gnttab_claim_grant_reference(&gref_head); |
2756 | + gnttab_grant_foreign_access_ref(grefs[i], dev->otherend_id, |
2757 | + gfn, 0); |
2758 | |
2759 | vaddr = vaddr + XEN_PAGE_SIZE; |
2760 | } |
2761 | |
2762 | return 0; |
2763 | - |
2764 | -fail: |
2765 | - for (j = 0; j < i; j++) |
2766 | - gnttab_end_foreign_access_ref(grefs[j], 0); |
2767 | - return err; |
2768 | } |
2769 | EXPORT_SYMBOL_GPL(xenbus_grant_ring); |
2770 | |
2771 | diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h |
2772 | index 18863d56273cc..6366b04c7d5f4 100644 |
2773 | --- a/include/linux/arm-smccc.h |
2774 | +++ b/include/linux/arm-smccc.h |
2775 | @@ -89,6 +89,22 @@ |
2776 | |
2777 | #include <linux/linkage.h> |
2778 | #include <linux/types.h> |
2779 | + |
2780 | +enum arm_smccc_conduit { |
2781 | + SMCCC_CONDUIT_NONE, |
2782 | + SMCCC_CONDUIT_SMC, |
2783 | + SMCCC_CONDUIT_HVC, |
2784 | +}; |
2785 | + |
2786 | +/** |
2787 | + * arm_smccc_1_1_get_conduit() |
2788 | + * |
2789 | + * Returns the conduit to be used for SMCCCv1.1 or later. |
2790 | + * |
2791 | + * When SMCCCv1.1 is not present, returns SMCCC_CONDUIT_NONE. |
2792 | + */ |
2793 | +enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void); |
2794 | + |
2795 | /** |
2796 | * struct arm_smccc_res - Result from SMC/HVC call |
2797 | * @a0-a3 result values from registers 0 to 3 |
2798 | @@ -311,5 +327,63 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, |
2799 | #define SMCCC_RET_NOT_SUPPORTED -1 |
2800 | #define SMCCC_RET_NOT_REQUIRED -2 |
2801 | |
2802 | +/* |
2803 | + * Like arm_smccc_1_1* but always returns SMCCC_RET_NOT_SUPPORTED. |
2804 | + * Used when the SMCCC conduit is not defined. The empty asm statement |
2805 | + * avoids compiler warnings about unused variables. |
2806 | + */ |
2807 | +#define __fail_smccc_1_1(...) \ |
2808 | + do { \ |
2809 | + __declare_args(__count_args(__VA_ARGS__), __VA_ARGS__); \ |
2810 | + asm ("" __constraints(__count_args(__VA_ARGS__))); \ |
2811 | + if (___res) \ |
2812 | + ___res->a0 = SMCCC_RET_NOT_SUPPORTED; \ |
2813 | + } while (0) |
2814 | + |
2815 | +/* |
2816 | + * arm_smccc_1_1_invoke() - make an SMCCC v1.1 compliant call |
2817 | + * |
2818 | + * This is a variadic macro taking one to eight source arguments, and |
2819 | + * an optional return structure. |
2820 | + * |
2821 | + * @a0-a7: arguments passed in registers 0 to 7 |
2822 | + * @res: result values from registers 0 to 3 |
2823 | + * |
2824 | + * This macro will make either an HVC call or an SMC call depending on the |
2825 | + * current SMCCC conduit. If no valid conduit is available then -1 |
2826 | + * (SMCCC_RET_NOT_SUPPORTED) is returned in @res.a0 (if supplied). |
2827 | + * |
2828 | + * The return value also provides the conduit that was used. |
2829 | + */ |
2830 | +#define arm_smccc_1_1_invoke(...) ({ \ |
2831 | + int method = arm_smccc_1_1_get_conduit(); \ |
2832 | + switch (method) { \ |
2833 | + case SMCCC_CONDUIT_HVC: \ |
2834 | + arm_smccc_1_1_hvc(__VA_ARGS__); \ |
2835 | + break; \ |
2836 | + case SMCCC_CONDUIT_SMC: \ |
2837 | + arm_smccc_1_1_smc(__VA_ARGS__); \ |
2838 | + break; \ |
2839 | + default: \ |
2840 | + __fail_smccc_1_1(__VA_ARGS__); \ |
2841 | + method = SMCCC_CONDUIT_NONE; \ |
2842 | + break; \ |
2843 | + } \ |
2844 | + method; \ |
2845 | + }) |
2846 | + |
2847 | +/* Paravirtualised time calls (defined by ARM DEN0057A) */ |
2848 | +#define ARM_SMCCC_HV_PV_TIME_FEATURES \ |
2849 | + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ |
2850 | + ARM_SMCCC_SMC_64, \ |
2851 | + ARM_SMCCC_OWNER_STANDARD_HYP, \ |
2852 | + 0x20) |
2853 | + |
2854 | +#define ARM_SMCCC_HV_PV_TIME_ST \ |
2855 | + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ |
2856 | + ARM_SMCCC_SMC_64, \ |
2857 | + ARM_SMCCC_OWNER_STANDARD_HYP, \ |
2858 | + 0x21) |
2859 | + |
2860 | #endif /*__ASSEMBLY__*/ |
2861 | #endif /*__LINUX_ARM_SMCCC_H*/ |
2862 | diff --git a/include/linux/bpf.h b/include/linux/bpf.h |
2863 | index 7995940d41877..fe520d40597ff 100644 |
2864 | --- a/include/linux/bpf.h |
2865 | +++ b/include/linux/bpf.h |
2866 | @@ -295,6 +295,11 @@ static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) |
2867 | |
2868 | /* verify correctness of eBPF program */ |
2869 | int bpf_check(struct bpf_prog **fp, union bpf_attr *attr); |
2870 | + |
2871 | +static inline bool unprivileged_ebpf_enabled(void) |
2872 | +{ |
2873 | + return !sysctl_unprivileged_bpf_disabled; |
2874 | +} |
2875 | #else |
2876 | static inline void bpf_register_prog_type(struct bpf_prog_type_list *tl) |
2877 | { |
2878 | @@ -322,6 +327,12 @@ static inline struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) |
2879 | { |
2880 | return ERR_PTR(-EOPNOTSUPP); |
2881 | } |
2882 | + |
2883 | +static inline bool unprivileged_ebpf_enabled(void) |
2884 | +{ |
2885 | + return false; |
2886 | +} |
2887 | + |
2888 | #endif /* CONFIG_BPF_SYSCALL */ |
2889 | |
2890 | /* verifier prototypes for helper functions called from eBPF programs */ |
2891 | diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h |
2892 | index d830eddacdc60..1c1ca41685162 100644 |
2893 | --- a/include/linux/compiler-gcc.h |
2894 | +++ b/include/linux/compiler-gcc.h |
2895 | @@ -107,7 +107,7 @@ |
2896 | #define __weak __attribute__((weak)) |
2897 | #define __alias(symbol) __attribute__((alias(#symbol))) |
2898 | |
2899 | -#ifdef RETPOLINE |
2900 | +#ifdef CONFIG_RETPOLINE |
2901 | #define __noretpoline __attribute__((indirect_branch("keep"))) |
2902 | #endif |
2903 | |
2904 | diff --git a/include/linux/module.h b/include/linux/module.h |
2905 | index 99f330ae13da5..be4a3a9fd89ca 100644 |
2906 | --- a/include/linux/module.h |
2907 | +++ b/include/linux/module.h |
2908 | @@ -791,7 +791,7 @@ static inline void module_bug_finalize(const Elf_Ehdr *hdr, |
2909 | static inline void module_bug_cleanup(struct module *mod) {} |
2910 | #endif /* CONFIG_GENERIC_BUG */ |
2911 | |
2912 | -#ifdef RETPOLINE |
2913 | +#ifdef CONFIG_RETPOLINE |
2914 | extern bool retpoline_module_ok(bool has_retpoline); |
2915 | #else |
2916 | static inline bool retpoline_module_ok(bool has_retpoline) |
2917 | diff --git a/include/xen/grant_table.h b/include/xen/grant_table.h |
2918 | index f9d8aac170fbc..c51ae64b6dcb8 100644 |
2919 | --- a/include/xen/grant_table.h |
2920 | +++ b/include/xen/grant_table.h |
2921 | @@ -97,17 +97,32 @@ int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly); |
2922 | * access has been ended, free the given page too. Access will be ended |
2923 | * immediately iff the grant entry is not in use, otherwise it will happen |
2924 | * some time later. page may be 0, in which case no freeing will occur. |
2925 | + * Note that the granted page might still be accessed (read or write) by the |
2926 | + * other side after gnttab_end_foreign_access() returns, so even if page was |
2927 | + * specified as 0 it is not allowed to just reuse the page for other |
2928 | + * purposes immediately. gnttab_end_foreign_access() will take an additional |
2929 | + * reference to the granted page in this case, which is dropped only after |
2930 | + * the grant is no longer in use. |
2931 | + * This requires that multi page allocations for areas subject to |
2932 | + * gnttab_end_foreign_access() are done via alloc_pages_exact() (and freeing |
2933 | + * via free_pages_exact()) in order to avoid high order pages. |
2934 | */ |
2935 | void gnttab_end_foreign_access(grant_ref_t ref, int readonly, |
2936 | unsigned long page); |
2937 | |
2938 | +/* |
2939 | + * End access through the given grant reference, iff the grant entry is |
2940 | + * no longer in use. In case of success ending foreign access, the |
2941 | + * grant reference is deallocated. |
2942 | + * Return 1 if the grant entry was freed, 0 if it is still in use. |
2943 | + */ |
2944 | +int gnttab_try_end_foreign_access(grant_ref_t ref); |
2945 | + |
2946 | int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn); |
2947 | |
2948 | unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref); |
2949 | unsigned long gnttab_end_foreign_transfer(grant_ref_t ref); |
2950 | |
2951 | -int gnttab_query_foreign_access(grant_ref_t ref); |
2952 | - |
2953 | /* |
2954 | * operations on reserved batches of grant references |
2955 | */ |
2956 | diff --git a/kernel/sysctl.c b/kernel/sysctl.c |
2957 | index 78b445562b81e..184d462339e65 100644 |
2958 | --- a/kernel/sysctl.c |
2959 | +++ b/kernel/sysctl.c |
2960 | @@ -222,6 +222,11 @@ static int sysrq_sysctl_handler(struct ctl_table *table, int write, |
2961 | #endif |
2962 | |
2963 | #ifdef CONFIG_BPF_SYSCALL |
2964 | + |
2965 | +void __weak unpriv_ebpf_notify(int new_state) |
2966 | +{ |
2967 | +} |
2968 | + |
2969 | static int bpf_unpriv_handler(struct ctl_table *table, int write, |
2970 | void *buffer, size_t *lenp, loff_t *ppos) |
2971 | { |
2972 | @@ -239,6 +244,9 @@ static int bpf_unpriv_handler(struct ctl_table *table, int write, |
2973 | return -EPERM; |
2974 | *(int *)table->data = unpriv_enable; |
2975 | } |
2976 | + |
2977 | + unpriv_ebpf_notify(unpriv_enable); |
2978 | + |
2979 | return ret; |
2980 | } |
2981 | #endif |
2982 | diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c |
2983 | index 9abcdf2e8dfe8..62b0552b7b718 100644 |
2984 | --- a/scripts/mod/modpost.c |
2985 | +++ b/scripts/mod/modpost.c |
2986 | @@ -2147,7 +2147,7 @@ static void add_intree_flag(struct buffer *b, int is_intree) |
2987 | /* Cannot check for assembler */ |
2988 | static void add_retpoline(struct buffer *b) |
2989 | { |
2990 | - buf_printf(b, "\n#ifdef RETPOLINE\n"); |
2991 | + buf_printf(b, "\n#ifdef CONFIG_RETPOLINE\n"); |
2992 | buf_printf(b, "MODULE_INFO(retpoline, \"Y\");\n"); |
2993 | buf_printf(b, "#endif\n"); |
2994 | } |
2995 | diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h |
2996 | index f6d1bc93589c7..f032dfed00a93 100644 |
2997 | --- a/tools/arch/x86/include/asm/cpufeatures.h |
2998 | +++ b/tools/arch/x86/include/asm/cpufeatures.h |
2999 | @@ -194,7 +194,7 @@ |
3000 | #define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ |
3001 | |
3002 | #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ |
3003 | -#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */ |
3004 | +#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCEs for Spectre variant 2 */ |
3005 | |
3006 | #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ |
3007 | #define X86_FEATURE_SSBD ( 7*32+17) /* Speculative Store Bypass Disable */ |