Annotation of /trunk/kernel-alx/patches-5.4/0298-5.4.199-all-fixes.patch
Parent Directory | Revision Log
Revision 3635 -
(hide annotations)
(download)
Mon Oct 24 12:34:12 2022 UTC (23 months ago) by niro
File size: 41286 byte(s)
Mon Oct 24 12:34:12 2022 UTC (23 months ago) by niro
File size: 41286 byte(s)
-sync kernel patches
1 | niro | 3635 | diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu |
2 | index c24afa60a30ea..726ac2e01b777 100644 | ||
3 | --- a/Documentation/ABI/testing/sysfs-devices-system-cpu | ||
4 | +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu | ||
5 | @@ -489,6 +489,7 @@ What: /sys/devices/system/cpu/vulnerabilities | ||
6 | /sys/devices/system/cpu/vulnerabilities/srbds | ||
7 | /sys/devices/system/cpu/vulnerabilities/tsx_async_abort | ||
8 | /sys/devices/system/cpu/vulnerabilities/itlb_multihit | ||
9 | + /sys/devices/system/cpu/vulnerabilities/mmio_stale_data | ||
10 | Date: January 2018 | ||
11 | Contact: Linux kernel mailing list <linux-kernel@vger.kernel.org> | ||
12 | Description: Information about CPU vulnerabilities | ||
13 | diff --git a/Documentation/admin-guide/hw-vuln/index.rst b/Documentation/admin-guide/hw-vuln/index.rst | ||
14 | index ca4dbdd9016d5..2adec1e6520a6 100644 | ||
15 | --- a/Documentation/admin-guide/hw-vuln/index.rst | ||
16 | +++ b/Documentation/admin-guide/hw-vuln/index.rst | ||
17 | @@ -15,3 +15,4 @@ are configurable at compile, boot or run time. | ||
18 | tsx_async_abort | ||
19 | multihit.rst | ||
20 | special-register-buffer-data-sampling.rst | ||
21 | + processor_mmio_stale_data.rst | ||
22 | diff --git a/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst b/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst | ||
23 | new file mode 100644 | ||
24 | index 0000000000000..9393c50b5afc9 | ||
25 | --- /dev/null | ||
26 | +++ b/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst | ||
27 | @@ -0,0 +1,246 @@ | ||
28 | +========================================= | ||
29 | +Processor MMIO Stale Data Vulnerabilities | ||
30 | +========================================= | ||
31 | + | ||
32 | +Processor MMIO Stale Data Vulnerabilities are a class of memory-mapped I/O | ||
33 | +(MMIO) vulnerabilities that can expose data. The sequences of operations for | ||
34 | +exposing data range from simple to very complex. Because most of the | ||
35 | +vulnerabilities require the attacker to have access to MMIO, many environments | ||
36 | +are not affected. System environments using virtualization where MMIO access is | ||
37 | +provided to untrusted guests may need mitigation. These vulnerabilities are | ||
38 | +not transient execution attacks. However, these vulnerabilities may propagate | ||
39 | +stale data into core fill buffers where the data can subsequently be inferred | ||
40 | +by an unmitigated transient execution attack. Mitigation for these | ||
41 | +vulnerabilities includes a combination of microcode update and software | ||
42 | +changes, depending on the platform and usage model. Some of these mitigations | ||
43 | +are similar to those used to mitigate Microarchitectural Data Sampling (MDS) or | ||
44 | +those used to mitigate Special Register Buffer Data Sampling (SRBDS). | ||
45 | + | ||
46 | +Data Propagators | ||
47 | +================ | ||
48 | +Propagators are operations that result in stale data being copied or moved from | ||
49 | +one microarchitectural buffer or register to another. Processor MMIO Stale Data | ||
50 | +Vulnerabilities are operations that may result in stale data being directly | ||
51 | +read into an architectural, software-visible state or sampled from a buffer or | ||
52 | +register. | ||
53 | + | ||
54 | +Fill Buffer Stale Data Propagator (FBSDP) | ||
55 | +----------------------------------------- | ||
56 | +Stale data may propagate from fill buffers (FB) into the non-coherent portion | ||
57 | +of the uncore on some non-coherent writes. Fill buffer propagation by itself | ||
58 | +does not make stale data architecturally visible. Stale data must be propagated | ||
59 | +to a location where it is subject to reading or sampling. | ||
60 | + | ||
61 | +Sideband Stale Data Propagator (SSDP) | ||
62 | +------------------------------------- | ||
63 | +The sideband stale data propagator (SSDP) is limited to the client (including | ||
64 | +Intel Xeon server E3) uncore implementation. The sideband response buffer is | ||
65 | +shared by all client cores. For non-coherent reads that go to sideband | ||
66 | +destinations, the uncore logic returns 64 bytes of data to the core, including | ||
67 | +both requested data and unrequested stale data, from a transaction buffer and | ||
68 | +the sideband response buffer. As a result, stale data from the sideband | ||
69 | +response and transaction buffers may now reside in a core fill buffer. | ||
70 | + | ||
71 | +Primary Stale Data Propagator (PSDP) | ||
72 | +------------------------------------ | ||
73 | +The primary stale data propagator (PSDP) is limited to the client (including | ||
74 | +Intel Xeon server E3) uncore implementation. Similar to the sideband response | ||
75 | +buffer, the primary response buffer is shared by all client cores. For some | ||
76 | +processors, MMIO primary reads will return 64 bytes of data to the core fill | ||
77 | +buffer including both requested data and unrequested stale data. This is | ||
78 | +similar to the sideband stale data propagator. | ||
79 | + | ||
80 | +Vulnerabilities | ||
81 | +=============== | ||
82 | +Device Register Partial Write (DRPW) (CVE-2022-21166) | ||
83 | +----------------------------------------------------- | ||
84 | +Some endpoint MMIO registers incorrectly handle writes that are smaller than | ||
85 | +the register size. Instead of aborting the write or only copying the correct | ||
86 | +subset of bytes (for example, 2 bytes for a 2-byte write), more bytes than | ||
87 | +specified by the write transaction may be written to the register. On | ||
88 | +processors affected by FBSDP, this may expose stale data from the fill buffers | ||
89 | +of the core that created the write transaction. | ||
90 | + | ||
91 | +Shared Buffers Data Sampling (SBDS) (CVE-2022-21125) | ||
92 | +---------------------------------------------------- | ||
93 | +After propagators may have moved data around the uncore and copied stale data | ||
94 | +into client core fill buffers, processors affected by MFBDS can leak data from | ||
95 | +the fill buffer. It is limited to the client (including Intel Xeon server E3) | ||
96 | +uncore implementation. | ||
97 | + | ||
98 | +Shared Buffers Data Read (SBDR) (CVE-2022-21123) | ||
99 | +------------------------------------------------ | ||
100 | +It is similar to Shared Buffer Data Sampling (SBDS) except that the data is | ||
101 | +directly read into the architectural software-visible state. It is limited to | ||
102 | +the client (including Intel Xeon server E3) uncore implementation. | ||
103 | + | ||
104 | +Affected Processors | ||
105 | +=================== | ||
106 | +Not all the CPUs are affected by all the variants. For instance, most | ||
107 | +processors for the server market (excluding Intel Xeon E3 processors) are | ||
108 | +impacted by only Device Register Partial Write (DRPW). | ||
109 | + | ||
110 | +Below is the list of affected Intel processors [#f1]_: | ||
111 | + | ||
112 | + =================== ============ ========= | ||
113 | + Common name Family_Model Steppings | ||
114 | + =================== ============ ========= | ||
115 | + HASWELL_X 06_3FH 2,4 | ||
116 | + SKYLAKE_L 06_4EH 3 | ||
117 | + BROADWELL_X 06_4FH All | ||
118 | + SKYLAKE_X 06_55H 3,4,6,7,11 | ||
119 | + BROADWELL_D 06_56H 3,4,5 | ||
120 | + SKYLAKE 06_5EH 3 | ||
121 | + ICELAKE_X 06_6AH 4,5,6 | ||
122 | + ICELAKE_D 06_6CH 1 | ||
123 | + ICELAKE_L 06_7EH 5 | ||
124 | + ATOM_TREMONT_D 06_86H All | ||
125 | + LAKEFIELD 06_8AH 1 | ||
126 | + KABYLAKE_L 06_8EH 9 to 12 | ||
127 | + ATOM_TREMONT 06_96H 1 | ||
128 | + ATOM_TREMONT_L 06_9CH 0 | ||
129 | + KABYLAKE 06_9EH 9 to 13 | ||
130 | + COMETLAKE 06_A5H 2,3,5 | ||
131 | + COMETLAKE_L 06_A6H 0,1 | ||
132 | + ROCKETLAKE 06_A7H 1 | ||
133 | + =================== ============ ========= | ||
134 | + | ||
135 | +If a CPU is in the affected processor list, but not affected by a variant, it | ||
136 | +is indicated by new bits in MSR IA32_ARCH_CAPABILITIES. As described in a later | ||
137 | +section, mitigation largely remains the same for all the variants, i.e. to | ||
138 | +clear the CPU fill buffers via VERW instruction. | ||
139 | + | ||
140 | +New bits in MSRs | ||
141 | +================ | ||
142 | +Newer processors and microcode update on existing affected processors added new | ||
143 | +bits to IA32_ARCH_CAPABILITIES MSR. These bits can be used to enumerate | ||
144 | +specific variants of Processor MMIO Stale Data vulnerabilities and mitigation | ||
145 | +capability. | ||
146 | + | ||
147 | +MSR IA32_ARCH_CAPABILITIES | ||
148 | +-------------------------- | ||
149 | +Bit 13 - SBDR_SSDP_NO - When set, processor is not affected by either the | ||
150 | + Shared Buffers Data Read (SBDR) vulnerability or the sideband stale | ||
151 | + data propagator (SSDP). | ||
152 | +Bit 14 - FBSDP_NO - When set, processor is not affected by the Fill Buffer | ||
153 | + Stale Data Propagator (FBSDP). | ||
154 | +Bit 15 - PSDP_NO - When set, processor is not affected by Primary Stale Data | ||
155 | + Propagator (PSDP). | ||
156 | +Bit 17 - FB_CLEAR - When set, VERW instruction will overwrite CPU fill buffer | ||
157 | + values as part of MD_CLEAR operations. Processors that do not | ||
158 | + enumerate MDS_NO (meaning they are affected by MDS) but that do | ||
159 | + enumerate support for both L1D_FLUSH and MD_CLEAR implicitly enumerate | ||
160 | + FB_CLEAR as part of their MD_CLEAR support. | ||
161 | +Bit 18 - FB_CLEAR_CTRL - Processor supports read and write to MSR | ||
162 | + IA32_MCU_OPT_CTRL[FB_CLEAR_DIS]. On such processors, the FB_CLEAR_DIS | ||
163 | + bit can be set to cause the VERW instruction to not perform the | ||
164 | + FB_CLEAR action. Not all processors that support FB_CLEAR will support | ||
165 | + FB_CLEAR_CTRL. | ||
166 | + | ||
167 | +MSR IA32_MCU_OPT_CTRL | ||
168 | +--------------------- | ||
169 | +Bit 3 - FB_CLEAR_DIS - When set, VERW instruction does not perform the FB_CLEAR | ||
170 | +action. This may be useful to reduce the performance impact of FB_CLEAR in | ||
171 | +cases where system software deems it warranted (for example, when performance | ||
172 | +is more critical, or the untrusted software has no MMIO access). Note that | ||
173 | +FB_CLEAR_DIS has no impact on enumeration (for example, it does not change | ||
174 | +FB_CLEAR or MD_CLEAR enumeration) and it may not be supported on all processors | ||
175 | +that enumerate FB_CLEAR. | ||
176 | + | ||
177 | +Mitigation | ||
178 | +========== | ||
179 | +Like MDS, all variants of Processor MMIO Stale Data vulnerabilities have the | ||
180 | +same mitigation strategy to force the CPU to clear the affected buffers before | ||
181 | +an attacker can extract the secrets. | ||
182 | + | ||
183 | +This is achieved by using the otherwise unused and obsolete VERW instruction in | ||
184 | +combination with a microcode update. The microcode clears the affected CPU | ||
185 | +buffers when the VERW instruction is executed. | ||
186 | + | ||
187 | +Kernel reuses the MDS function to invoke the buffer clearing: | ||
188 | + | ||
189 | + mds_clear_cpu_buffers() | ||
190 | + | ||
191 | +On MDS affected CPUs, the kernel already invokes CPU buffer clear on | ||
192 | +kernel/userspace, hypervisor/guest and C-state (idle) transitions. No | ||
193 | +additional mitigation is needed on such CPUs. | ||
194 | + | ||
195 | +For CPUs not affected by MDS or TAA, mitigation is needed only for the attacker | ||
196 | +with MMIO capability. Therefore, VERW is not required for kernel/userspace. For | ||
197 | +virtualization case, VERW is only needed at VMENTER for a guest with MMIO | ||
198 | +capability. | ||
199 | + | ||
200 | +Mitigation points | ||
201 | +----------------- | ||
202 | +Return to user space | ||
203 | +^^^^^^^^^^^^^^^^^^^^ | ||
204 | +Same mitigation as MDS when affected by MDS/TAA, otherwise no mitigation | ||
205 | +needed. | ||
206 | + | ||
207 | +C-State transition | ||
208 | +^^^^^^^^^^^^^^^^^^ | ||
209 | +Control register writes by CPU during C-state transition can propagate data | ||
210 | +from fill buffer to uncore buffers. Execute VERW before C-state transition to | ||
211 | +clear CPU fill buffers. | ||
212 | + | ||
213 | +Guest entry point | ||
214 | +^^^^^^^^^^^^^^^^^ | ||
215 | +Same mitigation as MDS when processor is also affected by MDS/TAA, otherwise | ||
216 | +execute VERW at VMENTER only for MMIO capable guests. On CPUs not affected by | ||
217 | +MDS/TAA, guest without MMIO access cannot extract secrets using Processor MMIO | ||
218 | +Stale Data vulnerabilities, so there is no need to execute VERW for such guests. | ||
219 | + | ||
220 | +Mitigation control on the kernel command line | ||
221 | +--------------------------------------------- | ||
222 | +The kernel command line allows to control the Processor MMIO Stale Data | ||
223 | +mitigations at boot time with the option "mmio_stale_data=". The valid | ||
224 | +arguments for this option are: | ||
225 | + | ||
226 | + ========== ================================================================= | ||
227 | + full If the CPU is vulnerable, enable mitigation; CPU buffer clearing | ||
228 | + on exit to userspace and when entering a VM. Idle transitions are | ||
229 | + protected as well. It does not automatically disable SMT. | ||
230 | + full,nosmt Same as full, with SMT disabled on vulnerable CPUs. This is the | ||
231 | + complete mitigation. | ||
232 | + off Disables mitigation completely. | ||
233 | + ========== ================================================================= | ||
234 | + | ||
235 | +If the CPU is affected and mmio_stale_data=off is not supplied on the kernel | ||
236 | +command line, then the kernel selects the appropriate mitigation. | ||
237 | + | ||
238 | +Mitigation status information | ||
239 | +----------------------------- | ||
240 | +The Linux kernel provides a sysfs interface to enumerate the current | ||
241 | +vulnerability status of the system: whether the system is vulnerable, and | ||
242 | +which mitigations are active. The relevant sysfs file is: | ||
243 | + | ||
244 | + /sys/devices/system/cpu/vulnerabilities/mmio_stale_data | ||
245 | + | ||
246 | +The possible values in this file are: | ||
247 | + | ||
248 | + .. list-table:: | ||
249 | + | ||
250 | + * - 'Not affected' | ||
251 | + - The processor is not vulnerable | ||
252 | + * - 'Vulnerable' | ||
253 | + - The processor is vulnerable, but no mitigation enabled | ||
254 | + * - 'Vulnerable: Clear CPU buffers attempted, no microcode' | ||
255 | + - The processor is vulnerable, but microcode is not updated. The | ||
256 | + mitigation is enabled on a best effort basis. | ||
257 | + * - 'Mitigation: Clear CPU buffers' | ||
258 | + - The processor is vulnerable and the CPU buffer clearing mitigation is | ||
259 | + enabled. | ||
260 | + | ||
261 | +If the processor is vulnerable then the following information is appended to | ||
262 | +the above information: | ||
263 | + | ||
264 | + ======================== =========================================== | ||
265 | + 'SMT vulnerable' SMT is enabled | ||
266 | + 'SMT disabled' SMT is disabled | ||
267 | + 'SMT Host state unknown' Kernel runs in a VM, Host SMT state unknown | ||
268 | + ======================== =========================================== | ||
269 | + | ||
270 | +References | ||
271 | +---------- | ||
272 | +.. [#f1] Affected Processors | ||
273 | + https://www.intel.com/content/www/us/en/developer/topic-technology/software-security-guidance/processors-affected-consolidated-product-cpu-model.html | ||
274 | diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt | ||
275 | index 979423e1b639f..cad17c7a311e9 100644 | ||
276 | --- a/Documentation/admin-guide/kernel-parameters.txt | ||
277 | +++ b/Documentation/admin-guide/kernel-parameters.txt | ||
278 | @@ -2681,6 +2681,7 @@ | ||
279 | kvm.nx_huge_pages=off [X86] | ||
280 | no_entry_flush [PPC] | ||
281 | no_uaccess_flush [PPC] | ||
282 | + mmio_stale_data=off [X86] | ||
283 | |||
284 | Exceptions: | ||
285 | This does not have any effect on | ||
286 | @@ -2702,6 +2703,7 @@ | ||
287 | Equivalent to: l1tf=flush,nosmt [X86] | ||
288 | mds=full,nosmt [X86] | ||
289 | tsx_async_abort=full,nosmt [X86] | ||
290 | + mmio_stale_data=full,nosmt [X86] | ||
291 | |||
292 | mminit_loglevel= | ||
293 | [KNL] When CONFIG_DEBUG_MEMORY_INIT is set, this | ||
294 | @@ -2711,6 +2713,40 @@ | ||
295 | log everything. Information is printed at KERN_DEBUG | ||
296 | so loglevel=8 may also need to be specified. | ||
297 | |||
298 | + mmio_stale_data= | ||
299 | + [X86,INTEL] Control mitigation for the Processor | ||
300 | + MMIO Stale Data vulnerabilities. | ||
301 | + | ||
302 | + Processor MMIO Stale Data is a class of | ||
303 | + vulnerabilities that may expose data after an MMIO | ||
304 | + operation. Exposed data could originate or end in | ||
305 | + the same CPU buffers as affected by MDS and TAA. | ||
306 | + Therefore, similar to MDS and TAA, the mitigation | ||
307 | + is to clear the affected CPU buffers. | ||
308 | + | ||
309 | + This parameter controls the mitigation. The | ||
310 | + options are: | ||
311 | + | ||
312 | + full - Enable mitigation on vulnerable CPUs | ||
313 | + | ||
314 | + full,nosmt - Enable mitigation and disable SMT on | ||
315 | + vulnerable CPUs. | ||
316 | + | ||
317 | + off - Unconditionally disable mitigation | ||
318 | + | ||
319 | + On MDS or TAA affected machines, | ||
320 | + mmio_stale_data=off can be prevented by an active | ||
321 | + MDS or TAA mitigation as these vulnerabilities are | ||
322 | + mitigated with the same mechanism so in order to | ||
323 | + disable this mitigation, you need to specify | ||
324 | + mds=off and tsx_async_abort=off too. | ||
325 | + | ||
326 | + Not specifying this option is equivalent to | ||
327 | + mmio_stale_data=full. | ||
328 | + | ||
329 | + For details see: | ||
330 | + Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst | ||
331 | + | ||
332 | module.sig_enforce | ||
333 | [KNL] When CONFIG_MODULE_SIG is set, this means that | ||
334 | modules without (valid) signatures will fail to load. | ||
335 | diff --git a/Makefile b/Makefile | ||
336 | index 1c99e688da213..a8d2f2a1c052a 100644 | ||
337 | --- a/Makefile | ||
338 | +++ b/Makefile | ||
339 | @@ -1,7 +1,7 @@ | ||
340 | # SPDX-License-Identifier: GPL-2.0 | ||
341 | VERSION = 5 | ||
342 | PATCHLEVEL = 4 | ||
343 | -SUBLEVEL = 198 | ||
344 | +SUBLEVEL = 199 | ||
345 | EXTRAVERSION = | ||
346 | NAME = Kleptomaniac Octopus | ||
347 | |||
348 | diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h | ||
349 | index 56eb9a6524e96..8c28a2365a92b 100644 | ||
350 | --- a/arch/x86/include/asm/cpufeatures.h | ||
351 | +++ b/arch/x86/include/asm/cpufeatures.h | ||
352 | @@ -405,5 +405,6 @@ | ||
353 | #define X86_BUG_TAA X86_BUG(22) /* CPU is affected by TSX Async Abort(TAA) */ | ||
354 | #define X86_BUG_ITLB_MULTIHIT X86_BUG(23) /* CPU may incur MCE during certain page attribute changes */ | ||
355 | #define X86_BUG_SRBDS X86_BUG(24) /* CPU may leak RNG bits if not mitigated */ | ||
356 | +#define X86_BUG_MMIO_STALE_DATA X86_BUG(25) /* CPU is affected by Processor MMIO Stale Data vulnerabilities */ | ||
357 | |||
358 | #endif /* _ASM_X86_CPUFEATURES_H */ | ||
359 | diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h | ||
360 | index c606c0b707382..5b07573c3bc87 100644 | ||
361 | --- a/arch/x86/include/asm/intel-family.h | ||
362 | +++ b/arch/x86/include/asm/intel-family.h | ||
363 | @@ -86,6 +86,14 @@ | ||
364 | #define INTEL_FAM6_COMETLAKE 0xA5 | ||
365 | #define INTEL_FAM6_COMETLAKE_L 0xA6 | ||
366 | |||
367 | +#define INTEL_FAM6_ROCKETLAKE 0xA7 | ||
368 | + | ||
369 | +/* Hybrid Core/Atom Processors */ | ||
370 | + | ||
371 | +#define INTEL_FAM6_LAKEFIELD 0x8A | ||
372 | +#define INTEL_FAM6_ALDERLAKE 0x97 | ||
373 | +#define INTEL_FAM6_ALDERLAKE_L 0x9A | ||
374 | + | ||
375 | /* "Small Core" Processors (Atom) */ | ||
376 | |||
377 | #define INTEL_FAM6_ATOM_BONNELL 0x1C /* Diamondville, Pineview */ | ||
378 | @@ -111,6 +119,7 @@ | ||
379 | |||
380 | #define INTEL_FAM6_ATOM_TREMONT_D 0x86 /* Jacobsville */ | ||
381 | #define INTEL_FAM6_ATOM_TREMONT 0x96 /* Elkhart Lake */ | ||
382 | +#define INTEL_FAM6_ATOM_TREMONT_L 0x9C /* Jasper Lake */ | ||
383 | |||
384 | /* Xeon Phi */ | ||
385 | |||
386 | diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h | ||
387 | index f312b6f6ac481..c56042916a7c3 100644 | ||
388 | --- a/arch/x86/include/asm/msr-index.h | ||
389 | +++ b/arch/x86/include/asm/msr-index.h | ||
390 | @@ -105,6 +105,30 @@ | ||
391 | * Not susceptible to | ||
392 | * TSX Async Abort (TAA) vulnerabilities. | ||
393 | */ | ||
394 | +#define ARCH_CAP_SBDR_SSDP_NO BIT(13) /* | ||
395 | + * Not susceptible to SBDR and SSDP | ||
396 | + * variants of Processor MMIO stale data | ||
397 | + * vulnerabilities. | ||
398 | + */ | ||
399 | +#define ARCH_CAP_FBSDP_NO BIT(14) /* | ||
400 | + * Not susceptible to FBSDP variant of | ||
401 | + * Processor MMIO stale data | ||
402 | + * vulnerabilities. | ||
403 | + */ | ||
404 | +#define ARCH_CAP_PSDP_NO BIT(15) /* | ||
405 | + * Not susceptible to PSDP variant of | ||
406 | + * Processor MMIO stale data | ||
407 | + * vulnerabilities. | ||
408 | + */ | ||
409 | +#define ARCH_CAP_FB_CLEAR BIT(17) /* | ||
410 | + * VERW clears CPU fill buffer | ||
411 | + * even on MDS_NO CPUs. | ||
412 | + */ | ||
413 | +#define ARCH_CAP_FB_CLEAR_CTRL BIT(18) /* | ||
414 | + * MSR_IA32_MCU_OPT_CTRL[FB_CLEAR_DIS] | ||
415 | + * bit available to control VERW | ||
416 | + * behavior. | ||
417 | + */ | ||
418 | |||
419 | #define MSR_IA32_FLUSH_CMD 0x0000010b | ||
420 | #define L1D_FLUSH BIT(0) /* | ||
421 | @@ -122,6 +146,7 @@ | ||
422 | /* SRBDS support */ | ||
423 | #define MSR_IA32_MCU_OPT_CTRL 0x00000123 | ||
424 | #define RNGDS_MITG_DIS BIT(0) | ||
425 | +#define FB_CLEAR_DIS BIT(3) /* CPU Fill buffer clear disable */ | ||
426 | |||
427 | #define MSR_IA32_SYSENTER_CS 0x00000174 | ||
428 | #define MSR_IA32_SYSENTER_ESP 0x00000175 | ||
429 | diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h | ||
430 | index 956df82bbc2bc..ece2b2c6d020d 100644 | ||
431 | --- a/arch/x86/include/asm/nospec-branch.h | ||
432 | +++ b/arch/x86/include/asm/nospec-branch.h | ||
433 | @@ -313,6 +313,8 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); | ||
434 | DECLARE_STATIC_KEY_FALSE(mds_user_clear); | ||
435 | DECLARE_STATIC_KEY_FALSE(mds_idle_clear); | ||
436 | |||
437 | +DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear); | ||
438 | + | ||
439 | #include <asm/segment.h> | ||
440 | |||
441 | /** | ||
442 | diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c | ||
443 | index e817aaeef254c..09d02b1f6f71f 100644 | ||
444 | --- a/arch/x86/kernel/cpu/bugs.c | ||
445 | +++ b/arch/x86/kernel/cpu/bugs.c | ||
446 | @@ -40,8 +40,10 @@ static void __init spectre_v2_select_mitigation(void); | ||
447 | static void __init ssb_select_mitigation(void); | ||
448 | static void __init l1tf_select_mitigation(void); | ||
449 | static void __init mds_select_mitigation(void); | ||
450 | -static void __init mds_print_mitigation(void); | ||
451 | +static void __init md_clear_update_mitigation(void); | ||
452 | +static void __init md_clear_select_mitigation(void); | ||
453 | static void __init taa_select_mitigation(void); | ||
454 | +static void __init mmio_select_mitigation(void); | ||
455 | static void __init srbds_select_mitigation(void); | ||
456 | |||
457 | /* The base value of the SPEC_CTRL MSR that always has to be preserved. */ | ||
458 | @@ -76,6 +78,10 @@ EXPORT_SYMBOL_GPL(mds_user_clear); | ||
459 | DEFINE_STATIC_KEY_FALSE(mds_idle_clear); | ||
460 | EXPORT_SYMBOL_GPL(mds_idle_clear); | ||
461 | |||
462 | +/* Controls CPU Fill buffer clear before KVM guest MMIO accesses */ | ||
463 | +DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); | ||
464 | +EXPORT_SYMBOL_GPL(mmio_stale_data_clear); | ||
465 | + | ||
466 | void __init check_bugs(void) | ||
467 | { | ||
468 | identify_boot_cpu(); | ||
469 | @@ -108,16 +114,9 @@ void __init check_bugs(void) | ||
470 | spectre_v2_select_mitigation(); | ||
471 | ssb_select_mitigation(); | ||
472 | l1tf_select_mitigation(); | ||
473 | - mds_select_mitigation(); | ||
474 | - taa_select_mitigation(); | ||
475 | + md_clear_select_mitigation(); | ||
476 | srbds_select_mitigation(); | ||
477 | |||
478 | - /* | ||
479 | - * As MDS and TAA mitigations are inter-related, print MDS | ||
480 | - * mitigation until after TAA mitigation selection is done. | ||
481 | - */ | ||
482 | - mds_print_mitigation(); | ||
483 | - | ||
484 | arch_smt_update(); | ||
485 | |||
486 | #ifdef CONFIG_X86_32 | ||
487 | @@ -257,14 +256,6 @@ static void __init mds_select_mitigation(void) | ||
488 | } | ||
489 | } | ||
490 | |||
491 | -static void __init mds_print_mitigation(void) | ||
492 | -{ | ||
493 | - if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) | ||
494 | - return; | ||
495 | - | ||
496 | - pr_info("%s\n", mds_strings[mds_mitigation]); | ||
497 | -} | ||
498 | - | ||
499 | static int __init mds_cmdline(char *str) | ||
500 | { | ||
501 | if (!boot_cpu_has_bug(X86_BUG_MDS)) | ||
502 | @@ -312,7 +303,7 @@ static void __init taa_select_mitigation(void) | ||
503 | /* TSX previously disabled by tsx=off */ | ||
504 | if (!boot_cpu_has(X86_FEATURE_RTM)) { | ||
505 | taa_mitigation = TAA_MITIGATION_TSX_DISABLED; | ||
506 | - goto out; | ||
507 | + return; | ||
508 | } | ||
509 | |||
510 | if (cpu_mitigations_off()) { | ||
511 | @@ -326,7 +317,7 @@ static void __init taa_select_mitigation(void) | ||
512 | */ | ||
513 | if (taa_mitigation == TAA_MITIGATION_OFF && | ||
514 | mds_mitigation == MDS_MITIGATION_OFF) | ||
515 | - goto out; | ||
516 | + return; | ||
517 | |||
518 | if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) | ||
519 | taa_mitigation = TAA_MITIGATION_VERW; | ||
520 | @@ -358,18 +349,6 @@ static void __init taa_select_mitigation(void) | ||
521 | |||
522 | if (taa_nosmt || cpu_mitigations_auto_nosmt()) | ||
523 | cpu_smt_disable(false); | ||
524 | - | ||
525 | - /* | ||
526 | - * Update MDS mitigation, if necessary, as the mds_user_clear is | ||
527 | - * now enabled for TAA mitigation. | ||
528 | - */ | ||
529 | - if (mds_mitigation == MDS_MITIGATION_OFF && | ||
530 | - boot_cpu_has_bug(X86_BUG_MDS)) { | ||
531 | - mds_mitigation = MDS_MITIGATION_FULL; | ||
532 | - mds_select_mitigation(); | ||
533 | - } | ||
534 | -out: | ||
535 | - pr_info("%s\n", taa_strings[taa_mitigation]); | ||
536 | } | ||
537 | |||
538 | static int __init tsx_async_abort_parse_cmdline(char *str) | ||
539 | @@ -393,6 +372,151 @@ static int __init tsx_async_abort_parse_cmdline(char *str) | ||
540 | } | ||
541 | early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); | ||
542 | |||
543 | +#undef pr_fmt | ||
544 | +#define pr_fmt(fmt) "MMIO Stale Data: " fmt | ||
545 | + | ||
546 | +enum mmio_mitigations { | ||
547 | + MMIO_MITIGATION_OFF, | ||
548 | + MMIO_MITIGATION_UCODE_NEEDED, | ||
549 | + MMIO_MITIGATION_VERW, | ||
550 | +}; | ||
551 | + | ||
552 | +/* Default mitigation for Processor MMIO Stale Data vulnerabilities */ | ||
553 | +static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW; | ||
554 | +static bool mmio_nosmt __ro_after_init = false; | ||
555 | + | ||
556 | +static const char * const mmio_strings[] = { | ||
557 | + [MMIO_MITIGATION_OFF] = "Vulnerable", | ||
558 | + [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", | ||
559 | + [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", | ||
560 | +}; | ||
561 | + | ||
562 | +static void __init mmio_select_mitigation(void) | ||
563 | +{ | ||
564 | + u64 ia32_cap; | ||
565 | + | ||
566 | + if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || | ||
567 | + cpu_mitigations_off()) { | ||
568 | + mmio_mitigation = MMIO_MITIGATION_OFF; | ||
569 | + return; | ||
570 | + } | ||
571 | + | ||
572 | + if (mmio_mitigation == MMIO_MITIGATION_OFF) | ||
573 | + return; | ||
574 | + | ||
575 | + ia32_cap = x86_read_arch_cap_msr(); | ||
576 | + | ||
577 | + /* | ||
578 | + * Enable CPU buffer clear mitigation for host and VMM, if also affected | ||
579 | + * by MDS or TAA. Otherwise, enable mitigation for VMM only. | ||
580 | + */ | ||
581 | + if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && | ||
582 | + boot_cpu_has(X86_FEATURE_RTM))) | ||
583 | + static_branch_enable(&mds_user_clear); | ||
584 | + else | ||
585 | + static_branch_enable(&mmio_stale_data_clear); | ||
586 | + | ||
587 | + /* | ||
588 | + * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can | ||
589 | + * be propagated to uncore buffers, clearing the Fill buffers on idle | ||
590 | + * is required irrespective of SMT state. | ||
591 | + */ | ||
592 | + if (!(ia32_cap & ARCH_CAP_FBSDP_NO)) | ||
593 | + static_branch_enable(&mds_idle_clear); | ||
594 | + | ||
595 | + /* | ||
596 | + * Check if the system has the right microcode. | ||
597 | + * | ||
598 | + * CPU Fill buffer clear mitigation is enumerated by either an explicit | ||
599 | + * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS | ||
600 | + * affected systems. | ||
601 | + */ | ||
602 | + if ((ia32_cap & ARCH_CAP_FB_CLEAR) || | ||
603 | + (boot_cpu_has(X86_FEATURE_MD_CLEAR) && | ||
604 | + boot_cpu_has(X86_FEATURE_FLUSH_L1D) && | ||
605 | + !(ia32_cap & ARCH_CAP_MDS_NO))) | ||
606 | + mmio_mitigation = MMIO_MITIGATION_VERW; | ||
607 | + else | ||
608 | + mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; | ||
609 | + | ||
610 | + if (mmio_nosmt || cpu_mitigations_auto_nosmt()) | ||
611 | + cpu_smt_disable(false); | ||
612 | +} | ||
613 | + | ||
614 | +static int __init mmio_stale_data_parse_cmdline(char *str) | ||
615 | +{ | ||
616 | + if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) | ||
617 | + return 0; | ||
618 | + | ||
619 | + if (!str) | ||
620 | + return -EINVAL; | ||
621 | + | ||
622 | + if (!strcmp(str, "off")) { | ||
623 | + mmio_mitigation = MMIO_MITIGATION_OFF; | ||
624 | + } else if (!strcmp(str, "full")) { | ||
625 | + mmio_mitigation = MMIO_MITIGATION_VERW; | ||
626 | + } else if (!strcmp(str, "full,nosmt")) { | ||
627 | + mmio_mitigation = MMIO_MITIGATION_VERW; | ||
628 | + mmio_nosmt = true; | ||
629 | + } | ||
630 | + | ||
631 | + return 0; | ||
632 | +} | ||
633 | +early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); | ||
634 | + | ||
635 | +#undef pr_fmt | ||
636 | +#define pr_fmt(fmt) "" fmt | ||
637 | + | ||
638 | +static void __init md_clear_update_mitigation(void) | ||
639 | +{ | ||
640 | + if (cpu_mitigations_off()) | ||
641 | + return; | ||
642 | + | ||
643 | + if (!static_key_enabled(&mds_user_clear)) | ||
644 | + goto out; | ||
645 | + | ||
646 | + /* | ||
647 | + * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data | ||
648 | + * mitigation, if necessary. | ||
649 | + */ | ||
650 | + if (mds_mitigation == MDS_MITIGATION_OFF && | ||
651 | + boot_cpu_has_bug(X86_BUG_MDS)) { | ||
652 | + mds_mitigation = MDS_MITIGATION_FULL; | ||
653 | + mds_select_mitigation(); | ||
654 | + } | ||
655 | + if (taa_mitigation == TAA_MITIGATION_OFF && | ||
656 | + boot_cpu_has_bug(X86_BUG_TAA)) { | ||
657 | + taa_mitigation = TAA_MITIGATION_VERW; | ||
658 | + taa_select_mitigation(); | ||
659 | + } | ||
660 | + if (mmio_mitigation == MMIO_MITIGATION_OFF && | ||
661 | + boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { | ||
662 | + mmio_mitigation = MMIO_MITIGATION_VERW; | ||
663 | + mmio_select_mitigation(); | ||
664 | + } | ||
665 | +out: | ||
666 | + if (boot_cpu_has_bug(X86_BUG_MDS)) | ||
667 | + pr_info("MDS: %s\n", mds_strings[mds_mitigation]); | ||
668 | + if (boot_cpu_has_bug(X86_BUG_TAA)) | ||
669 | + pr_info("TAA: %s\n", taa_strings[taa_mitigation]); | ||
670 | + if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) | ||
671 | + pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); | ||
672 | +} | ||
673 | + | ||
674 | +static void __init md_clear_select_mitigation(void) | ||
675 | +{ | ||
676 | + mds_select_mitigation(); | ||
677 | + taa_select_mitigation(); | ||
678 | + mmio_select_mitigation(); | ||
679 | + | ||
680 | + /* | ||
681 | + * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update | ||
682 | + * and print their mitigation after MDS, TAA and MMIO Stale Data | ||
683 | + * mitigation selection is done. | ||
684 | + */ | ||
685 | + md_clear_update_mitigation(); | ||
686 | +} | ||
687 | + | ||
688 | #undef pr_fmt | ||
689 | #define pr_fmt(fmt) "SRBDS: " fmt | ||
690 | |||
691 | @@ -454,11 +578,13 @@ static void __init srbds_select_mitigation(void) | ||
692 | return; | ||
693 | |||
694 | /* | ||
695 | - * Check to see if this is one of the MDS_NO systems supporting | ||
696 | - * TSX that are only exposed to SRBDS when TSX is enabled. | ||
697 | + * Check to see if this is one of the MDS_NO systems supporting TSX that | ||
698 | + * are only exposed to SRBDS when TSX is enabled or when CPU is affected | ||
699 | + * by Processor MMIO Stale Data vulnerability. | ||
700 | */ | ||
701 | ia32_cap = x86_read_arch_cap_msr(); | ||
702 | - if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM)) | ||
703 | + if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && | ||
704 | + !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) | ||
705 | srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; | ||
706 | else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) | ||
707 | srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; | ||
708 | @@ -1066,6 +1192,8 @@ static void update_indir_branch_cond(void) | ||
709 | /* Update the static key controlling the MDS CPU buffer clear in idle */ | ||
710 | static void update_mds_branch_idle(void) | ||
711 | { | ||
712 | + u64 ia32_cap = x86_read_arch_cap_msr(); | ||
713 | + | ||
714 | /* | ||
715 | * Enable the idle clearing if SMT is active on CPUs which are | ||
716 | * affected only by MSBDS and not any other MDS variant. | ||
717 | @@ -1077,14 +1205,17 @@ static void update_mds_branch_idle(void) | ||
718 | if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) | ||
719 | return; | ||
720 | |||
721 | - if (sched_smt_active()) | ||
722 | + if (sched_smt_active()) { | ||
723 | static_branch_enable(&mds_idle_clear); | ||
724 | - else | ||
725 | + } else if (mmio_mitigation == MMIO_MITIGATION_OFF || | ||
726 | + (ia32_cap & ARCH_CAP_FBSDP_NO)) { | ||
727 | static_branch_disable(&mds_idle_clear); | ||
728 | + } | ||
729 | } | ||
730 | |||
731 | #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" | ||
732 | #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" | ||
733 | +#define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" | ||
734 | |||
735 | void cpu_bugs_smt_update(void) | ||
736 | { | ||
737 | @@ -1129,6 +1260,16 @@ void cpu_bugs_smt_update(void) | ||
738 | break; | ||
739 | } | ||
740 | |||
741 | + switch (mmio_mitigation) { | ||
742 | + case MMIO_MITIGATION_VERW: | ||
743 | + case MMIO_MITIGATION_UCODE_NEEDED: | ||
744 | + if (sched_smt_active()) | ||
745 | + pr_warn_once(MMIO_MSG_SMT); | ||
746 | + break; | ||
747 | + case MMIO_MITIGATION_OFF: | ||
748 | + break; | ||
749 | + } | ||
750 | + | ||
751 | mutex_unlock(&spec_ctrl_mutex); | ||
752 | } | ||
753 | |||
754 | @@ -1692,6 +1833,20 @@ static ssize_t tsx_async_abort_show_state(char *buf) | ||
755 | sched_smt_active() ? "vulnerable" : "disabled"); | ||
756 | } | ||
757 | |||
758 | +static ssize_t mmio_stale_data_show_state(char *buf) | ||
759 | +{ | ||
760 | + if (mmio_mitigation == MMIO_MITIGATION_OFF) | ||
761 | + return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); | ||
762 | + | ||
763 | + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { | ||
764 | + return sysfs_emit(buf, "%s; SMT Host state unknown\n", | ||
765 | + mmio_strings[mmio_mitigation]); | ||
766 | + } | ||
767 | + | ||
768 | + return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], | ||
769 | + sched_smt_active() ? "vulnerable" : "disabled"); | ||
770 | +} | ||
771 | + | ||
772 | static char *stibp_state(void) | ||
773 | { | ||
774 | if (spectre_v2_in_eibrs_mode(spectre_v2_enabled)) | ||
775 | @@ -1792,6 +1947,9 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr | ||
776 | case X86_BUG_SRBDS: | ||
777 | return srbds_show_state(buf); | ||
778 | |||
779 | + case X86_BUG_MMIO_STALE_DATA: | ||
780 | + return mmio_stale_data_show_state(buf); | ||
781 | + | ||
782 | default: | ||
783 | break; | ||
784 | } | ||
785 | @@ -1843,4 +2001,9 @@ ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char * | ||
786 | { | ||
787 | return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); | ||
788 | } | ||
789 | + | ||
790 | +ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) | ||
791 | +{ | ||
792 | + return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); | ||
793 | +} | ||
794 | #endif | ||
795 | diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c | ||
796 | index 4c85ca112a2a9..305f30e45f3d3 100644 | ||
797 | --- a/arch/x86/kernel/cpu/common.c | ||
798 | +++ b/arch/x86/kernel/cpu/common.c | ||
799 | @@ -1099,18 +1099,42 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { | ||
800 | X86_FEATURE_ANY, issues) | ||
801 | |||
802 | #define SRBDS BIT(0) | ||
803 | +/* CPU is affected by X86_BUG_MMIO_STALE_DATA */ | ||
804 | +#define MMIO BIT(1) | ||
805 | +/* CPU is affected by Shared Buffers Data Sampling (SBDS), a variant of X86_BUG_MMIO_STALE_DATA */ | ||
806 | +#define MMIO_SBDS BIT(2) | ||
807 | |||
808 | static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { | ||
809 | VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS), | ||
810 | VULNBL_INTEL_STEPPINGS(HASWELL, X86_STEPPING_ANY, SRBDS), | ||
811 | VULNBL_INTEL_STEPPINGS(HASWELL_L, X86_STEPPING_ANY, SRBDS), | ||
812 | VULNBL_INTEL_STEPPINGS(HASWELL_G, X86_STEPPING_ANY, SRBDS), | ||
813 | + VULNBL_INTEL_STEPPINGS(HASWELL_X, BIT(2) | BIT(4), MMIO), | ||
814 | + VULNBL_INTEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x3, 0x5), MMIO), | ||
815 | VULNBL_INTEL_STEPPINGS(BROADWELL_G, X86_STEPPING_ANY, SRBDS), | ||
816 | + VULNBL_INTEL_STEPPINGS(BROADWELL_X, X86_STEPPING_ANY, MMIO), | ||
817 | VULNBL_INTEL_STEPPINGS(BROADWELL, X86_STEPPING_ANY, SRBDS), | ||
818 | + VULNBL_INTEL_STEPPINGS(SKYLAKE_L, X86_STEPPINGS(0x3, 0x3), SRBDS | MMIO), | ||
819 | VULNBL_INTEL_STEPPINGS(SKYLAKE_L, X86_STEPPING_ANY, SRBDS), | ||
820 | + VULNBL_INTEL_STEPPINGS(SKYLAKE_X, BIT(3) | BIT(4) | BIT(6) | | ||
821 | + BIT(7) | BIT(0xB), MMIO), | ||
822 | + VULNBL_INTEL_STEPPINGS(SKYLAKE, X86_STEPPINGS(0x3, 0x3), SRBDS | MMIO), | ||
823 | VULNBL_INTEL_STEPPINGS(SKYLAKE, X86_STEPPING_ANY, SRBDS), | ||
824 | - VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPINGS(0x0, 0xC), SRBDS), | ||
825 | - VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPINGS(0x0, 0xD), SRBDS), | ||
826 | + VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPINGS(0x9, 0xC), SRBDS | MMIO), | ||
827 | + VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPINGS(0x0, 0x8), SRBDS), | ||
828 | + VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPINGS(0x9, 0xD), SRBDS | MMIO), | ||
829 | + VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPINGS(0x0, 0x8), SRBDS), | ||
830 | + VULNBL_INTEL_STEPPINGS(ICELAKE_L, X86_STEPPINGS(0x5, 0x5), MMIO | MMIO_SBDS), | ||
831 | + VULNBL_INTEL_STEPPINGS(ICELAKE_D, X86_STEPPINGS(0x1, 0x1), MMIO), | ||
832 | + VULNBL_INTEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x4, 0x6), MMIO), | ||
833 | + VULNBL_INTEL_STEPPINGS(COMETLAKE, BIT(2) | BIT(3) | BIT(5), MMIO | MMIO_SBDS), | ||
834 | + VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPINGS(0x1, 0x1), MMIO | MMIO_SBDS), | ||
835 | + VULNBL_INTEL_STEPPINGS(COMETLAKE_L, X86_STEPPINGS(0x0, 0x0), MMIO), | ||
836 | + VULNBL_INTEL_STEPPINGS(LAKEFIELD, X86_STEPPINGS(0x1, 0x1), MMIO | MMIO_SBDS), | ||
837 | + VULNBL_INTEL_STEPPINGS(ROCKETLAKE, X86_STEPPINGS(0x1, 0x1), MMIO), | ||
838 | + VULNBL_INTEL_STEPPINGS(ATOM_TREMONT, X86_STEPPINGS(0x1, 0x1), MMIO | MMIO_SBDS), | ||
839 | + VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_D, X86_STEPPING_ANY, MMIO), | ||
840 | + VULNBL_INTEL_STEPPINGS(ATOM_TREMONT_L, X86_STEPPINGS(0x0, 0x0), MMIO | MMIO_SBDS), | ||
841 | {} | ||
842 | }; | ||
843 | |||
844 | @@ -1131,6 +1155,13 @@ u64 x86_read_arch_cap_msr(void) | ||
845 | return ia32_cap; | ||
846 | } | ||
847 | |||
848 | +static bool arch_cap_mmio_immune(u64 ia32_cap) | ||
849 | +{ | ||
850 | + return (ia32_cap & ARCH_CAP_FBSDP_NO && | ||
851 | + ia32_cap & ARCH_CAP_PSDP_NO && | ||
852 | + ia32_cap & ARCH_CAP_SBDR_SSDP_NO); | ||
853 | +} | ||
854 | + | ||
855 | static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) | ||
856 | { | ||
857 | u64 ia32_cap = x86_read_arch_cap_msr(); | ||
858 | @@ -1184,12 +1215,27 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) | ||
859 | /* | ||
860 | * SRBDS affects CPUs which support RDRAND or RDSEED and are listed | ||
861 | * in the vulnerability blacklist. | ||
862 | + * | ||
863 | + * Some of the implications and mitigation of Shared Buffers Data | ||
864 | + * Sampling (SBDS) are similar to SRBDS. Give SBDS same treatment as | ||
865 | + * SRBDS. | ||
866 | */ | ||
867 | if ((cpu_has(c, X86_FEATURE_RDRAND) || | ||
868 | cpu_has(c, X86_FEATURE_RDSEED)) && | ||
869 | - cpu_matches(cpu_vuln_blacklist, SRBDS)) | ||
870 | + cpu_matches(cpu_vuln_blacklist, SRBDS | MMIO_SBDS)) | ||
871 | setup_force_cpu_bug(X86_BUG_SRBDS); | ||
872 | |||
873 | + /* | ||
874 | + * Processor MMIO Stale Data bug enumeration | ||
875 | + * | ||
876 | + * Affected CPU list is generally enough to enumerate the vulnerability, | ||
877 | + * but for virtualization case check for ARCH_CAP MSR bits also, VMM may | ||
878 | + * not want the guest to enumerate the bug. | ||
879 | + */ | ||
880 | + if (cpu_matches(cpu_vuln_blacklist, MMIO) && | ||
881 | + !arch_cap_mmio_immune(ia32_cap)) | ||
882 | + setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA); | ||
883 | + | ||
884 | if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN)) | ||
885 | return; | ||
886 | |||
887 | diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c | ||
888 | index 0f7c051984062..4bd1bf6214eea 100644 | ||
889 | --- a/arch/x86/kvm/vmx/vmx.c | ||
890 | +++ b/arch/x86/kvm/vmx/vmx.c | ||
891 | @@ -204,6 +204,9 @@ static const struct { | ||
892 | #define L1D_CACHE_ORDER 4 | ||
893 | static void *vmx_l1d_flush_pages; | ||
894 | |||
895 | +/* Control for disabling CPU Fill buffer clear */ | ||
896 | +static bool __read_mostly vmx_fb_clear_ctrl_available; | ||
897 | + | ||
898 | static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) | ||
899 | { | ||
900 | struct page *page; | ||
901 | @@ -335,6 +338,60 @@ static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) | ||
902 | return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); | ||
903 | } | ||
904 | |||
905 | +static void vmx_setup_fb_clear_ctrl(void) | ||
906 | +{ | ||
907 | + u64 msr; | ||
908 | + | ||
909 | + if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES) && | ||
910 | + !boot_cpu_has_bug(X86_BUG_MDS) && | ||
911 | + !boot_cpu_has_bug(X86_BUG_TAA)) { | ||
912 | + rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr); | ||
913 | + if (msr & ARCH_CAP_FB_CLEAR_CTRL) | ||
914 | + vmx_fb_clear_ctrl_available = true; | ||
915 | + } | ||
916 | +} | ||
917 | + | ||
918 | +static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx) | ||
919 | +{ | ||
920 | + u64 msr; | ||
921 | + | ||
922 | + if (!vmx->disable_fb_clear) | ||
923 | + return; | ||
924 | + | ||
925 | + rdmsrl(MSR_IA32_MCU_OPT_CTRL, msr); | ||
926 | + msr |= FB_CLEAR_DIS; | ||
927 | + wrmsrl(MSR_IA32_MCU_OPT_CTRL, msr); | ||
928 | + /* Cache the MSR value to avoid reading it later */ | ||
929 | + vmx->msr_ia32_mcu_opt_ctrl = msr; | ||
930 | +} | ||
931 | + | ||
932 | +static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx) | ||
933 | +{ | ||
934 | + if (!vmx->disable_fb_clear) | ||
935 | + return; | ||
936 | + | ||
937 | + vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS; | ||
938 | + wrmsrl(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); | ||
939 | +} | ||
940 | + | ||
941 | +static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) | ||
942 | +{ | ||
943 | + vmx->disable_fb_clear = vmx_fb_clear_ctrl_available; | ||
944 | + | ||
945 | + /* | ||
946 | + * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS | ||
947 | + * at VMEntry. Skip the MSR read/write when a guest has no use case to | ||
948 | + * execute VERW. | ||
949 | + */ | ||
950 | + if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) || | ||
951 | + ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) && | ||
952 | + (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) && | ||
953 | + (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) && | ||
954 | + (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) && | ||
955 | + (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO))) | ||
956 | + vmx->disable_fb_clear = false; | ||
957 | +} | ||
958 | + | ||
959 | static const struct kernel_param_ops vmentry_l1d_flush_ops = { | ||
960 | .set = vmentry_l1d_flush_set, | ||
961 | .get = vmentry_l1d_flush_get, | ||
962 | @@ -2167,9 +2224,13 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) | ||
963 | } | ||
964 | break; | ||
965 | } | ||
966 | - ret = kvm_set_msr_common(vcpu, msr_info); | ||
967 | + ret = kvm_set_msr_common(vcpu, msr_info); | ||
968 | } | ||
969 | |||
970 | + /* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */ | ||
971 | + if (msr_index == MSR_IA32_ARCH_CAPABILITIES) | ||
972 | + vmx_update_fb_clear_dis(vcpu, vmx); | ||
973 | + | ||
974 | return ret; | ||
975 | } | ||
976 | |||
977 | @@ -4362,6 +4423,8 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) | ||
978 | vpid_sync_context(vmx->vpid); | ||
979 | if (init_event) | ||
980 | vmx_clear_hlt(vcpu); | ||
981 | + | ||
982 | + vmx_update_fb_clear_dis(vcpu, vmx); | ||
983 | } | ||
984 | |||
985 | static void enable_irq_window(struct kvm_vcpu *vcpu) | ||
986 | @@ -6555,6 +6618,11 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) | ||
987 | vmx_l1d_flush(vcpu); | ||
988 | else if (static_branch_unlikely(&mds_user_clear)) | ||
989 | mds_clear_cpu_buffers(); | ||
990 | + else if (static_branch_unlikely(&mmio_stale_data_clear) && | ||
991 | + kvm_arch_has_assigned_device(vcpu->kvm)) | ||
992 | + mds_clear_cpu_buffers(); | ||
993 | + | ||
994 | + vmx_disable_fb_clear(vmx); | ||
995 | |||
996 | if (vcpu->arch.cr2 != read_cr2()) | ||
997 | write_cr2(vcpu->arch.cr2); | ||
998 | @@ -6564,6 +6632,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) | ||
999 | |||
1000 | vcpu->arch.cr2 = read_cr2(); | ||
1001 | |||
1002 | + vmx_enable_fb_clear(vmx); | ||
1003 | + | ||
1004 | /* | ||
1005 | * We do not use IBRS in the kernel. If this vCPU has used the | ||
1006 | * SPEC_CTRL MSR it may have left it on; save the value and | ||
1007 | @@ -8038,8 +8108,11 @@ static int __init vmx_init(void) | ||
1008 | return r; | ||
1009 | } | ||
1010 | |||
1011 | + vmx_setup_fb_clear_ctrl(); | ||
1012 | + | ||
1013 | for_each_possible_cpu(cpu) { | ||
1014 | INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); | ||
1015 | + | ||
1016 | INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu)); | ||
1017 | spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); | ||
1018 | } | ||
1019 | diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h | ||
1020 | index 55731dd0096f2..7a3362ab59867 100644 | ||
1021 | --- a/arch/x86/kvm/vmx/vmx.h | ||
1022 | +++ b/arch/x86/kvm/vmx/vmx.h | ||
1023 | @@ -280,8 +280,11 @@ struct vcpu_vmx { | ||
1024 | u64 msr_ia32_feature_control; | ||
1025 | u64 msr_ia32_feature_control_valid_bits; | ||
1026 | u64 ept_pointer; | ||
1027 | + u64 msr_ia32_mcu_opt_ctrl; | ||
1028 | + bool disable_fb_clear; | ||
1029 | |||
1030 | struct pt_desc pt_desc; | ||
1031 | + | ||
1032 | }; | ||
1033 | |||
1034 | enum ept_pointers_status { | ||
1035 | diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c | ||
1036 | index 6dd77e426889b..d0b297583df88 100644 | ||
1037 | --- a/arch/x86/kvm/x86.c | ||
1038 | +++ b/arch/x86/kvm/x86.c | ||
1039 | @@ -1403,6 +1403,10 @@ static u64 kvm_get_arch_capabilities(void) | ||
1040 | |||
1041 | /* KVM does not emulate MSR_IA32_TSX_CTRL. */ | ||
1042 | data &= ~ARCH_CAP_TSX_CTRL_MSR; | ||
1043 | + | ||
1044 | + /* Guests don't need to know "Fill buffer clear control" exists */ | ||
1045 | + data &= ~ARCH_CAP_FB_CLEAR_CTRL; | ||
1046 | + | ||
1047 | return data; | ||
1048 | } | ||
1049 | |||
1050 | diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c | ||
1051 | index 81ec0a1020d60..9b5edf1dfe9e9 100644 | ||
1052 | --- a/drivers/base/cpu.c | ||
1053 | +++ b/drivers/base/cpu.c | ||
1054 | @@ -568,6 +568,12 @@ ssize_t __weak cpu_show_srbds(struct device *dev, | ||
1055 | return sysfs_emit(buf, "Not affected\n"); | ||
1056 | } | ||
1057 | |||
1058 | +ssize_t __weak cpu_show_mmio_stale_data(struct device *dev, | ||
1059 | + struct device_attribute *attr, char *buf) | ||
1060 | +{ | ||
1061 | + return sysfs_emit(buf, "Not affected\n"); | ||
1062 | +} | ||
1063 | + | ||
1064 | static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL); | ||
1065 | static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL); | ||
1066 | static DEVICE_ATTR(spectre_v2, 0444, cpu_show_spectre_v2, NULL); | ||
1067 | @@ -577,6 +583,7 @@ static DEVICE_ATTR(mds, 0444, cpu_show_mds, NULL); | ||
1068 | static DEVICE_ATTR(tsx_async_abort, 0444, cpu_show_tsx_async_abort, NULL); | ||
1069 | static DEVICE_ATTR(itlb_multihit, 0444, cpu_show_itlb_multihit, NULL); | ||
1070 | static DEVICE_ATTR(srbds, 0444, cpu_show_srbds, NULL); | ||
1071 | +static DEVICE_ATTR(mmio_stale_data, 0444, cpu_show_mmio_stale_data, NULL); | ||
1072 | |||
1073 | static struct attribute *cpu_root_vulnerabilities_attrs[] = { | ||
1074 | &dev_attr_meltdown.attr, | ||
1075 | @@ -588,6 +595,7 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = { | ||
1076 | &dev_attr_tsx_async_abort.attr, | ||
1077 | &dev_attr_itlb_multihit.attr, | ||
1078 | &dev_attr_srbds.attr, | ||
1079 | + &dev_attr_mmio_stale_data.attr, | ||
1080 | NULL | ||
1081 | }; | ||
1082 | |||
1083 | diff --git a/include/linux/cpu.h b/include/linux/cpu.h | ||
1084 | index 4e9822cb11f38..29a6fa2f518db 100644 | ||
1085 | --- a/include/linux/cpu.h | ||
1086 | +++ b/include/linux/cpu.h | ||
1087 | @@ -64,6 +64,10 @@ extern ssize_t cpu_show_tsx_async_abort(struct device *dev, | ||
1088 | char *buf); | ||
1089 | extern ssize_t cpu_show_itlb_multihit(struct device *dev, | ||
1090 | struct device_attribute *attr, char *buf); | ||
1091 | +extern ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf); | ||
1092 | +extern ssize_t cpu_show_mmio_stale_data(struct device *dev, | ||
1093 | + struct device_attribute *attr, | ||
1094 | + char *buf); | ||
1095 | |||
1096 | extern __printf(4, 5) | ||
1097 | struct device *cpu_device_create(struct device *parent, void *drvdata, |