Annotation of /trunk/kernel-alx/patches-5.4/0178-5.4.79-all-fixes.patch
Parent Directory | Revision Log
Revision 3635 -
(hide annotations)
(download)
Mon Oct 24 12:34:12 2022 UTC (20 months ago) by niro
File size: 56223 byte(s)
Mon Oct 24 12:34:12 2022 UTC (20 months ago) by niro
File size: 56223 byte(s)
-sync kernel patches
1 | niro | 3635 | diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt |
2 | index 5b4753e602def..fea15cd49fbc7 100644 | ||
3 | --- a/Documentation/admin-guide/kernel-parameters.txt | ||
4 | +++ b/Documentation/admin-guide/kernel-parameters.txt | ||
5 | @@ -2667,6 +2667,8 @@ | ||
6 | mds=off [X86] | ||
7 | tsx_async_abort=off [X86] | ||
8 | kvm.nx_huge_pages=off [X86] | ||
9 | + no_entry_flush [PPC] | ||
10 | + no_uaccess_flush [PPC] | ||
11 | |||
12 | Exceptions: | ||
13 | This does not have any effect on | ||
14 | @@ -2989,6 +2991,8 @@ | ||
15 | |||
16 | noefi Disable EFI runtime services support. | ||
17 | |||
18 | + no_entry_flush [PPC] Don't flush the L1-D cache when entering the kernel. | ||
19 | + | ||
20 | noexec [IA-64] | ||
21 | |||
22 | noexec [X86] | ||
23 | @@ -3038,6 +3042,9 @@ | ||
24 | nospec_store_bypass_disable | ||
25 | [HW] Disable all mitigations for the Speculative Store Bypass vulnerability | ||
26 | |||
27 | + no_uaccess_flush | ||
28 | + [PPC] Don't flush the L1-D cache after accessing user data. | ||
29 | + | ||
30 | noxsave [BUGS=X86] Disables x86 extended register state save | ||
31 | and restore using xsave. The kernel will fallback to | ||
32 | enabling legacy floating-point and sse state. | ||
33 | diff --git a/Makefile b/Makefile | ||
34 | index 5725b07aaddf0..f02539be5e073 100644 | ||
35 | --- a/Makefile | ||
36 | +++ b/Makefile | ||
37 | @@ -1,7 +1,7 @@ | ||
38 | # SPDX-License-Identifier: GPL-2.0 | ||
39 | VERSION = 5 | ||
40 | PATCHLEVEL = 4 | ||
41 | -SUBLEVEL = 78 | ||
42 | +SUBLEVEL = 79 | ||
43 | EXTRAVERSION = | ||
44 | NAME = Kleptomaniac Octopus | ||
45 | |||
46 | diff --git a/arch/mips/pci/pci-xtalk-bridge.c b/arch/mips/pci/pci-xtalk-bridge.c | ||
47 | index c4b1c6cf26606..adc9f83b2c448 100644 | ||
48 | --- a/arch/mips/pci/pci-xtalk-bridge.c | ||
49 | +++ b/arch/mips/pci/pci-xtalk-bridge.c | ||
50 | @@ -284,7 +284,7 @@ static int bridge_set_affinity(struct irq_data *d, const struct cpumask *mask, | ||
51 | ret = irq_chip_set_affinity_parent(d, mask, force); | ||
52 | if (ret >= 0) { | ||
53 | cpu = cpumask_first_and(mask, cpu_online_mask); | ||
54 | - data->nnasid = COMPACT_TO_NASID_NODEID(cpu_to_node(cpu)); | ||
55 | + data->nasid = COMPACT_TO_NASID_NODEID(cpu_to_node(cpu)); | ||
56 | bridge_write(data->bc, b_int_addr[pin].addr, | ||
57 | (((data->bc->intr_addr >> 30) & 0x30000) | | ||
58 | bit | (data->nasid << 8))); | ||
59 | diff --git a/arch/powerpc/include/asm/book3s/64/kup-radix.h b/arch/powerpc/include/asm/book3s/64/kup-radix.h | ||
60 | index c8d1076e0ebbf..c1e45f510591e 100644 | ||
61 | --- a/arch/powerpc/include/asm/book3s/64/kup-radix.h | ||
62 | +++ b/arch/powerpc/include/asm/book3s/64/kup-radix.h | ||
63 | @@ -11,13 +11,12 @@ | ||
64 | |||
65 | #ifdef __ASSEMBLY__ | ||
66 | |||
67 | -.macro kuap_restore_amr gpr | ||
68 | #ifdef CONFIG_PPC_KUAP | ||
69 | +.macro kuap_restore_amr gpr | ||
70 | BEGIN_MMU_FTR_SECTION_NESTED(67) | ||
71 | ld \gpr, STACK_REGS_KUAP(r1) | ||
72 | mtspr SPRN_AMR, \gpr | ||
73 | END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_RADIX_KUAP, 67) | ||
74 | -#endif | ||
75 | .endm | ||
76 | |||
77 | .macro kuap_check_amr gpr1, gpr2 | ||
78 | @@ -31,6 +30,7 @@ | ||
79 | END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_RADIX_KUAP, 67) | ||
80 | #endif | ||
81 | .endm | ||
82 | +#endif | ||
83 | |||
84 | .macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr | ||
85 | #ifdef CONFIG_PPC_KUAP | ||
86 | @@ -54,6 +54,8 @@ | ||
87 | |||
88 | #else /* !__ASSEMBLY__ */ | ||
89 | |||
90 | +DECLARE_STATIC_KEY_FALSE(uaccess_flush_key); | ||
91 | + | ||
92 | #ifdef CONFIG_PPC_KUAP | ||
93 | |||
94 | #include <asm/reg.h> | ||
95 | @@ -77,6 +79,18 @@ static inline void set_kuap(unsigned long value) | ||
96 | isync(); | ||
97 | } | ||
98 | |||
99 | +static inline bool | ||
100 | +bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) | ||
101 | +{ | ||
102 | + return WARN(mmu_has_feature(MMU_FTR_RADIX_KUAP) && | ||
103 | + (regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)), | ||
104 | + "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read"); | ||
105 | +} | ||
106 | +#else /* CONFIG_PPC_KUAP */ | ||
107 | +static inline void kuap_restore_amr(struct pt_regs *regs, unsigned long amr) { } | ||
108 | +static inline void set_kuap(unsigned long value) { } | ||
109 | +#endif /* !CONFIG_PPC_KUAP */ | ||
110 | + | ||
111 | static __always_inline void allow_user_access(void __user *to, const void __user *from, | ||
112 | unsigned long size, unsigned long dir) | ||
113 | { | ||
114 | @@ -94,17 +108,10 @@ static inline void prevent_user_access(void __user *to, const void __user *from, | ||
115 | unsigned long size, unsigned long dir) | ||
116 | { | ||
117 | set_kuap(AMR_KUAP_BLOCKED); | ||
118 | + if (static_branch_unlikely(&uaccess_flush_key)) | ||
119 | + do_uaccess_flush(); | ||
120 | } | ||
121 | |||
122 | -static inline bool | ||
123 | -bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) | ||
124 | -{ | ||
125 | - return WARN(mmu_has_feature(MMU_FTR_RADIX_KUAP) && | ||
126 | - (regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)), | ||
127 | - "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read"); | ||
128 | -} | ||
129 | -#endif /* CONFIG_PPC_KUAP */ | ||
130 | - | ||
131 | #endif /* __ASSEMBLY__ */ | ||
132 | |||
133 | #endif /* _ASM_POWERPC_BOOK3S_64_KUP_RADIX_H */ | ||
134 | diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h | ||
135 | index 33f4f72eb035b..6d0795d7b89c1 100644 | ||
136 | --- a/arch/powerpc/include/asm/exception-64s.h | ||
137 | +++ b/arch/powerpc/include/asm/exception-64s.h | ||
138 | @@ -61,11 +61,18 @@ | ||
139 | nop; \ | ||
140 | nop | ||
141 | |||
142 | +#define ENTRY_FLUSH_SLOT \ | ||
143 | + ENTRY_FLUSH_FIXUP_SECTION; \ | ||
144 | + nop; \ | ||
145 | + nop; \ | ||
146 | + nop; | ||
147 | + | ||
148 | /* | ||
149 | * r10 must be free to use, r13 must be paca | ||
150 | */ | ||
151 | #define INTERRUPT_TO_KERNEL \ | ||
152 | - STF_ENTRY_BARRIER_SLOT | ||
153 | + STF_ENTRY_BARRIER_SLOT; \ | ||
154 | + ENTRY_FLUSH_SLOT | ||
155 | |||
156 | /* | ||
157 | * Macros for annotating the expected destination of (h)rfid | ||
158 | @@ -127,6 +134,9 @@ | ||
159 | hrfid; \ | ||
160 | b hrfi_flush_fallback | ||
161 | |||
162 | +#else /* __ASSEMBLY__ */ | ||
163 | +/* Prototype for function defined in exceptions-64s.S */ | ||
164 | +void do_uaccess_flush(void); | ||
165 | #endif /* __ASSEMBLY__ */ | ||
166 | |||
167 | #endif /* _ASM_POWERPC_EXCEPTION_H */ | ||
168 | diff --git a/arch/powerpc/include/asm/feature-fixups.h b/arch/powerpc/include/asm/feature-fixups.h | ||
169 | index b0af97add7517..fbd406cd6916c 100644 | ||
170 | --- a/arch/powerpc/include/asm/feature-fixups.h | ||
171 | +++ b/arch/powerpc/include/asm/feature-fixups.h | ||
172 | @@ -205,6 +205,22 @@ label##3: \ | ||
173 | FTR_ENTRY_OFFSET 955b-956b; \ | ||
174 | .popsection; | ||
175 | |||
176 | +#define UACCESS_FLUSH_FIXUP_SECTION \ | ||
177 | +959: \ | ||
178 | + .pushsection __uaccess_flush_fixup,"a"; \ | ||
179 | + .align 2; \ | ||
180 | +960: \ | ||
181 | + FTR_ENTRY_OFFSET 959b-960b; \ | ||
182 | + .popsection; | ||
183 | + | ||
184 | +#define ENTRY_FLUSH_FIXUP_SECTION \ | ||
185 | +957: \ | ||
186 | + .pushsection __entry_flush_fixup,"a"; \ | ||
187 | + .align 2; \ | ||
188 | +958: \ | ||
189 | + FTR_ENTRY_OFFSET 957b-958b; \ | ||
190 | + .popsection; | ||
191 | + | ||
192 | #define RFI_FLUSH_FIXUP_SECTION \ | ||
193 | 951: \ | ||
194 | .pushsection __rfi_flush_fixup,"a"; \ | ||
195 | @@ -237,8 +253,11 @@ label##3: \ | ||
196 | #include <linux/types.h> | ||
197 | |||
198 | extern long stf_barrier_fallback; | ||
199 | +extern long entry_flush_fallback; | ||
200 | extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup; | ||
201 | extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup; | ||
202 | +extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup; | ||
203 | +extern long __start___entry_flush_fixup, __stop___entry_flush_fixup; | ||
204 | extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup; | ||
205 | extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup; | ||
206 | extern long __start__btb_flush_fixup, __stop__btb_flush_fixup; | ||
207 | diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h | ||
208 | index 94f24928916a8..ed4f5f536fc1d 100644 | ||
209 | --- a/arch/powerpc/include/asm/kup.h | ||
210 | +++ b/arch/powerpc/include/asm/kup.h | ||
211 | @@ -6,7 +6,7 @@ | ||
212 | #define KUAP_WRITE 2 | ||
213 | #define KUAP_READ_WRITE (KUAP_READ | KUAP_WRITE) | ||
214 | |||
215 | -#ifdef CONFIG_PPC64 | ||
216 | +#ifdef CONFIG_PPC_BOOK3S_64 | ||
217 | #include <asm/book3s/64/kup-radix.h> | ||
218 | #endif | ||
219 | #ifdef CONFIG_PPC_8xx | ||
220 | @@ -24,9 +24,15 @@ | ||
221 | .macro kuap_restore sp, current, gpr1, gpr2, gpr3 | ||
222 | .endm | ||
223 | |||
224 | +.macro kuap_restore_amr gpr | ||
225 | +.endm | ||
226 | + | ||
227 | .macro kuap_check current, gpr | ||
228 | .endm | ||
229 | |||
230 | +.macro kuap_check_amr gpr1, gpr2 | ||
231 | +.endm | ||
232 | + | ||
233 | #endif | ||
234 | |||
235 | #else /* !__ASSEMBLY__ */ | ||
236 | @@ -45,15 +51,26 @@ static inline void setup_kuep(bool disabled) { } | ||
237 | void setup_kuap(bool disabled); | ||
238 | #else | ||
239 | static inline void setup_kuap(bool disabled) { } | ||
240 | -static inline void allow_user_access(void __user *to, const void __user *from, | ||
241 | - unsigned long size, unsigned long dir) { } | ||
242 | -static inline void prevent_user_access(void __user *to, const void __user *from, | ||
243 | - unsigned long size, unsigned long dir) { } | ||
244 | + | ||
245 | static inline bool | ||
246 | bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) | ||
247 | { | ||
248 | return false; | ||
249 | } | ||
250 | + | ||
251 | +static inline void kuap_check_amr(void) { } | ||
252 | + | ||
253 | +/* | ||
254 | + * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush | ||
255 | + * the L1D cache after user accesses. Only include the empty stubs for other | ||
256 | + * platforms. | ||
257 | + */ | ||
258 | +#ifndef CONFIG_PPC_BOOK3S_64 | ||
259 | +static inline void allow_user_access(void __user *to, const void __user *from, | ||
260 | + unsigned long size, unsigned long dir) { } | ||
261 | +static inline void prevent_user_access(void __user *to, const void __user *from, | ||
262 | + unsigned long size, unsigned long dir) { } | ||
263 | +#endif /* CONFIG_PPC_BOOK3S_64 */ | ||
264 | #endif /* CONFIG_PPC_KUAP */ | ||
265 | |||
266 | static inline void allow_read_from_user(const void __user *from, unsigned long size) | ||
267 | diff --git a/arch/powerpc/include/asm/security_features.h b/arch/powerpc/include/asm/security_features.h | ||
268 | index 7c05e95a5c444..e9e3f85134e54 100644 | ||
269 | --- a/arch/powerpc/include/asm/security_features.h | ||
270 | +++ b/arch/powerpc/include/asm/security_features.h | ||
271 | @@ -84,12 +84,19 @@ static inline bool security_ftr_enabled(u64 feature) | ||
272 | // Software required to flush link stack on context switch | ||
273 | #define SEC_FTR_FLUSH_LINK_STACK 0x0000000000001000ull | ||
274 | |||
275 | +// The L1-D cache should be flushed when entering the kernel | ||
276 | +#define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull | ||
277 | + | ||
278 | +// The L1-D cache should be flushed after user accesses from the kernel | ||
279 | +#define SEC_FTR_L1D_FLUSH_UACCESS 0x0000000000008000ull | ||
280 | |||
281 | // Features enabled by default | ||
282 | #define SEC_FTR_DEFAULT \ | ||
283 | (SEC_FTR_L1D_FLUSH_HV | \ | ||
284 | SEC_FTR_L1D_FLUSH_PR | \ | ||
285 | SEC_FTR_BNDS_CHK_SPEC_BAR | \ | ||
286 | + SEC_FTR_L1D_FLUSH_ENTRY | \ | ||
287 | + SEC_FTR_L1D_FLUSH_UACCESS | \ | ||
288 | SEC_FTR_FAVOUR_SECURITY) | ||
289 | |||
290 | #endif /* _ASM_POWERPC_SECURITY_FEATURES_H */ | ||
291 | diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h | ||
292 | index 65676e2325b85..6f2f4497e13b3 100644 | ||
293 | --- a/arch/powerpc/include/asm/setup.h | ||
294 | +++ b/arch/powerpc/include/asm/setup.h | ||
295 | @@ -52,12 +52,16 @@ enum l1d_flush_type { | ||
296 | }; | ||
297 | |||
298 | void setup_rfi_flush(enum l1d_flush_type, bool enable); | ||
299 | +void setup_entry_flush(bool enable); | ||
300 | +void setup_uaccess_flush(bool enable); | ||
301 | void do_rfi_flush_fixups(enum l1d_flush_type types); | ||
302 | #ifdef CONFIG_PPC_BARRIER_NOSPEC | ||
303 | void setup_barrier_nospec(void); | ||
304 | #else | ||
305 | static inline void setup_barrier_nospec(void) { }; | ||
306 | #endif | ||
307 | +void do_uaccess_flush_fixups(enum l1d_flush_type types); | ||
308 | +void do_entry_flush_fixups(enum l1d_flush_type types); | ||
309 | void do_barrier_nospec_fixups(bool enable); | ||
310 | extern bool barrier_nospec_enabled; | ||
311 | |||
312 | diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S | ||
313 | index 70ac8a6ba0c18..88bba0a931d65 100644 | ||
314 | --- a/arch/powerpc/kernel/exceptions-64s.S | ||
315 | +++ b/arch/powerpc/kernel/exceptions-64s.S | ||
316 | @@ -1150,7 +1150,7 @@ EXC_REAL_BEGIN(data_access, 0x300, 0x80) | ||
317 | INT_HANDLER data_access, 0x300, ool=1, dar=1, dsisr=1, kvm=1 | ||
318 | EXC_REAL_END(data_access, 0x300, 0x80) | ||
319 | EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) | ||
320 | - INT_HANDLER data_access, 0x300, virt=1, dar=1, dsisr=1 | ||
321 | + INT_HANDLER data_access, 0x300, ool=1, virt=1, dar=1, dsisr=1 | ||
322 | EXC_VIRT_END(data_access, 0x4300, 0x80) | ||
323 | INT_KVM_HANDLER data_access, 0x300, EXC_STD, PACA_EXGEN, 1 | ||
324 | EXC_COMMON_BEGIN(data_access_common) | ||
325 | @@ -1205,7 +1205,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) | ||
326 | |||
327 | |||
328 | EXC_REAL_BEGIN(instruction_access, 0x400, 0x80) | ||
329 | - INT_HANDLER instruction_access, 0x400, kvm=1 | ||
330 | + INT_HANDLER instruction_access, 0x400, ool=1, kvm=1 | ||
331 | EXC_REAL_END(instruction_access, 0x400, 0x80) | ||
332 | EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) | ||
333 | INT_HANDLER instruction_access, 0x400, virt=1 | ||
334 | @@ -1225,7 +1225,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) | ||
335 | |||
336 | |||
337 | EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) | ||
338 | - INT_HANDLER instruction_access_slb, 0x480, area=PACA_EXSLB, kvm=1 | ||
339 | + INT_HANDLER instruction_access_slb, 0x480, ool=1, area=PACA_EXSLB, kvm=1 | ||
340 | EXC_REAL_END(instruction_access_slb, 0x480, 0x80) | ||
341 | EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) | ||
342 | INT_HANDLER instruction_access_slb, 0x480, virt=1, area=PACA_EXSLB | ||
343 | @@ -1365,17 +1365,17 @@ EXC_REAL_BEGIN(decrementer, 0x900, 0x80) | ||
344 | INT_HANDLER decrementer, 0x900, ool=1, bitmask=IRQS_DISABLED, kvm=1 | ||
345 | EXC_REAL_END(decrementer, 0x900, 0x80) | ||
346 | EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) | ||
347 | - INT_HANDLER decrementer, 0x900, virt=1, bitmask=IRQS_DISABLED | ||
348 | + INT_HANDLER decrementer, 0x900, ool=1, virt=1, bitmask=IRQS_DISABLED | ||
349 | EXC_VIRT_END(decrementer, 0x4900, 0x80) | ||
350 | INT_KVM_HANDLER decrementer, 0x900, EXC_STD, PACA_EXGEN, 0 | ||
351 | EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt) | ||
352 | |||
353 | |||
354 | EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) | ||
355 | - INT_HANDLER hdecrementer, 0x980, hsrr=EXC_HV, kvm=1 | ||
356 | + INT_HANDLER hdecrementer, 0x980, ool=1, hsrr=EXC_HV, kvm=1 | ||
357 | EXC_REAL_END(hdecrementer, 0x980, 0x80) | ||
358 | EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) | ||
359 | - INT_HANDLER hdecrementer, 0x980, virt=1, hsrr=EXC_HV, kvm=1 | ||
360 | + INT_HANDLER hdecrementer, 0x980, ool=1, virt=1, hsrr=EXC_HV, kvm=1 | ||
361 | EXC_VIRT_END(hdecrementer, 0x4980, 0x80) | ||
362 | INT_KVM_HANDLER hdecrementer, 0x980, EXC_HV, PACA_EXGEN, 0 | ||
363 | EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt) | ||
364 | @@ -2046,15 +2046,8 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback) | ||
365 | .endr | ||
366 | blr | ||
367 | |||
368 | -TRAMP_REAL_BEGIN(rfi_flush_fallback) | ||
369 | - SET_SCRATCH0(r13); | ||
370 | - GET_PACA(r13); | ||
371 | - std r1,PACA_EXRFI+EX_R12(r13) | ||
372 | - ld r1,PACAKSAVE(r13) | ||
373 | - std r9,PACA_EXRFI+EX_R9(r13) | ||
374 | - std r10,PACA_EXRFI+EX_R10(r13) | ||
375 | - std r11,PACA_EXRFI+EX_R11(r13) | ||
376 | - mfctr r9 | ||
377 | +/* Clobbers r10, r11, ctr */ | ||
378 | +.macro L1D_DISPLACEMENT_FLUSH | ||
379 | ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) | ||
380 | ld r11,PACA_L1D_FLUSH_SIZE(r13) | ||
381 | srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ | ||
382 | @@ -2065,7 +2058,7 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback) | ||
383 | sync | ||
384 | |||
385 | /* | ||
386 | - * The load adresses are at staggered offsets within cachelines, | ||
387 | + * The load addresses are at staggered offsets within cachelines, | ||
388 | * which suits some pipelines better (on others it should not | ||
389 | * hurt). | ||
390 | */ | ||
391 | @@ -2080,7 +2073,30 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback) | ||
392 | ld r11,(0x80 + 8)*7(r10) | ||
393 | addi r10,r10,0x80*8 | ||
394 | bdnz 1b | ||
395 | +.endm | ||
396 | + | ||
397 | +TRAMP_REAL_BEGIN(entry_flush_fallback) | ||
398 | + std r9,PACA_EXRFI+EX_R9(r13) | ||
399 | + std r10,PACA_EXRFI+EX_R10(r13) | ||
400 | + std r11,PACA_EXRFI+EX_R11(r13) | ||
401 | + mfctr r9 | ||
402 | + L1D_DISPLACEMENT_FLUSH | ||
403 | + mtctr r9 | ||
404 | + ld r9,PACA_EXRFI+EX_R9(r13) | ||
405 | + ld r10,PACA_EXRFI+EX_R10(r13) | ||
406 | + ld r11,PACA_EXRFI+EX_R11(r13) | ||
407 | + blr | ||
408 | |||
409 | +TRAMP_REAL_BEGIN(rfi_flush_fallback) | ||
410 | + SET_SCRATCH0(r13); | ||
411 | + GET_PACA(r13); | ||
412 | + std r1,PACA_EXRFI+EX_R12(r13) | ||
413 | + ld r1,PACAKSAVE(r13) | ||
414 | + std r9,PACA_EXRFI+EX_R9(r13) | ||
415 | + std r10,PACA_EXRFI+EX_R10(r13) | ||
416 | + std r11,PACA_EXRFI+EX_R11(r13) | ||
417 | + mfctr r9 | ||
418 | + L1D_DISPLACEMENT_FLUSH | ||
419 | mtctr r9 | ||
420 | ld r9,PACA_EXRFI+EX_R9(r13) | ||
421 | ld r10,PACA_EXRFI+EX_R10(r13) | ||
422 | @@ -2098,32 +2114,7 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) | ||
423 | std r10,PACA_EXRFI+EX_R10(r13) | ||
424 | std r11,PACA_EXRFI+EX_R11(r13) | ||
425 | mfctr r9 | ||
426 | - ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) | ||
427 | - ld r11,PACA_L1D_FLUSH_SIZE(r13) | ||
428 | - srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ | ||
429 | - mtctr r11 | ||
430 | - DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ | ||
431 | - | ||
432 | - /* order ld/st prior to dcbt stop all streams with flushing */ | ||
433 | - sync | ||
434 | - | ||
435 | - /* | ||
436 | - * The load adresses are at staggered offsets within cachelines, | ||
437 | - * which suits some pipelines better (on others it should not | ||
438 | - * hurt). | ||
439 | - */ | ||
440 | -1: | ||
441 | - ld r11,(0x80 + 8)*0(r10) | ||
442 | - ld r11,(0x80 + 8)*1(r10) | ||
443 | - ld r11,(0x80 + 8)*2(r10) | ||
444 | - ld r11,(0x80 + 8)*3(r10) | ||
445 | - ld r11,(0x80 + 8)*4(r10) | ||
446 | - ld r11,(0x80 + 8)*5(r10) | ||
447 | - ld r11,(0x80 + 8)*6(r10) | ||
448 | - ld r11,(0x80 + 8)*7(r10) | ||
449 | - addi r10,r10,0x80*8 | ||
450 | - bdnz 1b | ||
451 | - | ||
452 | + L1D_DISPLACEMENT_FLUSH | ||
453 | mtctr r9 | ||
454 | ld r9,PACA_EXRFI+EX_R9(r13) | ||
455 | ld r10,PACA_EXRFI+EX_R10(r13) | ||
456 | @@ -2132,6 +2123,19 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) | ||
457 | GET_SCRATCH0(r13); | ||
458 | hrfid | ||
459 | |||
460 | +USE_TEXT_SECTION() | ||
461 | + | ||
462 | +_GLOBAL(do_uaccess_flush) | ||
463 | + UACCESS_FLUSH_FIXUP_SECTION | ||
464 | + nop | ||
465 | + nop | ||
466 | + nop | ||
467 | + blr | ||
468 | + L1D_DISPLACEMENT_FLUSH | ||
469 | + blr | ||
470 | +_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) | ||
471 | +EXPORT_SYMBOL(do_uaccess_flush) | ||
472 | + | ||
473 | /* | ||
474 | * Real mode exceptions actually use this too, but alternate | ||
475 | * instruction code patches (which end up in the common .text area) | ||
476 | diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S | ||
477 | index 98d8b6832fcb5..f6428b90a6c77 100644 | ||
478 | --- a/arch/powerpc/kernel/head_8xx.S | ||
479 | +++ b/arch/powerpc/kernel/head_8xx.S | ||
480 | @@ -229,9 +229,7 @@ SystemCall: | ||
481 | |||
482 | InstructionTLBMiss: | ||
483 | mtspr SPRN_SPRG_SCRATCH0, r10 | ||
484 | -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) | ||
485 | mtspr SPRN_SPRG_SCRATCH1, r11 | ||
486 | -#endif | ||
487 | |||
488 | /* If we are faulting a kernel address, we have to use the | ||
489 | * kernel page tables. | ||
490 | @@ -278,11 +276,9 @@ InstructionTLBMiss: | ||
491 | #ifdef ITLB_MISS_KERNEL | ||
492 | mtcr r11 | ||
493 | #endif | ||
494 | -#ifdef CONFIG_SWAP | ||
495 | - rlwinm r11, r10, 32-5, _PAGE_PRESENT | ||
496 | + rlwinm r11, r10, 32-7, _PAGE_PRESENT | ||
497 | and r11, r11, r10 | ||
498 | rlwimi r10, r11, 0, _PAGE_PRESENT | ||
499 | -#endif | ||
500 | /* The Linux PTE won't go exactly into the MMU TLB. | ||
501 | * Software indicator bits 20 and 23 must be clear. | ||
502 | * Software indicator bits 22, 24, 25, 26, and 27 must be | ||
503 | @@ -296,9 +292,7 @@ InstructionTLBMiss: | ||
504 | |||
505 | /* Restore registers */ | ||
506 | 0: mfspr r10, SPRN_SPRG_SCRATCH0 | ||
507 | -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) | ||
508 | mfspr r11, SPRN_SPRG_SCRATCH1 | ||
509 | -#endif | ||
510 | rfi | ||
511 | patch_site 0b, patch__itlbmiss_exit_1 | ||
512 | |||
513 | @@ -308,9 +302,7 @@ InstructionTLBMiss: | ||
514 | addi r10, r10, 1 | ||
515 | stw r10, (itlb_miss_counter - PAGE_OFFSET)@l(0) | ||
516 | mfspr r10, SPRN_SPRG_SCRATCH0 | ||
517 | -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) | ||
518 | mfspr r11, SPRN_SPRG_SCRATCH1 | ||
519 | -#endif | ||
520 | rfi | ||
521 | #endif | ||
522 | |||
523 | @@ -394,11 +386,9 @@ DataStoreTLBMiss: | ||
524 | * r11 = ((r10 & PRESENT) & ((r10 & ACCESSED) >> 5)); | ||
525 | * r10 = (r10 & ~PRESENT) | r11; | ||
526 | */ | ||
527 | -#ifdef CONFIG_SWAP | ||
528 | - rlwinm r11, r10, 32-5, _PAGE_PRESENT | ||
529 | + rlwinm r11, r10, 32-7, _PAGE_PRESENT | ||
530 | and r11, r11, r10 | ||
531 | rlwimi r10, r11, 0, _PAGE_PRESENT | ||
532 | -#endif | ||
533 | /* The Linux PTE won't go exactly into the MMU TLB. | ||
534 | * Software indicator bits 24, 25, 26, and 27 must be | ||
535 | * set. All other Linux PTE bits control the behavior | ||
536 | diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c | ||
537 | index e50fbed366516..480c236724da2 100644 | ||
538 | --- a/arch/powerpc/kernel/setup_64.c | ||
539 | +++ b/arch/powerpc/kernel/setup_64.c | ||
540 | @@ -859,7 +859,13 @@ early_initcall(disable_hardlockup_detector); | ||
541 | static enum l1d_flush_type enabled_flush_types; | ||
542 | static void *l1d_flush_fallback_area; | ||
543 | static bool no_rfi_flush; | ||
544 | +static bool no_entry_flush; | ||
545 | +static bool no_uaccess_flush; | ||
546 | bool rfi_flush; | ||
547 | +bool entry_flush; | ||
548 | +bool uaccess_flush; | ||
549 | +DEFINE_STATIC_KEY_FALSE(uaccess_flush_key); | ||
550 | +EXPORT_SYMBOL(uaccess_flush_key); | ||
551 | |||
552 | static int __init handle_no_rfi_flush(char *p) | ||
553 | { | ||
554 | @@ -869,6 +875,22 @@ static int __init handle_no_rfi_flush(char *p) | ||
555 | } | ||
556 | early_param("no_rfi_flush", handle_no_rfi_flush); | ||
557 | |||
558 | +static int __init handle_no_entry_flush(char *p) | ||
559 | +{ | ||
560 | + pr_info("entry-flush: disabled on command line."); | ||
561 | + no_entry_flush = true; | ||
562 | + return 0; | ||
563 | +} | ||
564 | +early_param("no_entry_flush", handle_no_entry_flush); | ||
565 | + | ||
566 | +static int __init handle_no_uaccess_flush(char *p) | ||
567 | +{ | ||
568 | + pr_info("uaccess-flush: disabled on command line."); | ||
569 | + no_uaccess_flush = true; | ||
570 | + return 0; | ||
571 | +} | ||
572 | +early_param("no_uaccess_flush", handle_no_uaccess_flush); | ||
573 | + | ||
574 | /* | ||
575 | * The RFI flush is not KPTI, but because users will see doco that says to use | ||
576 | * nopti we hijack that option here to also disable the RFI flush. | ||
577 | @@ -900,6 +922,32 @@ void rfi_flush_enable(bool enable) | ||
578 | rfi_flush = enable; | ||
579 | } | ||
580 | |||
581 | +void entry_flush_enable(bool enable) | ||
582 | +{ | ||
583 | + if (enable) { | ||
584 | + do_entry_flush_fixups(enabled_flush_types); | ||
585 | + on_each_cpu(do_nothing, NULL, 1); | ||
586 | + } else { | ||
587 | + do_entry_flush_fixups(L1D_FLUSH_NONE); | ||
588 | + } | ||
589 | + | ||
590 | + entry_flush = enable; | ||
591 | +} | ||
592 | + | ||
593 | +void uaccess_flush_enable(bool enable) | ||
594 | +{ | ||
595 | + if (enable) { | ||
596 | + do_uaccess_flush_fixups(enabled_flush_types); | ||
597 | + static_branch_enable(&uaccess_flush_key); | ||
598 | + on_each_cpu(do_nothing, NULL, 1); | ||
599 | + } else { | ||
600 | + static_branch_disable(&uaccess_flush_key); | ||
601 | + do_uaccess_flush_fixups(L1D_FLUSH_NONE); | ||
602 | + } | ||
603 | + | ||
604 | + uaccess_flush = enable; | ||
605 | +} | ||
606 | + | ||
607 | static void __ref init_fallback_flush(void) | ||
608 | { | ||
609 | u64 l1d_size, limit; | ||
610 | @@ -958,10 +1006,28 @@ void setup_rfi_flush(enum l1d_flush_type types, bool enable) | ||
611 | |||
612 | enabled_flush_types = types; | ||
613 | |||
614 | - if (!no_rfi_flush && !cpu_mitigations_off()) | ||
615 | + if (!cpu_mitigations_off() && !no_rfi_flush) | ||
616 | rfi_flush_enable(enable); | ||
617 | } | ||
618 | |||
619 | +void setup_entry_flush(bool enable) | ||
620 | +{ | ||
621 | + if (cpu_mitigations_off()) | ||
622 | + return; | ||
623 | + | ||
624 | + if (!no_entry_flush) | ||
625 | + entry_flush_enable(enable); | ||
626 | +} | ||
627 | + | ||
628 | +void setup_uaccess_flush(bool enable) | ||
629 | +{ | ||
630 | + if (cpu_mitigations_off()) | ||
631 | + return; | ||
632 | + | ||
633 | + if (!no_uaccess_flush) | ||
634 | + uaccess_flush_enable(enable); | ||
635 | +} | ||
636 | + | ||
637 | #ifdef CONFIG_DEBUG_FS | ||
638 | static int rfi_flush_set(void *data, u64 val) | ||
639 | { | ||
640 | @@ -989,9 +1055,63 @@ static int rfi_flush_get(void *data, u64 *val) | ||
641 | |||
642 | DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n"); | ||
643 | |||
644 | +static int entry_flush_set(void *data, u64 val) | ||
645 | +{ | ||
646 | + bool enable; | ||
647 | + | ||
648 | + if (val == 1) | ||
649 | + enable = true; | ||
650 | + else if (val == 0) | ||
651 | + enable = false; | ||
652 | + else | ||
653 | + return -EINVAL; | ||
654 | + | ||
655 | + /* Only do anything if we're changing state */ | ||
656 | + if (enable != entry_flush) | ||
657 | + entry_flush_enable(enable); | ||
658 | + | ||
659 | + return 0; | ||
660 | +} | ||
661 | + | ||
662 | +static int entry_flush_get(void *data, u64 *val) | ||
663 | +{ | ||
664 | + *val = entry_flush ? 1 : 0; | ||
665 | + return 0; | ||
666 | +} | ||
667 | + | ||
668 | +DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n"); | ||
669 | + | ||
670 | +static int uaccess_flush_set(void *data, u64 val) | ||
671 | +{ | ||
672 | + bool enable; | ||
673 | + | ||
674 | + if (val == 1) | ||
675 | + enable = true; | ||
676 | + else if (val == 0) | ||
677 | + enable = false; | ||
678 | + else | ||
679 | + return -EINVAL; | ||
680 | + | ||
681 | + /* Only do anything if we're changing state */ | ||
682 | + if (enable != uaccess_flush) | ||
683 | + uaccess_flush_enable(enable); | ||
684 | + | ||
685 | + return 0; | ||
686 | +} | ||
687 | + | ||
688 | +static int uaccess_flush_get(void *data, u64 *val) | ||
689 | +{ | ||
690 | + *val = uaccess_flush ? 1 : 0; | ||
691 | + return 0; | ||
692 | +} | ||
693 | + | ||
694 | +DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n"); | ||
695 | + | ||
696 | static __init int rfi_flush_debugfs_init(void) | ||
697 | { | ||
698 | debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush); | ||
699 | + debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush); | ||
700 | + debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush); | ||
701 | return 0; | ||
702 | } | ||
703 | device_initcall(rfi_flush_debugfs_init); | ||
704 | diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S | ||
705 | index 060a1acd7c6d7..5229eeac8946d 100644 | ||
706 | --- a/arch/powerpc/kernel/vmlinux.lds.S | ||
707 | +++ b/arch/powerpc/kernel/vmlinux.lds.S | ||
708 | @@ -143,6 +143,20 @@ SECTIONS | ||
709 | __stop___stf_entry_barrier_fixup = .; | ||
710 | } | ||
711 | |||
712 | + . = ALIGN(8); | ||
713 | + __uaccess_flush_fixup : AT(ADDR(__uaccess_flush_fixup) - LOAD_OFFSET) { | ||
714 | + __start___uaccess_flush_fixup = .; | ||
715 | + *(__uaccess_flush_fixup) | ||
716 | + __stop___uaccess_flush_fixup = .; | ||
717 | + } | ||
718 | + | ||
719 | + . = ALIGN(8); | ||
720 | + __entry_flush_fixup : AT(ADDR(__entry_flush_fixup) - LOAD_OFFSET) { | ||
721 | + __start___entry_flush_fixup = .; | ||
722 | + *(__entry_flush_fixup) | ||
723 | + __stop___entry_flush_fixup = .; | ||
724 | + } | ||
725 | + | ||
726 | . = ALIGN(8); | ||
727 | __stf_exit_barrier_fixup : AT(ADDR(__stf_exit_barrier_fixup) - LOAD_OFFSET) { | ||
728 | __start___stf_exit_barrier_fixup = .; | ||
729 | diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c | ||
730 | index 4ba634b89ce53..e8b25f74454d6 100644 | ||
731 | --- a/arch/powerpc/lib/feature-fixups.c | ||
732 | +++ b/arch/powerpc/lib/feature-fixups.c | ||
733 | @@ -228,6 +228,110 @@ void do_stf_barrier_fixups(enum stf_barrier_type types) | ||
734 | do_stf_exit_barrier_fixups(types); | ||
735 | } | ||
736 | |||
737 | +void do_uaccess_flush_fixups(enum l1d_flush_type types) | ||
738 | +{ | ||
739 | + unsigned int instrs[4], *dest; | ||
740 | + long *start, *end; | ||
741 | + int i; | ||
742 | + | ||
743 | + start = PTRRELOC(&__start___uaccess_flush_fixup); | ||
744 | + end = PTRRELOC(&__stop___uaccess_flush_fixup); | ||
745 | + | ||
746 | + instrs[0] = 0x60000000; /* nop */ | ||
747 | + instrs[1] = 0x60000000; /* nop */ | ||
748 | + instrs[2] = 0x60000000; /* nop */ | ||
749 | + instrs[3] = 0x4e800020; /* blr */ | ||
750 | + | ||
751 | + i = 0; | ||
752 | + if (types == L1D_FLUSH_FALLBACK) { | ||
753 | + instrs[3] = 0x60000000; /* nop */ | ||
754 | + /* fallthrough to fallback flush */ | ||
755 | + } | ||
756 | + | ||
757 | + if (types & L1D_FLUSH_ORI) { | ||
758 | + instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */ | ||
759 | + instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/ | ||
760 | + } | ||
761 | + | ||
762 | + if (types & L1D_FLUSH_MTTRIG) | ||
763 | + instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ | ||
764 | + | ||
765 | + for (i = 0; start < end; start++, i++) { | ||
766 | + dest = (void *)start + *start; | ||
767 | + | ||
768 | + pr_devel("patching dest %lx\n", (unsigned long)dest); | ||
769 | + | ||
770 | + patch_instruction(dest, instrs[0]); | ||
771 | + | ||
772 | + patch_instruction((dest + 1), instrs[1]); | ||
773 | + patch_instruction((dest + 2), instrs[2]); | ||
774 | + patch_instruction((dest + 3), instrs[3]); | ||
775 | + } | ||
776 | + | ||
777 | + printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i, | ||
778 | + (types == L1D_FLUSH_NONE) ? "no" : | ||
779 | + (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : | ||
780 | + (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) | ||
781 | + ? "ori+mttrig type" | ||
782 | + : "ori type" : | ||
783 | + (types & L1D_FLUSH_MTTRIG) ? "mttrig type" | ||
784 | + : "unknown"); | ||
785 | +} | ||
786 | + | ||
787 | +void do_entry_flush_fixups(enum l1d_flush_type types) | ||
788 | +{ | ||
789 | + unsigned int instrs[3], *dest; | ||
790 | + long *start, *end; | ||
791 | + int i; | ||
792 | + | ||
793 | + start = PTRRELOC(&__start___entry_flush_fixup); | ||
794 | + end = PTRRELOC(&__stop___entry_flush_fixup); | ||
795 | + | ||
796 | + instrs[0] = 0x60000000; /* nop */ | ||
797 | + instrs[1] = 0x60000000; /* nop */ | ||
798 | + instrs[2] = 0x60000000; /* nop */ | ||
799 | + | ||
800 | + i = 0; | ||
801 | + if (types == L1D_FLUSH_FALLBACK) { | ||
802 | + instrs[i++] = 0x7d4802a6; /* mflr r10 */ | ||
803 | + instrs[i++] = 0x60000000; /* branch patched below */ | ||
804 | + instrs[i++] = 0x7d4803a6; /* mtlr r10 */ | ||
805 | + } | ||
806 | + | ||
807 | + if (types & L1D_FLUSH_ORI) { | ||
808 | + instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */ | ||
809 | + instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/ | ||
810 | + } | ||
811 | + | ||
812 | + if (types & L1D_FLUSH_MTTRIG) | ||
813 | + instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ | ||
814 | + | ||
815 | + for (i = 0; start < end; start++, i++) { | ||
816 | + dest = (void *)start + *start; | ||
817 | + | ||
818 | + pr_devel("patching dest %lx\n", (unsigned long)dest); | ||
819 | + | ||
820 | + patch_instruction(dest, instrs[0]); | ||
821 | + | ||
822 | + if (types == L1D_FLUSH_FALLBACK) | ||
823 | + patch_branch((dest + 1), (unsigned long)&entry_flush_fallback, | ||
824 | + BRANCH_SET_LINK); | ||
825 | + else | ||
826 | + patch_instruction((dest + 1), instrs[1]); | ||
827 | + | ||
828 | + patch_instruction((dest + 2), instrs[2]); | ||
829 | + } | ||
830 | + | ||
831 | + printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i, | ||
832 | + (types == L1D_FLUSH_NONE) ? "no" : | ||
833 | + (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : | ||
834 | + (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) | ||
835 | + ? "ori+mttrig type" | ||
836 | + : "ori type" : | ||
837 | + (types & L1D_FLUSH_MTTRIG) ? "mttrig type" | ||
838 | + : "unknown"); | ||
839 | +} | ||
840 | + | ||
841 | void do_rfi_flush_fixups(enum l1d_flush_type types) | ||
842 | { | ||
843 | unsigned int instrs[3], *dest; | ||
844 | diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c | ||
845 | index 83498604d322b..3a9f79d18f6b0 100644 | ||
846 | --- a/arch/powerpc/platforms/powernv/setup.c | ||
847 | +++ b/arch/powerpc/platforms/powernv/setup.c | ||
848 | @@ -122,12 +122,29 @@ static void pnv_setup_rfi_flush(void) | ||
849 | type = L1D_FLUSH_ORI; | ||
850 | } | ||
851 | |||
852 | + /* | ||
853 | + * If we are non-Power9 bare metal, we don't need to flush on kernel | ||
854 | + * entry or after user access: they fix a P9 specific vulnerability. | ||
855 | + */ | ||
856 | + if (!pvr_version_is(PVR_POWER9)) { | ||
857 | + security_ftr_clear(SEC_FTR_L1D_FLUSH_ENTRY); | ||
858 | + security_ftr_clear(SEC_FTR_L1D_FLUSH_UACCESS); | ||
859 | + } | ||
860 | + | ||
861 | enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && \ | ||
862 | (security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) || \ | ||
863 | security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV)); | ||
864 | |||
865 | setup_rfi_flush(type, enable); | ||
866 | setup_count_cache_flush(); | ||
867 | + | ||
868 | + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && | ||
869 | + security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY); | ||
870 | + setup_entry_flush(enable); | ||
871 | + | ||
872 | + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && | ||
873 | + security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS); | ||
874 | + setup_uaccess_flush(enable); | ||
875 | } | ||
876 | |||
877 | static void __init pnv_setup_arch(void) | ||
878 | diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c | ||
879 | index 0c8421dd01ab5..ce71235c8b81f 100644 | ||
880 | --- a/arch/powerpc/platforms/pseries/setup.c | ||
881 | +++ b/arch/powerpc/platforms/pseries/setup.c | ||
882 | @@ -561,6 +561,14 @@ void pseries_setup_rfi_flush(void) | ||
883 | |||
884 | setup_rfi_flush(types, enable); | ||
885 | setup_count_cache_flush(); | ||
886 | + | ||
887 | + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && | ||
888 | + security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY); | ||
889 | + setup_entry_flush(enable); | ||
890 | + | ||
891 | + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && | ||
892 | + security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS); | ||
893 | + setup_uaccess_flush(enable); | ||
894 | } | ||
895 | |||
896 | #ifdef CONFIG_PCI_IOV | ||
897 | diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c | ||
898 | index 484c32b7f79ff..39265b55929d2 100644 | ||
899 | --- a/arch/x86/kvm/emulate.c | ||
900 | +++ b/arch/x86/kvm/emulate.c | ||
901 | @@ -4050,6 +4050,12 @@ static int em_clflush(struct x86_emulate_ctxt *ctxt) | ||
902 | return X86EMUL_CONTINUE; | ||
903 | } | ||
904 | |||
905 | +static int em_clflushopt(struct x86_emulate_ctxt *ctxt) | ||
906 | +{ | ||
907 | + /* emulating clflushopt regardless of cpuid */ | ||
908 | + return X86EMUL_CONTINUE; | ||
909 | +} | ||
910 | + | ||
911 | static int em_movsxd(struct x86_emulate_ctxt *ctxt) | ||
912 | { | ||
913 | ctxt->dst.val = (s32) ctxt->src.val; | ||
914 | @@ -4592,7 +4598,7 @@ static const struct opcode group11[] = { | ||
915 | }; | ||
916 | |||
917 | static const struct gprefix pfx_0f_ae_7 = { | ||
918 | - I(SrcMem | ByteOp, em_clflush), N, N, N, | ||
919 | + I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N, | ||
920 | }; | ||
921 | |||
922 | static const struct group_dual group15 = { { | ||
923 | diff --git a/drivers/acpi/evged.c b/drivers/acpi/evged.c | ||
924 | index ccd900690b6f5..9df6991635c22 100644 | ||
925 | --- a/drivers/acpi/evged.c | ||
926 | +++ b/drivers/acpi/evged.c | ||
927 | @@ -101,7 +101,7 @@ static acpi_status acpi_ged_request_interrupt(struct acpi_resource *ares, | ||
928 | |||
929 | switch (gsi) { | ||
930 | case 0 ... 255: | ||
931 | - sprintf(ev_name, "_%c%02hhX", | ||
932 | + sprintf(ev_name, "_%c%02X", | ||
933 | trigger == ACPI_EDGE_SENSITIVE ? 'E' : 'L', gsi); | ||
934 | |||
935 | if (ACPI_SUCCESS(acpi_get_handle(handle, ev_name, &evt_handle))) | ||
936 | diff --git a/drivers/input/keyboard/sunkbd.c b/drivers/input/keyboard/sunkbd.c | ||
937 | index 27126e621eb60..d450f11b98a70 100644 | ||
938 | --- a/drivers/input/keyboard/sunkbd.c | ||
939 | +++ b/drivers/input/keyboard/sunkbd.c | ||
940 | @@ -99,7 +99,8 @@ static irqreturn_t sunkbd_interrupt(struct serio *serio, | ||
941 | switch (data) { | ||
942 | |||
943 | case SUNKBD_RET_RESET: | ||
944 | - schedule_work(&sunkbd->tq); | ||
945 | + if (sunkbd->enabled) | ||
946 | + schedule_work(&sunkbd->tq); | ||
947 | sunkbd->reset = -1; | ||
948 | break; | ||
949 | |||
950 | @@ -200,16 +201,12 @@ static int sunkbd_initialize(struct sunkbd *sunkbd) | ||
951 | } | ||
952 | |||
953 | /* | ||
954 | - * sunkbd_reinit() sets leds and beeps to a state the computer remembers they | ||
955 | - * were in. | ||
956 | + * sunkbd_set_leds_beeps() sets leds and beeps to a state the computer remembers | ||
957 | + * they were in. | ||
958 | */ | ||
959 | |||
960 | -static void sunkbd_reinit(struct work_struct *work) | ||
961 | +static void sunkbd_set_leds_beeps(struct sunkbd *sunkbd) | ||
962 | { | ||
963 | - struct sunkbd *sunkbd = container_of(work, struct sunkbd, tq); | ||
964 | - | ||
965 | - wait_event_interruptible_timeout(sunkbd->wait, sunkbd->reset >= 0, HZ); | ||
966 | - | ||
967 | serio_write(sunkbd->serio, SUNKBD_CMD_SETLED); | ||
968 | serio_write(sunkbd->serio, | ||
969 | (!!test_bit(LED_CAPSL, sunkbd->dev->led) << 3) | | ||
970 | @@ -222,11 +219,39 @@ static void sunkbd_reinit(struct work_struct *work) | ||
971 | SUNKBD_CMD_BELLOFF - !!test_bit(SND_BELL, sunkbd->dev->snd)); | ||
972 | } | ||
973 | |||
974 | + | ||
975 | +/* | ||
976 | + * sunkbd_reinit() wait for the keyboard reset to complete and restores state | ||
977 | + * of leds and beeps. | ||
978 | + */ | ||
979 | + | ||
980 | +static void sunkbd_reinit(struct work_struct *work) | ||
981 | +{ | ||
982 | + struct sunkbd *sunkbd = container_of(work, struct sunkbd, tq); | ||
983 | + | ||
984 | + /* | ||
985 | + * It is OK that we check sunkbd->enabled without pausing serio, | ||
986 | + * as we only want to catch true->false transition that will | ||
987 | + * happen once and we will be woken up for it. | ||
988 | + */ | ||
989 | + wait_event_interruptible_timeout(sunkbd->wait, | ||
990 | + sunkbd->reset >= 0 || !sunkbd->enabled, | ||
991 | + HZ); | ||
992 | + | ||
993 | + if (sunkbd->reset >= 0 && sunkbd->enabled) | ||
994 | + sunkbd_set_leds_beeps(sunkbd); | ||
995 | +} | ||
996 | + | ||
997 | static void sunkbd_enable(struct sunkbd *sunkbd, bool enable) | ||
998 | { | ||
999 | serio_pause_rx(sunkbd->serio); | ||
1000 | sunkbd->enabled = enable; | ||
1001 | serio_continue_rx(sunkbd->serio); | ||
1002 | + | ||
1003 | + if (!enable) { | ||
1004 | + wake_up_interruptible(&sunkbd->wait); | ||
1005 | + cancel_work_sync(&sunkbd->tq); | ||
1006 | + } | ||
1007 | } | ||
1008 | |||
1009 | /* | ||
1010 | diff --git a/drivers/net/ethernet/lantiq_xrx200.c b/drivers/net/ethernet/lantiq_xrx200.c | ||
1011 | index 96948276b2bc3..4e44a39267eb3 100644 | ||
1012 | --- a/drivers/net/ethernet/lantiq_xrx200.c | ||
1013 | +++ b/drivers/net/ethernet/lantiq_xrx200.c | ||
1014 | @@ -245,6 +245,7 @@ static int xrx200_tx_housekeeping(struct napi_struct *napi, int budget) | ||
1015 | int pkts = 0; | ||
1016 | int bytes = 0; | ||
1017 | |||
1018 | + netif_tx_lock(net_dev); | ||
1019 | while (pkts < budget) { | ||
1020 | struct ltq_dma_desc *desc = &ch->dma.desc_base[ch->tx_free]; | ||
1021 | |||
1022 | @@ -268,6 +269,7 @@ static int xrx200_tx_housekeeping(struct napi_struct *napi, int budget) | ||
1023 | net_dev->stats.tx_bytes += bytes; | ||
1024 | netdev_completed_queue(ch->priv->net_dev, pkts, bytes); | ||
1025 | |||
1026 | + netif_tx_unlock(net_dev); | ||
1027 | if (netif_queue_stopped(net_dev)) | ||
1028 | netif_wake_queue(net_dev); | ||
1029 | |||
1030 | diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c | ||
1031 | index 7089ffcc4e512..76547d35cd0e1 100644 | ||
1032 | --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c | ||
1033 | +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c | ||
1034 | @@ -853,11 +853,21 @@ static void cb_timeout_handler(struct work_struct *work) | ||
1035 | struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, | ||
1036 | cmd); | ||
1037 | |||
1038 | + mlx5_cmd_eq_recover(dev); | ||
1039 | + | ||
1040 | + /* Maybe got handled by eq recover ? */ | ||
1041 | + if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) { | ||
1042 | + mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx, | ||
1043 | + mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); | ||
1044 | + goto out; /* phew, already handled */ | ||
1045 | + } | ||
1046 | + | ||
1047 | ent->ret = -ETIMEDOUT; | ||
1048 | - mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", | ||
1049 | - mlx5_command_str(msg_to_opcode(ent->in)), | ||
1050 | - msg_to_opcode(ent->in)); | ||
1051 | + mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n", | ||
1052 | + ent->idx, mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); | ||
1053 | mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); | ||
1054 | + | ||
1055 | +out: | ||
1056 | cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */ | ||
1057 | } | ||
1058 | |||
1059 | @@ -865,6 +875,33 @@ static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg); | ||
1060 | static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, | ||
1061 | struct mlx5_cmd_msg *msg); | ||
1062 | |||
1063 | +static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode) | ||
1064 | +{ | ||
1065 | + if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL) | ||
1066 | + return true; | ||
1067 | + | ||
1068 | + return cmd->allowed_opcode == opcode; | ||
1069 | +} | ||
1070 | + | ||
1071 | +static int cmd_alloc_index_retry(struct mlx5_cmd *cmd) | ||
1072 | +{ | ||
1073 | + unsigned long alloc_end = jiffies + msecs_to_jiffies(1000); | ||
1074 | + int idx; | ||
1075 | + | ||
1076 | +retry: | ||
1077 | + idx = cmd_alloc_index(cmd); | ||
1078 | + if (idx < 0 && time_before(jiffies, alloc_end)) { | ||
1079 | + /* Index allocation can fail on heavy load of commands. This is a temporary | ||
1080 | + * situation as the current command already holds the semaphore, meaning that | ||
1081 | + * another command completion is being handled and it is expected to release | ||
1082 | + * the entry index soon. | ||
1083 | + */ | ||
1084 | + cpu_relax(); | ||
1085 | + goto retry; | ||
1086 | + } | ||
1087 | + return idx; | ||
1088 | +} | ||
1089 | + | ||
1090 | static void cmd_work_handler(struct work_struct *work) | ||
1091 | { | ||
1092 | struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work); | ||
1093 | @@ -882,7 +919,7 @@ static void cmd_work_handler(struct work_struct *work) | ||
1094 | sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem; | ||
1095 | down(sem); | ||
1096 | if (!ent->page_queue) { | ||
1097 | - alloc_ret = cmd_alloc_index(cmd); | ||
1098 | + alloc_ret = cmd_alloc_index_retry(cmd); | ||
1099 | if (alloc_ret < 0) { | ||
1100 | mlx5_core_err(dev, "failed to allocate command entry\n"); | ||
1101 | if (ent->callback) { | ||
1102 | @@ -931,7 +968,8 @@ static void cmd_work_handler(struct work_struct *work) | ||
1103 | |||
1104 | /* Skip sending command to fw if internal error */ | ||
1105 | if (pci_channel_offline(dev->pdev) || | ||
1106 | - dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { | ||
1107 | + dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR || | ||
1108 | + !opcode_allowed(&dev->cmd, ent->op)) { | ||
1109 | u8 status = 0; | ||
1110 | u32 drv_synd; | ||
1111 | |||
1112 | @@ -987,6 +1025,35 @@ static const char *deliv_status_to_str(u8 status) | ||
1113 | } | ||
1114 | } | ||
1115 | |||
1116 | +enum { | ||
1117 | + MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000, | ||
1118 | +}; | ||
1119 | + | ||
1120 | +static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev, | ||
1121 | + struct mlx5_cmd_work_ent *ent) | ||
1122 | +{ | ||
1123 | + unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC); | ||
1124 | + | ||
1125 | + mlx5_cmd_eq_recover(dev); | ||
1126 | + | ||
1127 | + /* Re-wait on the ent->done after executing the recovery flow. If the | ||
1128 | + * recovery flow (or any other recovery flow running simultaneously) | ||
1129 | + * has recovered an EQE, it should cause the entry to be completed by | ||
1130 | + * the command interface. | ||
1131 | + */ | ||
1132 | + if (wait_for_completion_timeout(&ent->done, timeout)) { | ||
1133 | + mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx, | ||
1134 | + mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); | ||
1135 | + return; | ||
1136 | + } | ||
1137 | + | ||
1138 | + mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx, | ||
1139 | + mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); | ||
1140 | + | ||
1141 | + ent->ret = -ETIMEDOUT; | ||
1142 | + mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); | ||
1143 | +} | ||
1144 | + | ||
1145 | static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) | ||
1146 | { | ||
1147 | unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC); | ||
1148 | @@ -998,12 +1065,10 @@ static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) | ||
1149 | ent->ret = -ECANCELED; | ||
1150 | goto out_err; | ||
1151 | } | ||
1152 | - if (cmd->mode == CMD_MODE_POLLING || ent->polling) { | ||
1153 | + if (cmd->mode == CMD_MODE_POLLING || ent->polling) | ||
1154 | wait_for_completion(&ent->done); | ||
1155 | - } else if (!wait_for_completion_timeout(&ent->done, timeout)) { | ||
1156 | - ent->ret = -ETIMEDOUT; | ||
1157 | - mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); | ||
1158 | - } | ||
1159 | + else if (!wait_for_completion_timeout(&ent->done, timeout)) | ||
1160 | + wait_func_handle_exec_timeout(dev, ent); | ||
1161 | |||
1162 | out_err: | ||
1163 | err = ent->ret; | ||
1164 | @@ -1422,6 +1487,22 @@ static void create_debugfs_files(struct mlx5_core_dev *dev) | ||
1165 | mlx5_cmdif_debugfs_init(dev); | ||
1166 | } | ||
1167 | |||
1168 | +void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode) | ||
1169 | +{ | ||
1170 | + struct mlx5_cmd *cmd = &dev->cmd; | ||
1171 | + int i; | ||
1172 | + | ||
1173 | + for (i = 0; i < cmd->max_reg_cmds; i++) | ||
1174 | + down(&cmd->sem); | ||
1175 | + down(&cmd->pages_sem); | ||
1176 | + | ||
1177 | + cmd->allowed_opcode = opcode; | ||
1178 | + | ||
1179 | + up(&cmd->pages_sem); | ||
1180 | + for (i = 0; i < cmd->max_reg_cmds; i++) | ||
1181 | + up(&cmd->sem); | ||
1182 | +} | ||
1183 | + | ||
1184 | static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) | ||
1185 | { | ||
1186 | struct mlx5_cmd *cmd = &dev->cmd; | ||
1187 | @@ -1714,12 +1795,13 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, | ||
1188 | int err; | ||
1189 | u8 status = 0; | ||
1190 | u32 drv_synd; | ||
1191 | + u16 opcode; | ||
1192 | u8 token; | ||
1193 | |||
1194 | + opcode = MLX5_GET(mbox_in, in, opcode); | ||
1195 | if (pci_channel_offline(dev->pdev) || | ||
1196 | - dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { | ||
1197 | - u16 opcode = MLX5_GET(mbox_in, in, opcode); | ||
1198 | - | ||
1199 | + dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR || | ||
1200 | + !opcode_allowed(&dev->cmd, opcode)) { | ||
1201 | err = mlx5_internal_err_ret_value(dev, opcode, &drv_synd, &status); | ||
1202 | MLX5_SET(mbox_out, out, status, status); | ||
1203 | MLX5_SET(mbox_out, out, syndrome, drv_synd); | ||
1204 | @@ -2021,6 +2103,7 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) | ||
1205 | mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); | ||
1206 | |||
1207 | cmd->mode = CMD_MODE_POLLING; | ||
1208 | + cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL; | ||
1209 | |||
1210 | create_msg_cache(dev); | ||
1211 | |||
1212 | diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c | ||
1213 | index 580c71cb9dfaa..0a20938b4aadb 100644 | ||
1214 | --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c | ||
1215 | +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c | ||
1216 | @@ -190,6 +190,29 @@ u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq) | ||
1217 | return count_eqe; | ||
1218 | } | ||
1219 | |||
1220 | +static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, unsigned long *flags) | ||
1221 | + __acquires(&eq->lock) | ||
1222 | +{ | ||
1223 | + if (in_irq()) | ||
1224 | + spin_lock(&eq->lock); | ||
1225 | + else | ||
1226 | + spin_lock_irqsave(&eq->lock, *flags); | ||
1227 | +} | ||
1228 | + | ||
1229 | +static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, unsigned long *flags) | ||
1230 | + __releases(&eq->lock) | ||
1231 | +{ | ||
1232 | + if (in_irq()) | ||
1233 | + spin_unlock(&eq->lock); | ||
1234 | + else | ||
1235 | + spin_unlock_irqrestore(&eq->lock, *flags); | ||
1236 | +} | ||
1237 | + | ||
1238 | +enum async_eq_nb_action { | ||
1239 | + ASYNC_EQ_IRQ_HANDLER = 0, | ||
1240 | + ASYNC_EQ_RECOVER = 1, | ||
1241 | +}; | ||
1242 | + | ||
1243 | static int mlx5_eq_async_int(struct notifier_block *nb, | ||
1244 | unsigned long action, void *data) | ||
1245 | { | ||
1246 | @@ -199,11 +222,14 @@ static int mlx5_eq_async_int(struct notifier_block *nb, | ||
1247 | struct mlx5_eq_table *eqt; | ||
1248 | struct mlx5_core_dev *dev; | ||
1249 | struct mlx5_eqe *eqe; | ||
1250 | + unsigned long flags; | ||
1251 | int num_eqes = 0; | ||
1252 | |||
1253 | dev = eq->dev; | ||
1254 | eqt = dev->priv.eq_table; | ||
1255 | |||
1256 | + mlx5_eq_async_int_lock(eq_async, &flags); | ||
1257 | + | ||
1258 | eqe = next_eqe_sw(eq); | ||
1259 | if (!eqe) | ||
1260 | goto out; | ||
1261 | @@ -224,8 +250,19 @@ static int mlx5_eq_async_int(struct notifier_block *nb, | ||
1262 | |||
1263 | out: | ||
1264 | eq_update_ci(eq, 1); | ||
1265 | + mlx5_eq_async_int_unlock(eq_async, &flags); | ||
1266 | |||
1267 | - return 0; | ||
1268 | + return unlikely(action == ASYNC_EQ_RECOVER) ? num_eqes : 0; | ||
1269 | +} | ||
1270 | + | ||
1271 | +void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev) | ||
1272 | +{ | ||
1273 | + struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq; | ||
1274 | + int eqes; | ||
1275 | + | ||
1276 | + eqes = mlx5_eq_async_int(&eq->irq_nb, ASYNC_EQ_RECOVER, NULL); | ||
1277 | + if (eqes) | ||
1278 | + mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes); | ||
1279 | } | ||
1280 | |||
1281 | static void init_eq_buf(struct mlx5_eq *eq) | ||
1282 | @@ -563,6 +600,40 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) | ||
1283 | gather_user_async_events(dev, mask); | ||
1284 | } | ||
1285 | |||
1286 | +static int | ||
1287 | +setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq, | ||
1288 | + struct mlx5_eq_param *param, const char *name) | ||
1289 | +{ | ||
1290 | + int err; | ||
1291 | + | ||
1292 | + eq->irq_nb.notifier_call = mlx5_eq_async_int; | ||
1293 | + spin_lock_init(&eq->lock); | ||
1294 | + | ||
1295 | + err = create_async_eq(dev, &eq->core, param); | ||
1296 | + if (err) { | ||
1297 | + mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err); | ||
1298 | + return err; | ||
1299 | + } | ||
1300 | + err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb); | ||
1301 | + if (err) { | ||
1302 | + mlx5_core_warn(dev, "failed to enable %s EQ %d\n", name, err); | ||
1303 | + destroy_async_eq(dev, &eq->core); | ||
1304 | + } | ||
1305 | + return err; | ||
1306 | +} | ||
1307 | + | ||
1308 | +static void cleanup_async_eq(struct mlx5_core_dev *dev, | ||
1309 | + struct mlx5_eq_async *eq, const char *name) | ||
1310 | +{ | ||
1311 | + int err; | ||
1312 | + | ||
1313 | + mlx5_eq_disable(dev, &eq->core, &eq->irq_nb); | ||
1314 | + err = destroy_async_eq(dev, &eq->core); | ||
1315 | + if (err) | ||
1316 | + mlx5_core_err(dev, "failed to destroy %s eq, err(%d)\n", | ||
1317 | + name, err); | ||
1318 | +} | ||
1319 | + | ||
1320 | static int create_async_eqs(struct mlx5_core_dev *dev) | ||
1321 | { | ||
1322 | struct mlx5_eq_table *table = dev->priv.eq_table; | ||
1323 | @@ -572,77 +643,48 @@ static int create_async_eqs(struct mlx5_core_dev *dev) | ||
1324 | MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR); | ||
1325 | mlx5_eq_notifier_register(dev, &table->cq_err_nb); | ||
1326 | |||
1327 | - table->cmd_eq.irq_nb.notifier_call = mlx5_eq_async_int; | ||
1328 | param = (struct mlx5_eq_param) { | ||
1329 | .irq_index = 0, | ||
1330 | .nent = MLX5_NUM_CMD_EQE, | ||
1331 | + .mask[0] = 1ull << MLX5_EVENT_TYPE_CMD, | ||
1332 | }; | ||
1333 | - | ||
1334 | - param.mask[0] = 1ull << MLX5_EVENT_TYPE_CMD; | ||
1335 | - err = create_async_eq(dev, &table->cmd_eq.core, ¶m); | ||
1336 | - if (err) { | ||
1337 | - mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err); | ||
1338 | - goto err0; | ||
1339 | - } | ||
1340 | - err = mlx5_eq_enable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb); | ||
1341 | - if (err) { | ||
1342 | - mlx5_core_warn(dev, "failed to enable cmd EQ %d\n", err); | ||
1343 | + mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_CREATE_EQ); | ||
1344 | + err = setup_async_eq(dev, &table->cmd_eq, ¶m, "cmd"); | ||
1345 | + if (err) | ||
1346 | goto err1; | ||
1347 | - } | ||
1348 | + | ||
1349 | mlx5_cmd_use_events(dev); | ||
1350 | + mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); | ||
1351 | |||
1352 | - table->async_eq.irq_nb.notifier_call = mlx5_eq_async_int; | ||
1353 | param = (struct mlx5_eq_param) { | ||
1354 | .irq_index = 0, | ||
1355 | .nent = MLX5_NUM_ASYNC_EQE, | ||
1356 | }; | ||
1357 | |||
1358 | gather_async_events_mask(dev, param.mask); | ||
1359 | - err = create_async_eq(dev, &table->async_eq.core, ¶m); | ||
1360 | - if (err) { | ||
1361 | - mlx5_core_warn(dev, "failed to create async EQ %d\n", err); | ||
1362 | + err = setup_async_eq(dev, &table->async_eq, ¶m, "async"); | ||
1363 | + if (err) | ||
1364 | goto err2; | ||
1365 | - } | ||
1366 | - err = mlx5_eq_enable(dev, &table->async_eq.core, | ||
1367 | - &table->async_eq.irq_nb); | ||
1368 | - if (err) { | ||
1369 | - mlx5_core_warn(dev, "failed to enable async EQ %d\n", err); | ||
1370 | - goto err3; | ||
1371 | - } | ||
1372 | |||
1373 | - table->pages_eq.irq_nb.notifier_call = mlx5_eq_async_int; | ||
1374 | param = (struct mlx5_eq_param) { | ||
1375 | .irq_index = 0, | ||
1376 | .nent = /* TODO: sriov max_vf + */ 1, | ||
1377 | + .mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST, | ||
1378 | }; | ||
1379 | |||
1380 | - param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST; | ||
1381 | - err = create_async_eq(dev, &table->pages_eq.core, ¶m); | ||
1382 | - if (err) { | ||
1383 | - mlx5_core_warn(dev, "failed to create pages EQ %d\n", err); | ||
1384 | - goto err4; | ||
1385 | - } | ||
1386 | - err = mlx5_eq_enable(dev, &table->pages_eq.core, | ||
1387 | - &table->pages_eq.irq_nb); | ||
1388 | - if (err) { | ||
1389 | - mlx5_core_warn(dev, "failed to enable pages EQ %d\n", err); | ||
1390 | - goto err5; | ||
1391 | - } | ||
1392 | + err = setup_async_eq(dev, &table->pages_eq, ¶m, "pages"); | ||
1393 | + if (err) | ||
1394 | + goto err3; | ||
1395 | |||
1396 | - return err; | ||
1397 | + return 0; | ||
1398 | |||
1399 | -err5: | ||
1400 | - destroy_async_eq(dev, &table->pages_eq.core); | ||
1401 | -err4: | ||
1402 | - mlx5_eq_disable(dev, &table->async_eq.core, &table->async_eq.irq_nb); | ||
1403 | err3: | ||
1404 | - destroy_async_eq(dev, &table->async_eq.core); | ||
1405 | + cleanup_async_eq(dev, &table->async_eq, "async"); | ||
1406 | err2: | ||
1407 | mlx5_cmd_use_polling(dev); | ||
1408 | - mlx5_eq_disable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb); | ||
1409 | + cleanup_async_eq(dev, &table->cmd_eq, "cmd"); | ||
1410 | err1: | ||
1411 | - destroy_async_eq(dev, &table->cmd_eq.core); | ||
1412 | -err0: | ||
1413 | + mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); | ||
1414 | mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); | ||
1415 | return err; | ||
1416 | } | ||
1417 | @@ -650,28 +692,11 @@ err0: | ||
1418 | static void destroy_async_eqs(struct mlx5_core_dev *dev) | ||
1419 | { | ||
1420 | struct mlx5_eq_table *table = dev->priv.eq_table; | ||
1421 | - int err; | ||
1422 | - | ||
1423 | - mlx5_eq_disable(dev, &table->pages_eq.core, &table->pages_eq.irq_nb); | ||
1424 | - err = destroy_async_eq(dev, &table->pages_eq.core); | ||
1425 | - if (err) | ||
1426 | - mlx5_core_err(dev, "failed to destroy pages eq, err(%d)\n", | ||
1427 | - err); | ||
1428 | - | ||
1429 | - mlx5_eq_disable(dev, &table->async_eq.core, &table->async_eq.irq_nb); | ||
1430 | - err = destroy_async_eq(dev, &table->async_eq.core); | ||
1431 | - if (err) | ||
1432 | - mlx5_core_err(dev, "failed to destroy async eq, err(%d)\n", | ||
1433 | - err); | ||
1434 | |||
1435 | + cleanup_async_eq(dev, &table->pages_eq, "pages"); | ||
1436 | + cleanup_async_eq(dev, &table->async_eq, "async"); | ||
1437 | mlx5_cmd_use_polling(dev); | ||
1438 | - | ||
1439 | - mlx5_eq_disable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb); | ||
1440 | - err = destroy_async_eq(dev, &table->cmd_eq.core); | ||
1441 | - if (err) | ||
1442 | - mlx5_core_err(dev, "failed to destroy command eq, err(%d)\n", | ||
1443 | - err); | ||
1444 | - | ||
1445 | + cleanup_async_eq(dev, &table->cmd_eq, "cmd"); | ||
1446 | mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); | ||
1447 | } | ||
1448 | |||
1449 | diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h | ||
1450 | index 4be4d2d362189..9aaf0eab7c2e1 100644 | ||
1451 | --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h | ||
1452 | +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h | ||
1453 | @@ -38,6 +38,7 @@ struct mlx5_eq { | ||
1454 | struct mlx5_eq_async { | ||
1455 | struct mlx5_eq core; | ||
1456 | struct notifier_block irq_nb; | ||
1457 | + spinlock_t lock; /* To avoid irq EQ handle races with resiliency flows */ | ||
1458 | }; | ||
1459 | |||
1460 | struct mlx5_eq_comp { | ||
1461 | @@ -82,6 +83,7 @@ void mlx5_cq_tasklet_cb(unsigned long data); | ||
1462 | struct cpumask *mlx5_eq_comp_cpumask(struct mlx5_core_dev *dev, int ix); | ||
1463 | |||
1464 | u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq); | ||
1465 | +void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev); | ||
1466 | void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev); | ||
1467 | void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev); | ||
1468 | |||
1469 | diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h | ||
1470 | index 6b4f86dfca382..2b65ffb3bd76e 100644 | ||
1471 | --- a/include/linux/mlx5/driver.h | ||
1472 | +++ b/include/linux/mlx5/driver.h | ||
1473 | @@ -299,6 +299,7 @@ struct mlx5_cmd { | ||
1474 | struct semaphore sem; | ||
1475 | struct semaphore pages_sem; | ||
1476 | int mode; | ||
1477 | + u16 allowed_opcode; | ||
1478 | struct mlx5_cmd_work_ent *ent_arr[MLX5_MAX_COMMANDS]; | ||
1479 | struct dma_pool *pool; | ||
1480 | struct mlx5_cmd_debug dbg; | ||
1481 | @@ -890,10 +891,15 @@ mlx5_frag_buf_get_idx_last_contig_stride(struct mlx5_frag_buf_ctrl *fbc, u32 ix) | ||
1482 | return min_t(u32, last_frag_stride_idx - fbc->strides_offset, fbc->sz_m1); | ||
1483 | } | ||
1484 | |||
1485 | +enum { | ||
1486 | + CMD_ALLOWED_OPCODE_ALL, | ||
1487 | +}; | ||
1488 | + | ||
1489 | int mlx5_cmd_init(struct mlx5_core_dev *dev); | ||
1490 | void mlx5_cmd_cleanup(struct mlx5_core_dev *dev); | ||
1491 | void mlx5_cmd_use_events(struct mlx5_core_dev *dev); | ||
1492 | void mlx5_cmd_use_polling(struct mlx5_core_dev *dev); | ||
1493 | +void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode); | ||
1494 | |||
1495 | struct mlx5_async_ctx { | ||
1496 | struct mlx5_core_dev *dev; | ||
1497 | diff --git a/net/can/proc.c b/net/can/proc.c | ||
1498 | index e6881bfc3ed11..077af42c26ba5 100644 | ||
1499 | --- a/net/can/proc.c | ||
1500 | +++ b/net/can/proc.c | ||
1501 | @@ -471,6 +471,9 @@ void can_init_proc(struct net *net) | ||
1502 | */ | ||
1503 | void can_remove_proc(struct net *net) | ||
1504 | { | ||
1505 | + if (!net->can.proc_dir) | ||
1506 | + return; | ||
1507 | + | ||
1508 | if (net->can.pde_version) | ||
1509 | remove_proc_entry(CAN_PROC_VERSION, net->can.proc_dir); | ||
1510 | |||
1511 | @@ -498,6 +501,5 @@ void can_remove_proc(struct net *net) | ||
1512 | if (net->can.pde_rcvlist_sff) | ||
1513 | remove_proc_entry(CAN_PROC_RCVLIST_SFF, net->can.proc_dir); | ||
1514 | |||
1515 | - if (net->can.proc_dir) | ||
1516 | - remove_proc_entry("can", net->proc_net); | ||
1517 | + remove_proc_entry("can", net->proc_net); | ||
1518 | } | ||
1519 | diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c | ||
1520 | index 4f14d8a06915a..38bb6d512b36d 100644 | ||
1521 | --- a/net/mac80211/sta_info.c | ||
1522 | +++ b/net/mac80211/sta_info.c | ||
1523 | @@ -244,6 +244,24 @@ struct sta_info *sta_info_get_by_idx(struct ieee80211_sub_if_data *sdata, | ||
1524 | */ | ||
1525 | void sta_info_free(struct ieee80211_local *local, struct sta_info *sta) | ||
1526 | { | ||
1527 | + /* | ||
1528 | + * If we had used sta_info_pre_move_state() then we might not | ||
1529 | + * have gone through the state transitions down again, so do | ||
1530 | + * it here now (and warn if it's inserted). | ||
1531 | + * | ||
1532 | + * This will clear state such as fast TX/RX that may have been | ||
1533 | + * allocated during state transitions. | ||
1534 | + */ | ||
1535 | + while (sta->sta_state > IEEE80211_STA_NONE) { | ||
1536 | + int ret; | ||
1537 | + | ||
1538 | + WARN_ON_ONCE(test_sta_flag(sta, WLAN_STA_INSERTED)); | ||
1539 | + | ||
1540 | + ret = sta_info_move_state(sta, sta->sta_state - 1); | ||
1541 | + if (WARN_ONCE(ret, "sta_info_move_state() returned %d\n", ret)) | ||
1542 | + break; | ||
1543 | + } | ||
1544 | + | ||
1545 | if (sta->rate_ctrl) | ||
1546 | rate_control_free_sta(sta); | ||
1547 | |||
1548 | diff --git a/tools/testing/selftests/powerpc/security/.gitignore b/tools/testing/selftests/powerpc/security/.gitignore | ||
1549 | index 0b969fba3beb2..b8afb4f2481e4 100644 | ||
1550 | --- a/tools/testing/selftests/powerpc/security/.gitignore | ||
1551 | +++ b/tools/testing/selftests/powerpc/security/.gitignore | ||
1552 | @@ -1 +1,2 @@ | ||
1553 | rfi_flush | ||
1554 | +entry_flush | ||
1555 | diff --git a/tools/testing/selftests/powerpc/security/Makefile b/tools/testing/selftests/powerpc/security/Makefile | ||
1556 | index 85861c46b4457..e550a287768fc 100644 | ||
1557 | --- a/tools/testing/selftests/powerpc/security/Makefile | ||
1558 | +++ b/tools/testing/selftests/powerpc/security/Makefile | ||
1559 | @@ -1,6 +1,6 @@ | ||
1560 | # SPDX-License-Identifier: GPL-2.0+ | ||
1561 | |||
1562 | -TEST_GEN_PROGS := rfi_flush | ||
1563 | +TEST_GEN_PROGS := rfi_flush entry_flush | ||
1564 | top_srcdir = ../../../../.. | ||
1565 | |||
1566 | CFLAGS += -I../../../../../usr/include | ||
1567 | diff --git a/tools/testing/selftests/powerpc/security/entry_flush.c b/tools/testing/selftests/powerpc/security/entry_flush.c | ||
1568 | new file mode 100644 | ||
1569 | index 0000000000000..e8d24f9a5d3e5 | ||
1570 | --- /dev/null | ||
1571 | +++ b/tools/testing/selftests/powerpc/security/entry_flush.c | ||
1572 | @@ -0,0 +1,163 @@ | ||
1573 | +// SPDX-License-Identifier: GPL-2.0+ | ||
1574 | + | ||
1575 | +/* | ||
1576 | + * Copyright 2018 IBM Corporation. | ||
1577 | + */ | ||
1578 | + | ||
1579 | +#define __SANE_USERSPACE_TYPES__ | ||
1580 | + | ||
1581 | +#include <sys/types.h> | ||
1582 | +#include <stdint.h> | ||
1583 | +#include <malloc.h> | ||
1584 | +#include <unistd.h> | ||
1585 | +#include <signal.h> | ||
1586 | +#include <stdlib.h> | ||
1587 | +#include <string.h> | ||
1588 | +#include <stdio.h> | ||
1589 | +#include "utils.h" | ||
1590 | + | ||
1591 | +#define CACHELINE_SIZE 128 | ||
1592 | + | ||
1593 | +struct perf_event_read { | ||
1594 | + __u64 nr; | ||
1595 | + __u64 l1d_misses; | ||
1596 | +}; | ||
1597 | + | ||
1598 | +static inline __u64 load(void *addr) | ||
1599 | +{ | ||
1600 | + __u64 tmp; | ||
1601 | + | ||
1602 | + asm volatile("ld %0,0(%1)" : "=r"(tmp) : "b"(addr)); | ||
1603 | + | ||
1604 | + return tmp; | ||
1605 | +} | ||
1606 | + | ||
1607 | +static void syscall_loop(char *p, unsigned long iterations, | ||
1608 | + unsigned long zero_size) | ||
1609 | +{ | ||
1610 | + for (unsigned long i = 0; i < iterations; i++) { | ||
1611 | + for (unsigned long j = 0; j < zero_size; j += CACHELINE_SIZE) | ||
1612 | + load(p + j); | ||
1613 | + getppid(); | ||
1614 | + } | ||
1615 | +} | ||
1616 | + | ||
1617 | +int entry_flush_test(void) | ||
1618 | +{ | ||
1619 | + char *p; | ||
1620 | + int repetitions = 10; | ||
1621 | + int fd, passes = 0, iter, rc = 0; | ||
1622 | + struct perf_event_read v; | ||
1623 | + __u64 l1d_misses_total = 0; | ||
1624 | + unsigned long iterations = 100000, zero_size = 24 * 1024; | ||
1625 | + unsigned long l1d_misses_expected; | ||
1626 | + int rfi_flush_orig; | ||
1627 | + int entry_flush, entry_flush_orig; | ||
1628 | + | ||
1629 | + SKIP_IF(geteuid() != 0); | ||
1630 | + | ||
1631 | + // The PMU event we use only works on Power7 or later | ||
1632 | + SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06)); | ||
1633 | + | ||
1634 | + if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) { | ||
1635 | + perror("Unable to read powerpc/rfi_flush debugfs file"); | ||
1636 | + SKIP_IF(1); | ||
1637 | + } | ||
1638 | + | ||
1639 | + if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) { | ||
1640 | + perror("Unable to read powerpc/entry_flush debugfs file"); | ||
1641 | + SKIP_IF(1); | ||
1642 | + } | ||
1643 | + | ||
1644 | + if (rfi_flush_orig != 0) { | ||
1645 | + if (write_debugfs_file("powerpc/rfi_flush", 0) < 0) { | ||
1646 | + perror("error writing to powerpc/rfi_flush debugfs file"); | ||
1647 | + FAIL_IF(1); | ||
1648 | + } | ||
1649 | + } | ||
1650 | + | ||
1651 | + entry_flush = entry_flush_orig; | ||
1652 | + | ||
1653 | + fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); | ||
1654 | + FAIL_IF(fd < 0); | ||
1655 | + | ||
1656 | + p = (char *)memalign(zero_size, CACHELINE_SIZE); | ||
1657 | + | ||
1658 | + FAIL_IF(perf_event_enable(fd)); | ||
1659 | + | ||
1660 | + // disable L1 prefetching | ||
1661 | + set_dscr(1); | ||
1662 | + | ||
1663 | + iter = repetitions; | ||
1664 | + | ||
1665 | + /* | ||
1666 | + * We expect to see l1d miss for each cacheline access when entry_flush | ||
1667 | + * is set. Allow a small variation on this. | ||
1668 | + */ | ||
1669 | + l1d_misses_expected = iterations * (zero_size / CACHELINE_SIZE - 2); | ||
1670 | + | ||
1671 | +again: | ||
1672 | + FAIL_IF(perf_event_reset(fd)); | ||
1673 | + | ||
1674 | + syscall_loop(p, iterations, zero_size); | ||
1675 | + | ||
1676 | + FAIL_IF(read(fd, &v, sizeof(v)) != sizeof(v)); | ||
1677 | + | ||
1678 | + if (entry_flush && v.l1d_misses >= l1d_misses_expected) | ||
1679 | + passes++; | ||
1680 | + else if (!entry_flush && v.l1d_misses < (l1d_misses_expected / 2)) | ||
1681 | + passes++; | ||
1682 | + | ||
1683 | + l1d_misses_total += v.l1d_misses; | ||
1684 | + | ||
1685 | + while (--iter) | ||
1686 | + goto again; | ||
1687 | + | ||
1688 | + if (passes < repetitions) { | ||
1689 | + printf("FAIL (L1D misses with entry_flush=%d: %llu %c %lu) [%d/%d failures]\n", | ||
1690 | + entry_flush, l1d_misses_total, entry_flush ? '<' : '>', | ||
1691 | + entry_flush ? repetitions * l1d_misses_expected : | ||
1692 | + repetitions * l1d_misses_expected / 2, | ||
1693 | + repetitions - passes, repetitions); | ||
1694 | + rc = 1; | ||
1695 | + } else | ||
1696 | + printf("PASS (L1D misses with entry_flush=%d: %llu %c %lu) [%d/%d pass]\n", | ||
1697 | + entry_flush, l1d_misses_total, entry_flush ? '>' : '<', | ||
1698 | + entry_flush ? repetitions * l1d_misses_expected : | ||
1699 | + repetitions * l1d_misses_expected / 2, | ||
1700 | + passes, repetitions); | ||
1701 | + | ||
1702 | + if (entry_flush == entry_flush_orig) { | ||
1703 | + entry_flush = !entry_flush_orig; | ||
1704 | + if (write_debugfs_file("powerpc/entry_flush", entry_flush) < 0) { | ||
1705 | + perror("error writing to powerpc/entry_flush debugfs file"); | ||
1706 | + return 1; | ||
1707 | + } | ||
1708 | + iter = repetitions; | ||
1709 | + l1d_misses_total = 0; | ||
1710 | + passes = 0; | ||
1711 | + goto again; | ||
1712 | + } | ||
1713 | + | ||
1714 | + perf_event_disable(fd); | ||
1715 | + close(fd); | ||
1716 | + | ||
1717 | + set_dscr(0); | ||
1718 | + | ||
1719 | + if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) { | ||
1720 | + perror("unable to restore original value of powerpc/rfi_flush debugfs file"); | ||
1721 | + return 1; | ||
1722 | + } | ||
1723 | + | ||
1724 | + if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) { | ||
1725 | + perror("unable to restore original value of powerpc/entry_flush debugfs file"); | ||
1726 | + return 1; | ||
1727 | + } | ||
1728 | + | ||
1729 | + return rc; | ||
1730 | +} | ||
1731 | + | ||
1732 | +int main(int argc, char *argv[]) | ||
1733 | +{ | ||
1734 | + return test_harness(entry_flush_test, "entry_flush_test"); | ||
1735 | +} | ||
1736 | diff --git a/tools/testing/selftests/powerpc/security/rfi_flush.c b/tools/testing/selftests/powerpc/security/rfi_flush.c | ||
1737 | index 0a7d0afb26b88..533315e68133d 100644 | ||
1738 | --- a/tools/testing/selftests/powerpc/security/rfi_flush.c | ||
1739 | +++ b/tools/testing/selftests/powerpc/security/rfi_flush.c | ||
1740 | @@ -50,16 +50,30 @@ int rfi_flush_test(void) | ||
1741 | __u64 l1d_misses_total = 0; | ||
1742 | unsigned long iterations = 100000, zero_size = 24 * 1024; | ||
1743 | unsigned long l1d_misses_expected; | ||
1744 | - int rfi_flush_org, rfi_flush; | ||
1745 | + int rfi_flush_orig, rfi_flush; | ||
1746 | + int have_entry_flush, entry_flush_orig; | ||
1747 | |||
1748 | SKIP_IF(geteuid() != 0); | ||
1749 | |||
1750 | - if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_org)) { | ||
1751 | + if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) { | ||
1752 | perror("Unable to read powerpc/rfi_flush debugfs file"); | ||
1753 | SKIP_IF(1); | ||
1754 | } | ||
1755 | |||
1756 | - rfi_flush = rfi_flush_org; | ||
1757 | + if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) { | ||
1758 | + have_entry_flush = 0; | ||
1759 | + } else { | ||
1760 | + have_entry_flush = 1; | ||
1761 | + | ||
1762 | + if (entry_flush_orig != 0) { | ||
1763 | + if (write_debugfs_file("powerpc/entry_flush", 0) < 0) { | ||
1764 | + perror("error writing to powerpc/entry_flush debugfs file"); | ||
1765 | + return 1; | ||
1766 | + } | ||
1767 | + } | ||
1768 | + } | ||
1769 | + | ||
1770 | + rfi_flush = rfi_flush_orig; | ||
1771 | |||
1772 | fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); | ||
1773 | FAIL_IF(fd < 0); | ||
1774 | @@ -68,6 +82,7 @@ int rfi_flush_test(void) | ||
1775 | |||
1776 | FAIL_IF(perf_event_enable(fd)); | ||
1777 | |||
1778 | + // disable L1 prefetching | ||
1779 | set_dscr(1); | ||
1780 | |||
1781 | iter = repetitions; | ||
1782 | @@ -109,8 +124,8 @@ again: | ||
1783 | repetitions * l1d_misses_expected / 2, | ||
1784 | passes, repetitions); | ||
1785 | |||
1786 | - if (rfi_flush == rfi_flush_org) { | ||
1787 | - rfi_flush = !rfi_flush_org; | ||
1788 | + if (rfi_flush == rfi_flush_orig) { | ||
1789 | + rfi_flush = !rfi_flush_orig; | ||
1790 | if (write_debugfs_file("powerpc/rfi_flush", rfi_flush) < 0) { | ||
1791 | perror("error writing to powerpc/rfi_flush debugfs file"); | ||
1792 | return 1; | ||
1793 | @@ -126,11 +141,19 @@ again: | ||
1794 | |||
1795 | set_dscr(0); | ||
1796 | |||
1797 | - if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_org) < 0) { | ||
1798 | + if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) { | ||
1799 | perror("unable to restore original value of powerpc/rfi_flush debugfs file"); | ||
1800 | return 1; | ||
1801 | } | ||
1802 | |||
1803 | + if (have_entry_flush) { | ||
1804 | + if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) { | ||
1805 | + perror("unable to restore original value of powerpc/entry_flush " | ||
1806 | + "debugfs file"); | ||
1807 | + return 1; | ||
1808 | + } | ||
1809 | + } | ||
1810 | + | ||
1811 | return rc; | ||
1812 | } | ||
1813 |