Contents of /trunk/kernel-alx/patches-5.4/0178-5.4.79-all-fixes.patch
Parent Directory | Revision Log
Revision 3635 -
(show annotations)
(download)
Mon Oct 24 12:34:12 2022 UTC (19 months, 1 week ago) by niro
File size: 56223 byte(s)
Mon Oct 24 12:34:12 2022 UTC (19 months, 1 week ago) by niro
File size: 56223 byte(s)
-sync kernel patches
1 | diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt |
2 | index 5b4753e602def..fea15cd49fbc7 100644 |
3 | --- a/Documentation/admin-guide/kernel-parameters.txt |
4 | +++ b/Documentation/admin-guide/kernel-parameters.txt |
5 | @@ -2667,6 +2667,8 @@ |
6 | mds=off [X86] |
7 | tsx_async_abort=off [X86] |
8 | kvm.nx_huge_pages=off [X86] |
9 | + no_entry_flush [PPC] |
10 | + no_uaccess_flush [PPC] |
11 | |
12 | Exceptions: |
13 | This does not have any effect on |
14 | @@ -2989,6 +2991,8 @@ |
15 | |
16 | noefi Disable EFI runtime services support. |
17 | |
18 | + no_entry_flush [PPC] Don't flush the L1-D cache when entering the kernel. |
19 | + |
20 | noexec [IA-64] |
21 | |
22 | noexec [X86] |
23 | @@ -3038,6 +3042,9 @@ |
24 | nospec_store_bypass_disable |
25 | [HW] Disable all mitigations for the Speculative Store Bypass vulnerability |
26 | |
27 | + no_uaccess_flush |
28 | + [PPC] Don't flush the L1-D cache after accessing user data. |
29 | + |
30 | noxsave [BUGS=X86] Disables x86 extended register state save |
31 | and restore using xsave. The kernel will fallback to |
32 | enabling legacy floating-point and sse state. |
33 | diff --git a/Makefile b/Makefile |
34 | index 5725b07aaddf0..f02539be5e073 100644 |
35 | --- a/Makefile |
36 | +++ b/Makefile |
37 | @@ -1,7 +1,7 @@ |
38 | # SPDX-License-Identifier: GPL-2.0 |
39 | VERSION = 5 |
40 | PATCHLEVEL = 4 |
41 | -SUBLEVEL = 78 |
42 | +SUBLEVEL = 79 |
43 | EXTRAVERSION = |
44 | NAME = Kleptomaniac Octopus |
45 | |
46 | diff --git a/arch/mips/pci/pci-xtalk-bridge.c b/arch/mips/pci/pci-xtalk-bridge.c |
47 | index c4b1c6cf26606..adc9f83b2c448 100644 |
48 | --- a/arch/mips/pci/pci-xtalk-bridge.c |
49 | +++ b/arch/mips/pci/pci-xtalk-bridge.c |
50 | @@ -284,7 +284,7 @@ static int bridge_set_affinity(struct irq_data *d, const struct cpumask *mask, |
51 | ret = irq_chip_set_affinity_parent(d, mask, force); |
52 | if (ret >= 0) { |
53 | cpu = cpumask_first_and(mask, cpu_online_mask); |
54 | - data->nnasid = COMPACT_TO_NASID_NODEID(cpu_to_node(cpu)); |
55 | + data->nasid = COMPACT_TO_NASID_NODEID(cpu_to_node(cpu)); |
56 | bridge_write(data->bc, b_int_addr[pin].addr, |
57 | (((data->bc->intr_addr >> 30) & 0x30000) | |
58 | bit | (data->nasid << 8))); |
59 | diff --git a/arch/powerpc/include/asm/book3s/64/kup-radix.h b/arch/powerpc/include/asm/book3s/64/kup-radix.h |
60 | index c8d1076e0ebbf..c1e45f510591e 100644 |
61 | --- a/arch/powerpc/include/asm/book3s/64/kup-radix.h |
62 | +++ b/arch/powerpc/include/asm/book3s/64/kup-radix.h |
63 | @@ -11,13 +11,12 @@ |
64 | |
65 | #ifdef __ASSEMBLY__ |
66 | |
67 | -.macro kuap_restore_amr gpr |
68 | #ifdef CONFIG_PPC_KUAP |
69 | +.macro kuap_restore_amr gpr |
70 | BEGIN_MMU_FTR_SECTION_NESTED(67) |
71 | ld \gpr, STACK_REGS_KUAP(r1) |
72 | mtspr SPRN_AMR, \gpr |
73 | END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_RADIX_KUAP, 67) |
74 | -#endif |
75 | .endm |
76 | |
77 | .macro kuap_check_amr gpr1, gpr2 |
78 | @@ -31,6 +30,7 @@ |
79 | END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_RADIX_KUAP, 67) |
80 | #endif |
81 | .endm |
82 | +#endif |
83 | |
84 | .macro kuap_save_amr_and_lock gpr1, gpr2, use_cr, msr_pr_cr |
85 | #ifdef CONFIG_PPC_KUAP |
86 | @@ -54,6 +54,8 @@ |
87 | |
88 | #else /* !__ASSEMBLY__ */ |
89 | |
90 | +DECLARE_STATIC_KEY_FALSE(uaccess_flush_key); |
91 | + |
92 | #ifdef CONFIG_PPC_KUAP |
93 | |
94 | #include <asm/reg.h> |
95 | @@ -77,6 +79,18 @@ static inline void set_kuap(unsigned long value) |
96 | isync(); |
97 | } |
98 | |
99 | +static inline bool |
100 | +bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) |
101 | +{ |
102 | + return WARN(mmu_has_feature(MMU_FTR_RADIX_KUAP) && |
103 | + (regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)), |
104 | + "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read"); |
105 | +} |
106 | +#else /* CONFIG_PPC_KUAP */ |
107 | +static inline void kuap_restore_amr(struct pt_regs *regs, unsigned long amr) { } |
108 | +static inline void set_kuap(unsigned long value) { } |
109 | +#endif /* !CONFIG_PPC_KUAP */ |
110 | + |
111 | static __always_inline void allow_user_access(void __user *to, const void __user *from, |
112 | unsigned long size, unsigned long dir) |
113 | { |
114 | @@ -94,17 +108,10 @@ static inline void prevent_user_access(void __user *to, const void __user *from, |
115 | unsigned long size, unsigned long dir) |
116 | { |
117 | set_kuap(AMR_KUAP_BLOCKED); |
118 | + if (static_branch_unlikely(&uaccess_flush_key)) |
119 | + do_uaccess_flush(); |
120 | } |
121 | |
122 | -static inline bool |
123 | -bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) |
124 | -{ |
125 | - return WARN(mmu_has_feature(MMU_FTR_RADIX_KUAP) && |
126 | - (regs->kuap & (is_write ? AMR_KUAP_BLOCK_WRITE : AMR_KUAP_BLOCK_READ)), |
127 | - "Bug: %s fault blocked by AMR!", is_write ? "Write" : "Read"); |
128 | -} |
129 | -#endif /* CONFIG_PPC_KUAP */ |
130 | - |
131 | #endif /* __ASSEMBLY__ */ |
132 | |
133 | #endif /* _ASM_POWERPC_BOOK3S_64_KUP_RADIX_H */ |
134 | diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h |
135 | index 33f4f72eb035b..6d0795d7b89c1 100644 |
136 | --- a/arch/powerpc/include/asm/exception-64s.h |
137 | +++ b/arch/powerpc/include/asm/exception-64s.h |
138 | @@ -61,11 +61,18 @@ |
139 | nop; \ |
140 | nop |
141 | |
142 | +#define ENTRY_FLUSH_SLOT \ |
143 | + ENTRY_FLUSH_FIXUP_SECTION; \ |
144 | + nop; \ |
145 | + nop; \ |
146 | + nop; |
147 | + |
148 | /* |
149 | * r10 must be free to use, r13 must be paca |
150 | */ |
151 | #define INTERRUPT_TO_KERNEL \ |
152 | - STF_ENTRY_BARRIER_SLOT |
153 | + STF_ENTRY_BARRIER_SLOT; \ |
154 | + ENTRY_FLUSH_SLOT |
155 | |
156 | /* |
157 | * Macros for annotating the expected destination of (h)rfid |
158 | @@ -127,6 +134,9 @@ |
159 | hrfid; \ |
160 | b hrfi_flush_fallback |
161 | |
162 | +#else /* __ASSEMBLY__ */ |
163 | +/* Prototype for function defined in exceptions-64s.S */ |
164 | +void do_uaccess_flush(void); |
165 | #endif /* __ASSEMBLY__ */ |
166 | |
167 | #endif /* _ASM_POWERPC_EXCEPTION_H */ |
168 | diff --git a/arch/powerpc/include/asm/feature-fixups.h b/arch/powerpc/include/asm/feature-fixups.h |
169 | index b0af97add7517..fbd406cd6916c 100644 |
170 | --- a/arch/powerpc/include/asm/feature-fixups.h |
171 | +++ b/arch/powerpc/include/asm/feature-fixups.h |
172 | @@ -205,6 +205,22 @@ label##3: \ |
173 | FTR_ENTRY_OFFSET 955b-956b; \ |
174 | .popsection; |
175 | |
176 | +#define UACCESS_FLUSH_FIXUP_SECTION \ |
177 | +959: \ |
178 | + .pushsection __uaccess_flush_fixup,"a"; \ |
179 | + .align 2; \ |
180 | +960: \ |
181 | + FTR_ENTRY_OFFSET 959b-960b; \ |
182 | + .popsection; |
183 | + |
184 | +#define ENTRY_FLUSH_FIXUP_SECTION \ |
185 | +957: \ |
186 | + .pushsection __entry_flush_fixup,"a"; \ |
187 | + .align 2; \ |
188 | +958: \ |
189 | + FTR_ENTRY_OFFSET 957b-958b; \ |
190 | + .popsection; |
191 | + |
192 | #define RFI_FLUSH_FIXUP_SECTION \ |
193 | 951: \ |
194 | .pushsection __rfi_flush_fixup,"a"; \ |
195 | @@ -237,8 +253,11 @@ label##3: \ |
196 | #include <linux/types.h> |
197 | |
198 | extern long stf_barrier_fallback; |
199 | +extern long entry_flush_fallback; |
200 | extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup; |
201 | extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup; |
202 | +extern long __start___uaccess_flush_fixup, __stop___uaccess_flush_fixup; |
203 | +extern long __start___entry_flush_fixup, __stop___entry_flush_fixup; |
204 | extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup; |
205 | extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup; |
206 | extern long __start__btb_flush_fixup, __stop__btb_flush_fixup; |
207 | diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h |
208 | index 94f24928916a8..ed4f5f536fc1d 100644 |
209 | --- a/arch/powerpc/include/asm/kup.h |
210 | +++ b/arch/powerpc/include/asm/kup.h |
211 | @@ -6,7 +6,7 @@ |
212 | #define KUAP_WRITE 2 |
213 | #define KUAP_READ_WRITE (KUAP_READ | KUAP_WRITE) |
214 | |
215 | -#ifdef CONFIG_PPC64 |
216 | +#ifdef CONFIG_PPC_BOOK3S_64 |
217 | #include <asm/book3s/64/kup-radix.h> |
218 | #endif |
219 | #ifdef CONFIG_PPC_8xx |
220 | @@ -24,9 +24,15 @@ |
221 | .macro kuap_restore sp, current, gpr1, gpr2, gpr3 |
222 | .endm |
223 | |
224 | +.macro kuap_restore_amr gpr |
225 | +.endm |
226 | + |
227 | .macro kuap_check current, gpr |
228 | .endm |
229 | |
230 | +.macro kuap_check_amr gpr1, gpr2 |
231 | +.endm |
232 | + |
233 | #endif |
234 | |
235 | #else /* !__ASSEMBLY__ */ |
236 | @@ -45,15 +51,26 @@ static inline void setup_kuep(bool disabled) { } |
237 | void setup_kuap(bool disabled); |
238 | #else |
239 | static inline void setup_kuap(bool disabled) { } |
240 | -static inline void allow_user_access(void __user *to, const void __user *from, |
241 | - unsigned long size, unsigned long dir) { } |
242 | -static inline void prevent_user_access(void __user *to, const void __user *from, |
243 | - unsigned long size, unsigned long dir) { } |
244 | + |
245 | static inline bool |
246 | bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) |
247 | { |
248 | return false; |
249 | } |
250 | + |
251 | +static inline void kuap_check_amr(void) { } |
252 | + |
253 | +/* |
254 | + * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush |
255 | + * the L1D cache after user accesses. Only include the empty stubs for other |
256 | + * platforms. |
257 | + */ |
258 | +#ifndef CONFIG_PPC_BOOK3S_64 |
259 | +static inline void allow_user_access(void __user *to, const void __user *from, |
260 | + unsigned long size, unsigned long dir) { } |
261 | +static inline void prevent_user_access(void __user *to, const void __user *from, |
262 | + unsigned long size, unsigned long dir) { } |
263 | +#endif /* CONFIG_PPC_BOOK3S_64 */ |
264 | #endif /* CONFIG_PPC_KUAP */ |
265 | |
266 | static inline void allow_read_from_user(const void __user *from, unsigned long size) |
267 | diff --git a/arch/powerpc/include/asm/security_features.h b/arch/powerpc/include/asm/security_features.h |
268 | index 7c05e95a5c444..e9e3f85134e54 100644 |
269 | --- a/arch/powerpc/include/asm/security_features.h |
270 | +++ b/arch/powerpc/include/asm/security_features.h |
271 | @@ -84,12 +84,19 @@ static inline bool security_ftr_enabled(u64 feature) |
272 | // Software required to flush link stack on context switch |
273 | #define SEC_FTR_FLUSH_LINK_STACK 0x0000000000001000ull |
274 | |
275 | +// The L1-D cache should be flushed when entering the kernel |
276 | +#define SEC_FTR_L1D_FLUSH_ENTRY 0x0000000000004000ull |
277 | + |
278 | +// The L1-D cache should be flushed after user accesses from the kernel |
279 | +#define SEC_FTR_L1D_FLUSH_UACCESS 0x0000000000008000ull |
280 | |
281 | // Features enabled by default |
282 | #define SEC_FTR_DEFAULT \ |
283 | (SEC_FTR_L1D_FLUSH_HV | \ |
284 | SEC_FTR_L1D_FLUSH_PR | \ |
285 | SEC_FTR_BNDS_CHK_SPEC_BAR | \ |
286 | + SEC_FTR_L1D_FLUSH_ENTRY | \ |
287 | + SEC_FTR_L1D_FLUSH_UACCESS | \ |
288 | SEC_FTR_FAVOUR_SECURITY) |
289 | |
290 | #endif /* _ASM_POWERPC_SECURITY_FEATURES_H */ |
291 | diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h |
292 | index 65676e2325b85..6f2f4497e13b3 100644 |
293 | --- a/arch/powerpc/include/asm/setup.h |
294 | +++ b/arch/powerpc/include/asm/setup.h |
295 | @@ -52,12 +52,16 @@ enum l1d_flush_type { |
296 | }; |
297 | |
298 | void setup_rfi_flush(enum l1d_flush_type, bool enable); |
299 | +void setup_entry_flush(bool enable); |
300 | +void setup_uaccess_flush(bool enable); |
301 | void do_rfi_flush_fixups(enum l1d_flush_type types); |
302 | #ifdef CONFIG_PPC_BARRIER_NOSPEC |
303 | void setup_barrier_nospec(void); |
304 | #else |
305 | static inline void setup_barrier_nospec(void) { }; |
306 | #endif |
307 | +void do_uaccess_flush_fixups(enum l1d_flush_type types); |
308 | +void do_entry_flush_fixups(enum l1d_flush_type types); |
309 | void do_barrier_nospec_fixups(bool enable); |
310 | extern bool barrier_nospec_enabled; |
311 | |
312 | diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S |
313 | index 70ac8a6ba0c18..88bba0a931d65 100644 |
314 | --- a/arch/powerpc/kernel/exceptions-64s.S |
315 | +++ b/arch/powerpc/kernel/exceptions-64s.S |
316 | @@ -1150,7 +1150,7 @@ EXC_REAL_BEGIN(data_access, 0x300, 0x80) |
317 | INT_HANDLER data_access, 0x300, ool=1, dar=1, dsisr=1, kvm=1 |
318 | EXC_REAL_END(data_access, 0x300, 0x80) |
319 | EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) |
320 | - INT_HANDLER data_access, 0x300, virt=1, dar=1, dsisr=1 |
321 | + INT_HANDLER data_access, 0x300, ool=1, virt=1, dar=1, dsisr=1 |
322 | EXC_VIRT_END(data_access, 0x4300, 0x80) |
323 | INT_KVM_HANDLER data_access, 0x300, EXC_STD, PACA_EXGEN, 1 |
324 | EXC_COMMON_BEGIN(data_access_common) |
325 | @@ -1205,7 +1205,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) |
326 | |
327 | |
328 | EXC_REAL_BEGIN(instruction_access, 0x400, 0x80) |
329 | - INT_HANDLER instruction_access, 0x400, kvm=1 |
330 | + INT_HANDLER instruction_access, 0x400, ool=1, kvm=1 |
331 | EXC_REAL_END(instruction_access, 0x400, 0x80) |
332 | EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) |
333 | INT_HANDLER instruction_access, 0x400, virt=1 |
334 | @@ -1225,7 +1225,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) |
335 | |
336 | |
337 | EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) |
338 | - INT_HANDLER instruction_access_slb, 0x480, area=PACA_EXSLB, kvm=1 |
339 | + INT_HANDLER instruction_access_slb, 0x480, ool=1, area=PACA_EXSLB, kvm=1 |
340 | EXC_REAL_END(instruction_access_slb, 0x480, 0x80) |
341 | EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) |
342 | INT_HANDLER instruction_access_slb, 0x480, virt=1, area=PACA_EXSLB |
343 | @@ -1365,17 +1365,17 @@ EXC_REAL_BEGIN(decrementer, 0x900, 0x80) |
344 | INT_HANDLER decrementer, 0x900, ool=1, bitmask=IRQS_DISABLED, kvm=1 |
345 | EXC_REAL_END(decrementer, 0x900, 0x80) |
346 | EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) |
347 | - INT_HANDLER decrementer, 0x900, virt=1, bitmask=IRQS_DISABLED |
348 | + INT_HANDLER decrementer, 0x900, ool=1, virt=1, bitmask=IRQS_DISABLED |
349 | EXC_VIRT_END(decrementer, 0x4900, 0x80) |
350 | INT_KVM_HANDLER decrementer, 0x900, EXC_STD, PACA_EXGEN, 0 |
351 | EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt) |
352 | |
353 | |
354 | EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80) |
355 | - INT_HANDLER hdecrementer, 0x980, hsrr=EXC_HV, kvm=1 |
356 | + INT_HANDLER hdecrementer, 0x980, ool=1, hsrr=EXC_HV, kvm=1 |
357 | EXC_REAL_END(hdecrementer, 0x980, 0x80) |
358 | EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80) |
359 | - INT_HANDLER hdecrementer, 0x980, virt=1, hsrr=EXC_HV, kvm=1 |
360 | + INT_HANDLER hdecrementer, 0x980, ool=1, virt=1, hsrr=EXC_HV, kvm=1 |
361 | EXC_VIRT_END(hdecrementer, 0x4980, 0x80) |
362 | INT_KVM_HANDLER hdecrementer, 0x980, EXC_HV, PACA_EXGEN, 0 |
363 | EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt) |
364 | @@ -2046,15 +2046,8 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback) |
365 | .endr |
366 | blr |
367 | |
368 | -TRAMP_REAL_BEGIN(rfi_flush_fallback) |
369 | - SET_SCRATCH0(r13); |
370 | - GET_PACA(r13); |
371 | - std r1,PACA_EXRFI+EX_R12(r13) |
372 | - ld r1,PACAKSAVE(r13) |
373 | - std r9,PACA_EXRFI+EX_R9(r13) |
374 | - std r10,PACA_EXRFI+EX_R10(r13) |
375 | - std r11,PACA_EXRFI+EX_R11(r13) |
376 | - mfctr r9 |
377 | +/* Clobbers r10, r11, ctr */ |
378 | +.macro L1D_DISPLACEMENT_FLUSH |
379 | ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) |
380 | ld r11,PACA_L1D_FLUSH_SIZE(r13) |
381 | srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ |
382 | @@ -2065,7 +2058,7 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback) |
383 | sync |
384 | |
385 | /* |
386 | - * The load adresses are at staggered offsets within cachelines, |
387 | + * The load addresses are at staggered offsets within cachelines, |
388 | * which suits some pipelines better (on others it should not |
389 | * hurt). |
390 | */ |
391 | @@ -2080,7 +2073,30 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback) |
392 | ld r11,(0x80 + 8)*7(r10) |
393 | addi r10,r10,0x80*8 |
394 | bdnz 1b |
395 | +.endm |
396 | + |
397 | +TRAMP_REAL_BEGIN(entry_flush_fallback) |
398 | + std r9,PACA_EXRFI+EX_R9(r13) |
399 | + std r10,PACA_EXRFI+EX_R10(r13) |
400 | + std r11,PACA_EXRFI+EX_R11(r13) |
401 | + mfctr r9 |
402 | + L1D_DISPLACEMENT_FLUSH |
403 | + mtctr r9 |
404 | + ld r9,PACA_EXRFI+EX_R9(r13) |
405 | + ld r10,PACA_EXRFI+EX_R10(r13) |
406 | + ld r11,PACA_EXRFI+EX_R11(r13) |
407 | + blr |
408 | |
409 | +TRAMP_REAL_BEGIN(rfi_flush_fallback) |
410 | + SET_SCRATCH0(r13); |
411 | + GET_PACA(r13); |
412 | + std r1,PACA_EXRFI+EX_R12(r13) |
413 | + ld r1,PACAKSAVE(r13) |
414 | + std r9,PACA_EXRFI+EX_R9(r13) |
415 | + std r10,PACA_EXRFI+EX_R10(r13) |
416 | + std r11,PACA_EXRFI+EX_R11(r13) |
417 | + mfctr r9 |
418 | + L1D_DISPLACEMENT_FLUSH |
419 | mtctr r9 |
420 | ld r9,PACA_EXRFI+EX_R9(r13) |
421 | ld r10,PACA_EXRFI+EX_R10(r13) |
422 | @@ -2098,32 +2114,7 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) |
423 | std r10,PACA_EXRFI+EX_R10(r13) |
424 | std r11,PACA_EXRFI+EX_R11(r13) |
425 | mfctr r9 |
426 | - ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) |
427 | - ld r11,PACA_L1D_FLUSH_SIZE(r13) |
428 | - srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ |
429 | - mtctr r11 |
430 | - DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ |
431 | - |
432 | - /* order ld/st prior to dcbt stop all streams with flushing */ |
433 | - sync |
434 | - |
435 | - /* |
436 | - * The load adresses are at staggered offsets within cachelines, |
437 | - * which suits some pipelines better (on others it should not |
438 | - * hurt). |
439 | - */ |
440 | -1: |
441 | - ld r11,(0x80 + 8)*0(r10) |
442 | - ld r11,(0x80 + 8)*1(r10) |
443 | - ld r11,(0x80 + 8)*2(r10) |
444 | - ld r11,(0x80 + 8)*3(r10) |
445 | - ld r11,(0x80 + 8)*4(r10) |
446 | - ld r11,(0x80 + 8)*5(r10) |
447 | - ld r11,(0x80 + 8)*6(r10) |
448 | - ld r11,(0x80 + 8)*7(r10) |
449 | - addi r10,r10,0x80*8 |
450 | - bdnz 1b |
451 | - |
452 | + L1D_DISPLACEMENT_FLUSH |
453 | mtctr r9 |
454 | ld r9,PACA_EXRFI+EX_R9(r13) |
455 | ld r10,PACA_EXRFI+EX_R10(r13) |
456 | @@ -2132,6 +2123,19 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) |
457 | GET_SCRATCH0(r13); |
458 | hrfid |
459 | |
460 | +USE_TEXT_SECTION() |
461 | + |
462 | +_GLOBAL(do_uaccess_flush) |
463 | + UACCESS_FLUSH_FIXUP_SECTION |
464 | + nop |
465 | + nop |
466 | + nop |
467 | + blr |
468 | + L1D_DISPLACEMENT_FLUSH |
469 | + blr |
470 | +_ASM_NOKPROBE_SYMBOL(do_uaccess_flush) |
471 | +EXPORT_SYMBOL(do_uaccess_flush) |
472 | + |
473 | /* |
474 | * Real mode exceptions actually use this too, but alternate |
475 | * instruction code patches (which end up in the common .text area) |
476 | diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S |
477 | index 98d8b6832fcb5..f6428b90a6c77 100644 |
478 | --- a/arch/powerpc/kernel/head_8xx.S |
479 | +++ b/arch/powerpc/kernel/head_8xx.S |
480 | @@ -229,9 +229,7 @@ SystemCall: |
481 | |
482 | InstructionTLBMiss: |
483 | mtspr SPRN_SPRG_SCRATCH0, r10 |
484 | -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) |
485 | mtspr SPRN_SPRG_SCRATCH1, r11 |
486 | -#endif |
487 | |
488 | /* If we are faulting a kernel address, we have to use the |
489 | * kernel page tables. |
490 | @@ -278,11 +276,9 @@ InstructionTLBMiss: |
491 | #ifdef ITLB_MISS_KERNEL |
492 | mtcr r11 |
493 | #endif |
494 | -#ifdef CONFIG_SWAP |
495 | - rlwinm r11, r10, 32-5, _PAGE_PRESENT |
496 | + rlwinm r11, r10, 32-7, _PAGE_PRESENT |
497 | and r11, r11, r10 |
498 | rlwimi r10, r11, 0, _PAGE_PRESENT |
499 | -#endif |
500 | /* The Linux PTE won't go exactly into the MMU TLB. |
501 | * Software indicator bits 20 and 23 must be clear. |
502 | * Software indicator bits 22, 24, 25, 26, and 27 must be |
503 | @@ -296,9 +292,7 @@ InstructionTLBMiss: |
504 | |
505 | /* Restore registers */ |
506 | 0: mfspr r10, SPRN_SPRG_SCRATCH0 |
507 | -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) |
508 | mfspr r11, SPRN_SPRG_SCRATCH1 |
509 | -#endif |
510 | rfi |
511 | patch_site 0b, patch__itlbmiss_exit_1 |
512 | |
513 | @@ -308,9 +302,7 @@ InstructionTLBMiss: |
514 | addi r10, r10, 1 |
515 | stw r10, (itlb_miss_counter - PAGE_OFFSET)@l(0) |
516 | mfspr r10, SPRN_SPRG_SCRATCH0 |
517 | -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) |
518 | mfspr r11, SPRN_SPRG_SCRATCH1 |
519 | -#endif |
520 | rfi |
521 | #endif |
522 | |
523 | @@ -394,11 +386,9 @@ DataStoreTLBMiss: |
524 | * r11 = ((r10 & PRESENT) & ((r10 & ACCESSED) >> 5)); |
525 | * r10 = (r10 & ~PRESENT) | r11; |
526 | */ |
527 | -#ifdef CONFIG_SWAP |
528 | - rlwinm r11, r10, 32-5, _PAGE_PRESENT |
529 | + rlwinm r11, r10, 32-7, _PAGE_PRESENT |
530 | and r11, r11, r10 |
531 | rlwimi r10, r11, 0, _PAGE_PRESENT |
532 | -#endif |
533 | /* The Linux PTE won't go exactly into the MMU TLB. |
534 | * Software indicator bits 24, 25, 26, and 27 must be |
535 | * set. All other Linux PTE bits control the behavior |
536 | diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c |
537 | index e50fbed366516..480c236724da2 100644 |
538 | --- a/arch/powerpc/kernel/setup_64.c |
539 | +++ b/arch/powerpc/kernel/setup_64.c |
540 | @@ -859,7 +859,13 @@ early_initcall(disable_hardlockup_detector); |
541 | static enum l1d_flush_type enabled_flush_types; |
542 | static void *l1d_flush_fallback_area; |
543 | static bool no_rfi_flush; |
544 | +static bool no_entry_flush; |
545 | +static bool no_uaccess_flush; |
546 | bool rfi_flush; |
547 | +bool entry_flush; |
548 | +bool uaccess_flush; |
549 | +DEFINE_STATIC_KEY_FALSE(uaccess_flush_key); |
550 | +EXPORT_SYMBOL(uaccess_flush_key); |
551 | |
552 | static int __init handle_no_rfi_flush(char *p) |
553 | { |
554 | @@ -869,6 +875,22 @@ static int __init handle_no_rfi_flush(char *p) |
555 | } |
556 | early_param("no_rfi_flush", handle_no_rfi_flush); |
557 | |
558 | +static int __init handle_no_entry_flush(char *p) |
559 | +{ |
560 | + pr_info("entry-flush: disabled on command line."); |
561 | + no_entry_flush = true; |
562 | + return 0; |
563 | +} |
564 | +early_param("no_entry_flush", handle_no_entry_flush); |
565 | + |
566 | +static int __init handle_no_uaccess_flush(char *p) |
567 | +{ |
568 | + pr_info("uaccess-flush: disabled on command line."); |
569 | + no_uaccess_flush = true; |
570 | + return 0; |
571 | +} |
572 | +early_param("no_uaccess_flush", handle_no_uaccess_flush); |
573 | + |
574 | /* |
575 | * The RFI flush is not KPTI, but because users will see doco that says to use |
576 | * nopti we hijack that option here to also disable the RFI flush. |
577 | @@ -900,6 +922,32 @@ void rfi_flush_enable(bool enable) |
578 | rfi_flush = enable; |
579 | } |
580 | |
581 | +void entry_flush_enable(bool enable) |
582 | +{ |
583 | + if (enable) { |
584 | + do_entry_flush_fixups(enabled_flush_types); |
585 | + on_each_cpu(do_nothing, NULL, 1); |
586 | + } else { |
587 | + do_entry_flush_fixups(L1D_FLUSH_NONE); |
588 | + } |
589 | + |
590 | + entry_flush = enable; |
591 | +} |
592 | + |
593 | +void uaccess_flush_enable(bool enable) |
594 | +{ |
595 | + if (enable) { |
596 | + do_uaccess_flush_fixups(enabled_flush_types); |
597 | + static_branch_enable(&uaccess_flush_key); |
598 | + on_each_cpu(do_nothing, NULL, 1); |
599 | + } else { |
600 | + static_branch_disable(&uaccess_flush_key); |
601 | + do_uaccess_flush_fixups(L1D_FLUSH_NONE); |
602 | + } |
603 | + |
604 | + uaccess_flush = enable; |
605 | +} |
606 | + |
607 | static void __ref init_fallback_flush(void) |
608 | { |
609 | u64 l1d_size, limit; |
610 | @@ -958,10 +1006,28 @@ void setup_rfi_flush(enum l1d_flush_type types, bool enable) |
611 | |
612 | enabled_flush_types = types; |
613 | |
614 | - if (!no_rfi_flush && !cpu_mitigations_off()) |
615 | + if (!cpu_mitigations_off() && !no_rfi_flush) |
616 | rfi_flush_enable(enable); |
617 | } |
618 | |
619 | +void setup_entry_flush(bool enable) |
620 | +{ |
621 | + if (cpu_mitigations_off()) |
622 | + return; |
623 | + |
624 | + if (!no_entry_flush) |
625 | + entry_flush_enable(enable); |
626 | +} |
627 | + |
628 | +void setup_uaccess_flush(bool enable) |
629 | +{ |
630 | + if (cpu_mitigations_off()) |
631 | + return; |
632 | + |
633 | + if (!no_uaccess_flush) |
634 | + uaccess_flush_enable(enable); |
635 | +} |
636 | + |
637 | #ifdef CONFIG_DEBUG_FS |
638 | static int rfi_flush_set(void *data, u64 val) |
639 | { |
640 | @@ -989,9 +1055,63 @@ static int rfi_flush_get(void *data, u64 *val) |
641 | |
642 | DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n"); |
643 | |
644 | +static int entry_flush_set(void *data, u64 val) |
645 | +{ |
646 | + bool enable; |
647 | + |
648 | + if (val == 1) |
649 | + enable = true; |
650 | + else if (val == 0) |
651 | + enable = false; |
652 | + else |
653 | + return -EINVAL; |
654 | + |
655 | + /* Only do anything if we're changing state */ |
656 | + if (enable != entry_flush) |
657 | + entry_flush_enable(enable); |
658 | + |
659 | + return 0; |
660 | +} |
661 | + |
662 | +static int entry_flush_get(void *data, u64 *val) |
663 | +{ |
664 | + *val = entry_flush ? 1 : 0; |
665 | + return 0; |
666 | +} |
667 | + |
668 | +DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n"); |
669 | + |
670 | +static int uaccess_flush_set(void *data, u64 val) |
671 | +{ |
672 | + bool enable; |
673 | + |
674 | + if (val == 1) |
675 | + enable = true; |
676 | + else if (val == 0) |
677 | + enable = false; |
678 | + else |
679 | + return -EINVAL; |
680 | + |
681 | + /* Only do anything if we're changing state */ |
682 | + if (enable != uaccess_flush) |
683 | + uaccess_flush_enable(enable); |
684 | + |
685 | + return 0; |
686 | +} |
687 | + |
688 | +static int uaccess_flush_get(void *data, u64 *val) |
689 | +{ |
690 | + *val = uaccess_flush ? 1 : 0; |
691 | + return 0; |
692 | +} |
693 | + |
694 | +DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n"); |
695 | + |
696 | static __init int rfi_flush_debugfs_init(void) |
697 | { |
698 | debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush); |
699 | + debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush); |
700 | + debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush); |
701 | return 0; |
702 | } |
703 | device_initcall(rfi_flush_debugfs_init); |
704 | diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S |
705 | index 060a1acd7c6d7..5229eeac8946d 100644 |
706 | --- a/arch/powerpc/kernel/vmlinux.lds.S |
707 | +++ b/arch/powerpc/kernel/vmlinux.lds.S |
708 | @@ -143,6 +143,20 @@ SECTIONS |
709 | __stop___stf_entry_barrier_fixup = .; |
710 | } |
711 | |
712 | + . = ALIGN(8); |
713 | + __uaccess_flush_fixup : AT(ADDR(__uaccess_flush_fixup) - LOAD_OFFSET) { |
714 | + __start___uaccess_flush_fixup = .; |
715 | + *(__uaccess_flush_fixup) |
716 | + __stop___uaccess_flush_fixup = .; |
717 | + } |
718 | + |
719 | + . = ALIGN(8); |
720 | + __entry_flush_fixup : AT(ADDR(__entry_flush_fixup) - LOAD_OFFSET) { |
721 | + __start___entry_flush_fixup = .; |
722 | + *(__entry_flush_fixup) |
723 | + __stop___entry_flush_fixup = .; |
724 | + } |
725 | + |
726 | . = ALIGN(8); |
727 | __stf_exit_barrier_fixup : AT(ADDR(__stf_exit_barrier_fixup) - LOAD_OFFSET) { |
728 | __start___stf_exit_barrier_fixup = .; |
729 | diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c |
730 | index 4ba634b89ce53..e8b25f74454d6 100644 |
731 | --- a/arch/powerpc/lib/feature-fixups.c |
732 | +++ b/arch/powerpc/lib/feature-fixups.c |
733 | @@ -228,6 +228,110 @@ void do_stf_barrier_fixups(enum stf_barrier_type types) |
734 | do_stf_exit_barrier_fixups(types); |
735 | } |
736 | |
737 | +void do_uaccess_flush_fixups(enum l1d_flush_type types) |
738 | +{ |
739 | + unsigned int instrs[4], *dest; |
740 | + long *start, *end; |
741 | + int i; |
742 | + |
743 | + start = PTRRELOC(&__start___uaccess_flush_fixup); |
744 | + end = PTRRELOC(&__stop___uaccess_flush_fixup); |
745 | + |
746 | + instrs[0] = 0x60000000; /* nop */ |
747 | + instrs[1] = 0x60000000; /* nop */ |
748 | + instrs[2] = 0x60000000; /* nop */ |
749 | + instrs[3] = 0x4e800020; /* blr */ |
750 | + |
751 | + i = 0; |
752 | + if (types == L1D_FLUSH_FALLBACK) { |
753 | + instrs[3] = 0x60000000; /* nop */ |
754 | + /* fallthrough to fallback flush */ |
755 | + } |
756 | + |
757 | + if (types & L1D_FLUSH_ORI) { |
758 | + instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */ |
759 | + instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/ |
760 | + } |
761 | + |
762 | + if (types & L1D_FLUSH_MTTRIG) |
763 | + instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ |
764 | + |
765 | + for (i = 0; start < end; start++, i++) { |
766 | + dest = (void *)start + *start; |
767 | + |
768 | + pr_devel("patching dest %lx\n", (unsigned long)dest); |
769 | + |
770 | + patch_instruction(dest, instrs[0]); |
771 | + |
772 | + patch_instruction((dest + 1), instrs[1]); |
773 | + patch_instruction((dest + 2), instrs[2]); |
774 | + patch_instruction((dest + 3), instrs[3]); |
775 | + } |
776 | + |
777 | + printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i, |
778 | + (types == L1D_FLUSH_NONE) ? "no" : |
779 | + (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : |
780 | + (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) |
781 | + ? "ori+mttrig type" |
782 | + : "ori type" : |
783 | + (types & L1D_FLUSH_MTTRIG) ? "mttrig type" |
784 | + : "unknown"); |
785 | +} |
786 | + |
787 | +void do_entry_flush_fixups(enum l1d_flush_type types) |
788 | +{ |
789 | + unsigned int instrs[3], *dest; |
790 | + long *start, *end; |
791 | + int i; |
792 | + |
793 | + start = PTRRELOC(&__start___entry_flush_fixup); |
794 | + end = PTRRELOC(&__stop___entry_flush_fixup); |
795 | + |
796 | + instrs[0] = 0x60000000; /* nop */ |
797 | + instrs[1] = 0x60000000; /* nop */ |
798 | + instrs[2] = 0x60000000; /* nop */ |
799 | + |
800 | + i = 0; |
801 | + if (types == L1D_FLUSH_FALLBACK) { |
802 | + instrs[i++] = 0x7d4802a6; /* mflr r10 */ |
803 | + instrs[i++] = 0x60000000; /* branch patched below */ |
804 | + instrs[i++] = 0x7d4803a6; /* mtlr r10 */ |
805 | + } |
806 | + |
807 | + if (types & L1D_FLUSH_ORI) { |
808 | + instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */ |
809 | + instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/ |
810 | + } |
811 | + |
812 | + if (types & L1D_FLUSH_MTTRIG) |
813 | + instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ |
814 | + |
815 | + for (i = 0; start < end; start++, i++) { |
816 | + dest = (void *)start + *start; |
817 | + |
818 | + pr_devel("patching dest %lx\n", (unsigned long)dest); |
819 | + |
820 | + patch_instruction(dest, instrs[0]); |
821 | + |
822 | + if (types == L1D_FLUSH_FALLBACK) |
823 | + patch_branch((dest + 1), (unsigned long)&entry_flush_fallback, |
824 | + BRANCH_SET_LINK); |
825 | + else |
826 | + patch_instruction((dest + 1), instrs[1]); |
827 | + |
828 | + patch_instruction((dest + 2), instrs[2]); |
829 | + } |
830 | + |
831 | + printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i, |
832 | + (types == L1D_FLUSH_NONE) ? "no" : |
833 | + (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : |
834 | + (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) |
835 | + ? "ori+mttrig type" |
836 | + : "ori type" : |
837 | + (types & L1D_FLUSH_MTTRIG) ? "mttrig type" |
838 | + : "unknown"); |
839 | +} |
840 | + |
841 | void do_rfi_flush_fixups(enum l1d_flush_type types) |
842 | { |
843 | unsigned int instrs[3], *dest; |
844 | diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c |
845 | index 83498604d322b..3a9f79d18f6b0 100644 |
846 | --- a/arch/powerpc/platforms/powernv/setup.c |
847 | +++ b/arch/powerpc/platforms/powernv/setup.c |
848 | @@ -122,12 +122,29 @@ static void pnv_setup_rfi_flush(void) |
849 | type = L1D_FLUSH_ORI; |
850 | } |
851 | |
852 | + /* |
853 | + * If we are non-Power9 bare metal, we don't need to flush on kernel |
854 | + * entry or after user access: they fix a P9 specific vulnerability. |
855 | + */ |
856 | + if (!pvr_version_is(PVR_POWER9)) { |
857 | + security_ftr_clear(SEC_FTR_L1D_FLUSH_ENTRY); |
858 | + security_ftr_clear(SEC_FTR_L1D_FLUSH_UACCESS); |
859 | + } |
860 | + |
861 | enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && \ |
862 | (security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) || \ |
863 | security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV)); |
864 | |
865 | setup_rfi_flush(type, enable); |
866 | setup_count_cache_flush(); |
867 | + |
868 | + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && |
869 | + security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY); |
870 | + setup_entry_flush(enable); |
871 | + |
872 | + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && |
873 | + security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS); |
874 | + setup_uaccess_flush(enable); |
875 | } |
876 | |
877 | static void __init pnv_setup_arch(void) |
878 | diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c |
879 | index 0c8421dd01ab5..ce71235c8b81f 100644 |
880 | --- a/arch/powerpc/platforms/pseries/setup.c |
881 | +++ b/arch/powerpc/platforms/pseries/setup.c |
882 | @@ -561,6 +561,14 @@ void pseries_setup_rfi_flush(void) |
883 | |
884 | setup_rfi_flush(types, enable); |
885 | setup_count_cache_flush(); |
886 | + |
887 | + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && |
888 | + security_ftr_enabled(SEC_FTR_L1D_FLUSH_ENTRY); |
889 | + setup_entry_flush(enable); |
890 | + |
891 | + enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && |
892 | + security_ftr_enabled(SEC_FTR_L1D_FLUSH_UACCESS); |
893 | + setup_uaccess_flush(enable); |
894 | } |
895 | |
896 | #ifdef CONFIG_PCI_IOV |
897 | diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c |
898 | index 484c32b7f79ff..39265b55929d2 100644 |
899 | --- a/arch/x86/kvm/emulate.c |
900 | +++ b/arch/x86/kvm/emulate.c |
901 | @@ -4050,6 +4050,12 @@ static int em_clflush(struct x86_emulate_ctxt *ctxt) |
902 | return X86EMUL_CONTINUE; |
903 | } |
904 | |
905 | +static int em_clflushopt(struct x86_emulate_ctxt *ctxt) |
906 | +{ |
907 | + /* emulating clflushopt regardless of cpuid */ |
908 | + return X86EMUL_CONTINUE; |
909 | +} |
910 | + |
911 | static int em_movsxd(struct x86_emulate_ctxt *ctxt) |
912 | { |
913 | ctxt->dst.val = (s32) ctxt->src.val; |
914 | @@ -4592,7 +4598,7 @@ static const struct opcode group11[] = { |
915 | }; |
916 | |
917 | static const struct gprefix pfx_0f_ae_7 = { |
918 | - I(SrcMem | ByteOp, em_clflush), N, N, N, |
919 | + I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N, |
920 | }; |
921 | |
922 | static const struct group_dual group15 = { { |
923 | diff --git a/drivers/acpi/evged.c b/drivers/acpi/evged.c |
924 | index ccd900690b6f5..9df6991635c22 100644 |
925 | --- a/drivers/acpi/evged.c |
926 | +++ b/drivers/acpi/evged.c |
927 | @@ -101,7 +101,7 @@ static acpi_status acpi_ged_request_interrupt(struct acpi_resource *ares, |
928 | |
929 | switch (gsi) { |
930 | case 0 ... 255: |
931 | - sprintf(ev_name, "_%c%02hhX", |
932 | + sprintf(ev_name, "_%c%02X", |
933 | trigger == ACPI_EDGE_SENSITIVE ? 'E' : 'L', gsi); |
934 | |
935 | if (ACPI_SUCCESS(acpi_get_handle(handle, ev_name, &evt_handle))) |
936 | diff --git a/drivers/input/keyboard/sunkbd.c b/drivers/input/keyboard/sunkbd.c |
937 | index 27126e621eb60..d450f11b98a70 100644 |
938 | --- a/drivers/input/keyboard/sunkbd.c |
939 | +++ b/drivers/input/keyboard/sunkbd.c |
940 | @@ -99,7 +99,8 @@ static irqreturn_t sunkbd_interrupt(struct serio *serio, |
941 | switch (data) { |
942 | |
943 | case SUNKBD_RET_RESET: |
944 | - schedule_work(&sunkbd->tq); |
945 | + if (sunkbd->enabled) |
946 | + schedule_work(&sunkbd->tq); |
947 | sunkbd->reset = -1; |
948 | break; |
949 | |
950 | @@ -200,16 +201,12 @@ static int sunkbd_initialize(struct sunkbd *sunkbd) |
951 | } |
952 | |
953 | /* |
954 | - * sunkbd_reinit() sets leds and beeps to a state the computer remembers they |
955 | - * were in. |
956 | + * sunkbd_set_leds_beeps() sets leds and beeps to a state the computer remembers |
957 | + * they were in. |
958 | */ |
959 | |
960 | -static void sunkbd_reinit(struct work_struct *work) |
961 | +static void sunkbd_set_leds_beeps(struct sunkbd *sunkbd) |
962 | { |
963 | - struct sunkbd *sunkbd = container_of(work, struct sunkbd, tq); |
964 | - |
965 | - wait_event_interruptible_timeout(sunkbd->wait, sunkbd->reset >= 0, HZ); |
966 | - |
967 | serio_write(sunkbd->serio, SUNKBD_CMD_SETLED); |
968 | serio_write(sunkbd->serio, |
969 | (!!test_bit(LED_CAPSL, sunkbd->dev->led) << 3) | |
970 | @@ -222,11 +219,39 @@ static void sunkbd_reinit(struct work_struct *work) |
971 | SUNKBD_CMD_BELLOFF - !!test_bit(SND_BELL, sunkbd->dev->snd)); |
972 | } |
973 | |
974 | + |
975 | +/* |
976 | + * sunkbd_reinit() wait for the keyboard reset to complete and restores state |
977 | + * of leds and beeps. |
978 | + */ |
979 | + |
980 | +static void sunkbd_reinit(struct work_struct *work) |
981 | +{ |
982 | + struct sunkbd *sunkbd = container_of(work, struct sunkbd, tq); |
983 | + |
984 | + /* |
985 | + * It is OK that we check sunkbd->enabled without pausing serio, |
986 | + * as we only want to catch true->false transition that will |
987 | + * happen once and we will be woken up for it. |
988 | + */ |
989 | + wait_event_interruptible_timeout(sunkbd->wait, |
990 | + sunkbd->reset >= 0 || !sunkbd->enabled, |
991 | + HZ); |
992 | + |
993 | + if (sunkbd->reset >= 0 && sunkbd->enabled) |
994 | + sunkbd_set_leds_beeps(sunkbd); |
995 | +} |
996 | + |
997 | static void sunkbd_enable(struct sunkbd *sunkbd, bool enable) |
998 | { |
999 | serio_pause_rx(sunkbd->serio); |
1000 | sunkbd->enabled = enable; |
1001 | serio_continue_rx(sunkbd->serio); |
1002 | + |
1003 | + if (!enable) { |
1004 | + wake_up_interruptible(&sunkbd->wait); |
1005 | + cancel_work_sync(&sunkbd->tq); |
1006 | + } |
1007 | } |
1008 | |
1009 | /* |
1010 | diff --git a/drivers/net/ethernet/lantiq_xrx200.c b/drivers/net/ethernet/lantiq_xrx200.c |
1011 | index 96948276b2bc3..4e44a39267eb3 100644 |
1012 | --- a/drivers/net/ethernet/lantiq_xrx200.c |
1013 | +++ b/drivers/net/ethernet/lantiq_xrx200.c |
1014 | @@ -245,6 +245,7 @@ static int xrx200_tx_housekeeping(struct napi_struct *napi, int budget) |
1015 | int pkts = 0; |
1016 | int bytes = 0; |
1017 | |
1018 | + netif_tx_lock(net_dev); |
1019 | while (pkts < budget) { |
1020 | struct ltq_dma_desc *desc = &ch->dma.desc_base[ch->tx_free]; |
1021 | |
1022 | @@ -268,6 +269,7 @@ static int xrx200_tx_housekeeping(struct napi_struct *napi, int budget) |
1023 | net_dev->stats.tx_bytes += bytes; |
1024 | netdev_completed_queue(ch->priv->net_dev, pkts, bytes); |
1025 | |
1026 | + netif_tx_unlock(net_dev); |
1027 | if (netif_queue_stopped(net_dev)) |
1028 | netif_wake_queue(net_dev); |
1029 | |
1030 | diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c |
1031 | index 7089ffcc4e512..76547d35cd0e1 100644 |
1032 | --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c |
1033 | +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c |
1034 | @@ -853,11 +853,21 @@ static void cb_timeout_handler(struct work_struct *work) |
1035 | struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, |
1036 | cmd); |
1037 | |
1038 | + mlx5_cmd_eq_recover(dev); |
1039 | + |
1040 | + /* Maybe got handled by eq recover ? */ |
1041 | + if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) { |
1042 | + mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx, |
1043 | + mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); |
1044 | + goto out; /* phew, already handled */ |
1045 | + } |
1046 | + |
1047 | ent->ret = -ETIMEDOUT; |
1048 | - mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n", |
1049 | - mlx5_command_str(msg_to_opcode(ent->in)), |
1050 | - msg_to_opcode(ent->in)); |
1051 | + mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n", |
1052 | + ent->idx, mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); |
1053 | mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); |
1054 | + |
1055 | +out: |
1056 | cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */ |
1057 | } |
1058 | |
1059 | @@ -865,6 +875,33 @@ static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg); |
1060 | static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev, |
1061 | struct mlx5_cmd_msg *msg); |
1062 | |
1063 | +static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode) |
1064 | +{ |
1065 | + if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL) |
1066 | + return true; |
1067 | + |
1068 | + return cmd->allowed_opcode == opcode; |
1069 | +} |
1070 | + |
1071 | +static int cmd_alloc_index_retry(struct mlx5_cmd *cmd) |
1072 | +{ |
1073 | + unsigned long alloc_end = jiffies + msecs_to_jiffies(1000); |
1074 | + int idx; |
1075 | + |
1076 | +retry: |
1077 | + idx = cmd_alloc_index(cmd); |
1078 | + if (idx < 0 && time_before(jiffies, alloc_end)) { |
1079 | + /* Index allocation can fail on heavy load of commands. This is a temporary |
1080 | + * situation as the current command already holds the semaphore, meaning that |
1081 | + * another command completion is being handled and it is expected to release |
1082 | + * the entry index soon. |
1083 | + */ |
1084 | + cpu_relax(); |
1085 | + goto retry; |
1086 | + } |
1087 | + return idx; |
1088 | +} |
1089 | + |
1090 | static void cmd_work_handler(struct work_struct *work) |
1091 | { |
1092 | struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work); |
1093 | @@ -882,7 +919,7 @@ static void cmd_work_handler(struct work_struct *work) |
1094 | sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem; |
1095 | down(sem); |
1096 | if (!ent->page_queue) { |
1097 | - alloc_ret = cmd_alloc_index(cmd); |
1098 | + alloc_ret = cmd_alloc_index_retry(cmd); |
1099 | if (alloc_ret < 0) { |
1100 | mlx5_core_err(dev, "failed to allocate command entry\n"); |
1101 | if (ent->callback) { |
1102 | @@ -931,7 +968,8 @@ static void cmd_work_handler(struct work_struct *work) |
1103 | |
1104 | /* Skip sending command to fw if internal error */ |
1105 | if (pci_channel_offline(dev->pdev) || |
1106 | - dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { |
1107 | + dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR || |
1108 | + !opcode_allowed(&dev->cmd, ent->op)) { |
1109 | u8 status = 0; |
1110 | u32 drv_synd; |
1111 | |
1112 | @@ -987,6 +1025,35 @@ static const char *deliv_status_to_str(u8 status) |
1113 | } |
1114 | } |
1115 | |
1116 | +enum { |
1117 | + MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000, |
1118 | +}; |
1119 | + |
1120 | +static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev, |
1121 | + struct mlx5_cmd_work_ent *ent) |
1122 | +{ |
1123 | + unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC); |
1124 | + |
1125 | + mlx5_cmd_eq_recover(dev); |
1126 | + |
1127 | + /* Re-wait on the ent->done after executing the recovery flow. If the |
1128 | + * recovery flow (or any other recovery flow running simultaneously) |
1129 | + * has recovered an EQE, it should cause the entry to be completed by |
1130 | + * the command interface. |
1131 | + */ |
1132 | + if (wait_for_completion_timeout(&ent->done, timeout)) { |
1133 | + mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx, |
1134 | + mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); |
1135 | + return; |
1136 | + } |
1137 | + |
1138 | + mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx, |
1139 | + mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in)); |
1140 | + |
1141 | + ent->ret = -ETIMEDOUT; |
1142 | + mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); |
1143 | +} |
1144 | + |
1145 | static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) |
1146 | { |
1147 | unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC); |
1148 | @@ -998,12 +1065,10 @@ static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) |
1149 | ent->ret = -ECANCELED; |
1150 | goto out_err; |
1151 | } |
1152 | - if (cmd->mode == CMD_MODE_POLLING || ent->polling) { |
1153 | + if (cmd->mode == CMD_MODE_POLLING || ent->polling) |
1154 | wait_for_completion(&ent->done); |
1155 | - } else if (!wait_for_completion_timeout(&ent->done, timeout)) { |
1156 | - ent->ret = -ETIMEDOUT; |
1157 | - mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true); |
1158 | - } |
1159 | + else if (!wait_for_completion_timeout(&ent->done, timeout)) |
1160 | + wait_func_handle_exec_timeout(dev, ent); |
1161 | |
1162 | out_err: |
1163 | err = ent->ret; |
1164 | @@ -1422,6 +1487,22 @@ static void create_debugfs_files(struct mlx5_core_dev *dev) |
1165 | mlx5_cmdif_debugfs_init(dev); |
1166 | } |
1167 | |
1168 | +void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode) |
1169 | +{ |
1170 | + struct mlx5_cmd *cmd = &dev->cmd; |
1171 | + int i; |
1172 | + |
1173 | + for (i = 0; i < cmd->max_reg_cmds; i++) |
1174 | + down(&cmd->sem); |
1175 | + down(&cmd->pages_sem); |
1176 | + |
1177 | + cmd->allowed_opcode = opcode; |
1178 | + |
1179 | + up(&cmd->pages_sem); |
1180 | + for (i = 0; i < cmd->max_reg_cmds; i++) |
1181 | + up(&cmd->sem); |
1182 | +} |
1183 | + |
1184 | static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode) |
1185 | { |
1186 | struct mlx5_cmd *cmd = &dev->cmd; |
1187 | @@ -1714,12 +1795,13 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, |
1188 | int err; |
1189 | u8 status = 0; |
1190 | u32 drv_synd; |
1191 | + u16 opcode; |
1192 | u8 token; |
1193 | |
1194 | + opcode = MLX5_GET(mbox_in, in, opcode); |
1195 | if (pci_channel_offline(dev->pdev) || |
1196 | - dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { |
1197 | - u16 opcode = MLX5_GET(mbox_in, in, opcode); |
1198 | - |
1199 | + dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR || |
1200 | + !opcode_allowed(&dev->cmd, opcode)) { |
1201 | err = mlx5_internal_err_ret_value(dev, opcode, &drv_synd, &status); |
1202 | MLX5_SET(mbox_out, out, status, status); |
1203 | MLX5_SET(mbox_out, out, syndrome, drv_synd); |
1204 | @@ -2021,6 +2103,7 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev) |
1205 | mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma)); |
1206 | |
1207 | cmd->mode = CMD_MODE_POLLING; |
1208 | + cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL; |
1209 | |
1210 | create_msg_cache(dev); |
1211 | |
1212 | diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c |
1213 | index 580c71cb9dfaa..0a20938b4aadb 100644 |
1214 | --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c |
1215 | +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c |
1216 | @@ -190,6 +190,29 @@ u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq) |
1217 | return count_eqe; |
1218 | } |
1219 | |
1220 | +static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, unsigned long *flags) |
1221 | + __acquires(&eq->lock) |
1222 | +{ |
1223 | + if (in_irq()) |
1224 | + spin_lock(&eq->lock); |
1225 | + else |
1226 | + spin_lock_irqsave(&eq->lock, *flags); |
1227 | +} |
1228 | + |
1229 | +static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, unsigned long *flags) |
1230 | + __releases(&eq->lock) |
1231 | +{ |
1232 | + if (in_irq()) |
1233 | + spin_unlock(&eq->lock); |
1234 | + else |
1235 | + spin_unlock_irqrestore(&eq->lock, *flags); |
1236 | +} |
1237 | + |
1238 | +enum async_eq_nb_action { |
1239 | + ASYNC_EQ_IRQ_HANDLER = 0, |
1240 | + ASYNC_EQ_RECOVER = 1, |
1241 | +}; |
1242 | + |
1243 | static int mlx5_eq_async_int(struct notifier_block *nb, |
1244 | unsigned long action, void *data) |
1245 | { |
1246 | @@ -199,11 +222,14 @@ static int mlx5_eq_async_int(struct notifier_block *nb, |
1247 | struct mlx5_eq_table *eqt; |
1248 | struct mlx5_core_dev *dev; |
1249 | struct mlx5_eqe *eqe; |
1250 | + unsigned long flags; |
1251 | int num_eqes = 0; |
1252 | |
1253 | dev = eq->dev; |
1254 | eqt = dev->priv.eq_table; |
1255 | |
1256 | + mlx5_eq_async_int_lock(eq_async, &flags); |
1257 | + |
1258 | eqe = next_eqe_sw(eq); |
1259 | if (!eqe) |
1260 | goto out; |
1261 | @@ -224,8 +250,19 @@ static int mlx5_eq_async_int(struct notifier_block *nb, |
1262 | |
1263 | out: |
1264 | eq_update_ci(eq, 1); |
1265 | + mlx5_eq_async_int_unlock(eq_async, &flags); |
1266 | |
1267 | - return 0; |
1268 | + return unlikely(action == ASYNC_EQ_RECOVER) ? num_eqes : 0; |
1269 | +} |
1270 | + |
1271 | +void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev) |
1272 | +{ |
1273 | + struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq; |
1274 | + int eqes; |
1275 | + |
1276 | + eqes = mlx5_eq_async_int(&eq->irq_nb, ASYNC_EQ_RECOVER, NULL); |
1277 | + if (eqes) |
1278 | + mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes); |
1279 | } |
1280 | |
1281 | static void init_eq_buf(struct mlx5_eq *eq) |
1282 | @@ -563,6 +600,40 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) |
1283 | gather_user_async_events(dev, mask); |
1284 | } |
1285 | |
1286 | +static int |
1287 | +setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq, |
1288 | + struct mlx5_eq_param *param, const char *name) |
1289 | +{ |
1290 | + int err; |
1291 | + |
1292 | + eq->irq_nb.notifier_call = mlx5_eq_async_int; |
1293 | + spin_lock_init(&eq->lock); |
1294 | + |
1295 | + err = create_async_eq(dev, &eq->core, param); |
1296 | + if (err) { |
1297 | + mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err); |
1298 | + return err; |
1299 | + } |
1300 | + err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb); |
1301 | + if (err) { |
1302 | + mlx5_core_warn(dev, "failed to enable %s EQ %d\n", name, err); |
1303 | + destroy_async_eq(dev, &eq->core); |
1304 | + } |
1305 | + return err; |
1306 | +} |
1307 | + |
1308 | +static void cleanup_async_eq(struct mlx5_core_dev *dev, |
1309 | + struct mlx5_eq_async *eq, const char *name) |
1310 | +{ |
1311 | + int err; |
1312 | + |
1313 | + mlx5_eq_disable(dev, &eq->core, &eq->irq_nb); |
1314 | + err = destroy_async_eq(dev, &eq->core); |
1315 | + if (err) |
1316 | + mlx5_core_err(dev, "failed to destroy %s eq, err(%d)\n", |
1317 | + name, err); |
1318 | +} |
1319 | + |
1320 | static int create_async_eqs(struct mlx5_core_dev *dev) |
1321 | { |
1322 | struct mlx5_eq_table *table = dev->priv.eq_table; |
1323 | @@ -572,77 +643,48 @@ static int create_async_eqs(struct mlx5_core_dev *dev) |
1324 | MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR); |
1325 | mlx5_eq_notifier_register(dev, &table->cq_err_nb); |
1326 | |
1327 | - table->cmd_eq.irq_nb.notifier_call = mlx5_eq_async_int; |
1328 | param = (struct mlx5_eq_param) { |
1329 | .irq_index = 0, |
1330 | .nent = MLX5_NUM_CMD_EQE, |
1331 | + .mask[0] = 1ull << MLX5_EVENT_TYPE_CMD, |
1332 | }; |
1333 | - |
1334 | - param.mask[0] = 1ull << MLX5_EVENT_TYPE_CMD; |
1335 | - err = create_async_eq(dev, &table->cmd_eq.core, ¶m); |
1336 | - if (err) { |
1337 | - mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err); |
1338 | - goto err0; |
1339 | - } |
1340 | - err = mlx5_eq_enable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb); |
1341 | - if (err) { |
1342 | - mlx5_core_warn(dev, "failed to enable cmd EQ %d\n", err); |
1343 | + mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_CREATE_EQ); |
1344 | + err = setup_async_eq(dev, &table->cmd_eq, ¶m, "cmd"); |
1345 | + if (err) |
1346 | goto err1; |
1347 | - } |
1348 | + |
1349 | mlx5_cmd_use_events(dev); |
1350 | + mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); |
1351 | |
1352 | - table->async_eq.irq_nb.notifier_call = mlx5_eq_async_int; |
1353 | param = (struct mlx5_eq_param) { |
1354 | .irq_index = 0, |
1355 | .nent = MLX5_NUM_ASYNC_EQE, |
1356 | }; |
1357 | |
1358 | gather_async_events_mask(dev, param.mask); |
1359 | - err = create_async_eq(dev, &table->async_eq.core, ¶m); |
1360 | - if (err) { |
1361 | - mlx5_core_warn(dev, "failed to create async EQ %d\n", err); |
1362 | + err = setup_async_eq(dev, &table->async_eq, ¶m, "async"); |
1363 | + if (err) |
1364 | goto err2; |
1365 | - } |
1366 | - err = mlx5_eq_enable(dev, &table->async_eq.core, |
1367 | - &table->async_eq.irq_nb); |
1368 | - if (err) { |
1369 | - mlx5_core_warn(dev, "failed to enable async EQ %d\n", err); |
1370 | - goto err3; |
1371 | - } |
1372 | |
1373 | - table->pages_eq.irq_nb.notifier_call = mlx5_eq_async_int; |
1374 | param = (struct mlx5_eq_param) { |
1375 | .irq_index = 0, |
1376 | .nent = /* TODO: sriov max_vf + */ 1, |
1377 | + .mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST, |
1378 | }; |
1379 | |
1380 | - param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST; |
1381 | - err = create_async_eq(dev, &table->pages_eq.core, ¶m); |
1382 | - if (err) { |
1383 | - mlx5_core_warn(dev, "failed to create pages EQ %d\n", err); |
1384 | - goto err4; |
1385 | - } |
1386 | - err = mlx5_eq_enable(dev, &table->pages_eq.core, |
1387 | - &table->pages_eq.irq_nb); |
1388 | - if (err) { |
1389 | - mlx5_core_warn(dev, "failed to enable pages EQ %d\n", err); |
1390 | - goto err5; |
1391 | - } |
1392 | + err = setup_async_eq(dev, &table->pages_eq, ¶m, "pages"); |
1393 | + if (err) |
1394 | + goto err3; |
1395 | |
1396 | - return err; |
1397 | + return 0; |
1398 | |
1399 | -err5: |
1400 | - destroy_async_eq(dev, &table->pages_eq.core); |
1401 | -err4: |
1402 | - mlx5_eq_disable(dev, &table->async_eq.core, &table->async_eq.irq_nb); |
1403 | err3: |
1404 | - destroy_async_eq(dev, &table->async_eq.core); |
1405 | + cleanup_async_eq(dev, &table->async_eq, "async"); |
1406 | err2: |
1407 | mlx5_cmd_use_polling(dev); |
1408 | - mlx5_eq_disable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb); |
1409 | + cleanup_async_eq(dev, &table->cmd_eq, "cmd"); |
1410 | err1: |
1411 | - destroy_async_eq(dev, &table->cmd_eq.core); |
1412 | -err0: |
1413 | + mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); |
1414 | mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); |
1415 | return err; |
1416 | } |
1417 | @@ -650,28 +692,11 @@ err0: |
1418 | static void destroy_async_eqs(struct mlx5_core_dev *dev) |
1419 | { |
1420 | struct mlx5_eq_table *table = dev->priv.eq_table; |
1421 | - int err; |
1422 | - |
1423 | - mlx5_eq_disable(dev, &table->pages_eq.core, &table->pages_eq.irq_nb); |
1424 | - err = destroy_async_eq(dev, &table->pages_eq.core); |
1425 | - if (err) |
1426 | - mlx5_core_err(dev, "failed to destroy pages eq, err(%d)\n", |
1427 | - err); |
1428 | - |
1429 | - mlx5_eq_disable(dev, &table->async_eq.core, &table->async_eq.irq_nb); |
1430 | - err = destroy_async_eq(dev, &table->async_eq.core); |
1431 | - if (err) |
1432 | - mlx5_core_err(dev, "failed to destroy async eq, err(%d)\n", |
1433 | - err); |
1434 | |
1435 | + cleanup_async_eq(dev, &table->pages_eq, "pages"); |
1436 | + cleanup_async_eq(dev, &table->async_eq, "async"); |
1437 | mlx5_cmd_use_polling(dev); |
1438 | - |
1439 | - mlx5_eq_disable(dev, &table->cmd_eq.core, &table->cmd_eq.irq_nb); |
1440 | - err = destroy_async_eq(dev, &table->cmd_eq.core); |
1441 | - if (err) |
1442 | - mlx5_core_err(dev, "failed to destroy command eq, err(%d)\n", |
1443 | - err); |
1444 | - |
1445 | + cleanup_async_eq(dev, &table->cmd_eq, "cmd"); |
1446 | mlx5_eq_notifier_unregister(dev, &table->cq_err_nb); |
1447 | } |
1448 | |
1449 | diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h |
1450 | index 4be4d2d362189..9aaf0eab7c2e1 100644 |
1451 | --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h |
1452 | +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h |
1453 | @@ -38,6 +38,7 @@ struct mlx5_eq { |
1454 | struct mlx5_eq_async { |
1455 | struct mlx5_eq core; |
1456 | struct notifier_block irq_nb; |
1457 | + spinlock_t lock; /* To avoid irq EQ handle races with resiliency flows */ |
1458 | }; |
1459 | |
1460 | struct mlx5_eq_comp { |
1461 | @@ -82,6 +83,7 @@ void mlx5_cq_tasklet_cb(unsigned long data); |
1462 | struct cpumask *mlx5_eq_comp_cpumask(struct mlx5_core_dev *dev, int ix); |
1463 | |
1464 | u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq); |
1465 | +void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev); |
1466 | void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev); |
1467 | void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev); |
1468 | |
1469 | diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h |
1470 | index 6b4f86dfca382..2b65ffb3bd76e 100644 |
1471 | --- a/include/linux/mlx5/driver.h |
1472 | +++ b/include/linux/mlx5/driver.h |
1473 | @@ -299,6 +299,7 @@ struct mlx5_cmd { |
1474 | struct semaphore sem; |
1475 | struct semaphore pages_sem; |
1476 | int mode; |
1477 | + u16 allowed_opcode; |
1478 | struct mlx5_cmd_work_ent *ent_arr[MLX5_MAX_COMMANDS]; |
1479 | struct dma_pool *pool; |
1480 | struct mlx5_cmd_debug dbg; |
1481 | @@ -890,10 +891,15 @@ mlx5_frag_buf_get_idx_last_contig_stride(struct mlx5_frag_buf_ctrl *fbc, u32 ix) |
1482 | return min_t(u32, last_frag_stride_idx - fbc->strides_offset, fbc->sz_m1); |
1483 | } |
1484 | |
1485 | +enum { |
1486 | + CMD_ALLOWED_OPCODE_ALL, |
1487 | +}; |
1488 | + |
1489 | int mlx5_cmd_init(struct mlx5_core_dev *dev); |
1490 | void mlx5_cmd_cleanup(struct mlx5_core_dev *dev); |
1491 | void mlx5_cmd_use_events(struct mlx5_core_dev *dev); |
1492 | void mlx5_cmd_use_polling(struct mlx5_core_dev *dev); |
1493 | +void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode); |
1494 | |
1495 | struct mlx5_async_ctx { |
1496 | struct mlx5_core_dev *dev; |
1497 | diff --git a/net/can/proc.c b/net/can/proc.c |
1498 | index e6881bfc3ed11..077af42c26ba5 100644 |
1499 | --- a/net/can/proc.c |
1500 | +++ b/net/can/proc.c |
1501 | @@ -471,6 +471,9 @@ void can_init_proc(struct net *net) |
1502 | */ |
1503 | void can_remove_proc(struct net *net) |
1504 | { |
1505 | + if (!net->can.proc_dir) |
1506 | + return; |
1507 | + |
1508 | if (net->can.pde_version) |
1509 | remove_proc_entry(CAN_PROC_VERSION, net->can.proc_dir); |
1510 | |
1511 | @@ -498,6 +501,5 @@ void can_remove_proc(struct net *net) |
1512 | if (net->can.pde_rcvlist_sff) |
1513 | remove_proc_entry(CAN_PROC_RCVLIST_SFF, net->can.proc_dir); |
1514 | |
1515 | - if (net->can.proc_dir) |
1516 | - remove_proc_entry("can", net->proc_net); |
1517 | + remove_proc_entry("can", net->proc_net); |
1518 | } |
1519 | diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c |
1520 | index 4f14d8a06915a..38bb6d512b36d 100644 |
1521 | --- a/net/mac80211/sta_info.c |
1522 | +++ b/net/mac80211/sta_info.c |
1523 | @@ -244,6 +244,24 @@ struct sta_info *sta_info_get_by_idx(struct ieee80211_sub_if_data *sdata, |
1524 | */ |
1525 | void sta_info_free(struct ieee80211_local *local, struct sta_info *sta) |
1526 | { |
1527 | + /* |
1528 | + * If we had used sta_info_pre_move_state() then we might not |
1529 | + * have gone through the state transitions down again, so do |
1530 | + * it here now (and warn if it's inserted). |
1531 | + * |
1532 | + * This will clear state such as fast TX/RX that may have been |
1533 | + * allocated during state transitions. |
1534 | + */ |
1535 | + while (sta->sta_state > IEEE80211_STA_NONE) { |
1536 | + int ret; |
1537 | + |
1538 | + WARN_ON_ONCE(test_sta_flag(sta, WLAN_STA_INSERTED)); |
1539 | + |
1540 | + ret = sta_info_move_state(sta, sta->sta_state - 1); |
1541 | + if (WARN_ONCE(ret, "sta_info_move_state() returned %d\n", ret)) |
1542 | + break; |
1543 | + } |
1544 | + |
1545 | if (sta->rate_ctrl) |
1546 | rate_control_free_sta(sta); |
1547 | |
1548 | diff --git a/tools/testing/selftests/powerpc/security/.gitignore b/tools/testing/selftests/powerpc/security/.gitignore |
1549 | index 0b969fba3beb2..b8afb4f2481e4 100644 |
1550 | --- a/tools/testing/selftests/powerpc/security/.gitignore |
1551 | +++ b/tools/testing/selftests/powerpc/security/.gitignore |
1552 | @@ -1 +1,2 @@ |
1553 | rfi_flush |
1554 | +entry_flush |
1555 | diff --git a/tools/testing/selftests/powerpc/security/Makefile b/tools/testing/selftests/powerpc/security/Makefile |
1556 | index 85861c46b4457..e550a287768fc 100644 |
1557 | --- a/tools/testing/selftests/powerpc/security/Makefile |
1558 | +++ b/tools/testing/selftests/powerpc/security/Makefile |
1559 | @@ -1,6 +1,6 @@ |
1560 | # SPDX-License-Identifier: GPL-2.0+ |
1561 | |
1562 | -TEST_GEN_PROGS := rfi_flush |
1563 | +TEST_GEN_PROGS := rfi_flush entry_flush |
1564 | top_srcdir = ../../../../.. |
1565 | |
1566 | CFLAGS += -I../../../../../usr/include |
1567 | diff --git a/tools/testing/selftests/powerpc/security/entry_flush.c b/tools/testing/selftests/powerpc/security/entry_flush.c |
1568 | new file mode 100644 |
1569 | index 0000000000000..e8d24f9a5d3e5 |
1570 | --- /dev/null |
1571 | +++ b/tools/testing/selftests/powerpc/security/entry_flush.c |
1572 | @@ -0,0 +1,163 @@ |
1573 | +// SPDX-License-Identifier: GPL-2.0+ |
1574 | + |
1575 | +/* |
1576 | + * Copyright 2018 IBM Corporation. |
1577 | + */ |
1578 | + |
1579 | +#define __SANE_USERSPACE_TYPES__ |
1580 | + |
1581 | +#include <sys/types.h> |
1582 | +#include <stdint.h> |
1583 | +#include <malloc.h> |
1584 | +#include <unistd.h> |
1585 | +#include <signal.h> |
1586 | +#include <stdlib.h> |
1587 | +#include <string.h> |
1588 | +#include <stdio.h> |
1589 | +#include "utils.h" |
1590 | + |
1591 | +#define CACHELINE_SIZE 128 |
1592 | + |
1593 | +struct perf_event_read { |
1594 | + __u64 nr; |
1595 | + __u64 l1d_misses; |
1596 | +}; |
1597 | + |
1598 | +static inline __u64 load(void *addr) |
1599 | +{ |
1600 | + __u64 tmp; |
1601 | + |
1602 | + asm volatile("ld %0,0(%1)" : "=r"(tmp) : "b"(addr)); |
1603 | + |
1604 | + return tmp; |
1605 | +} |
1606 | + |
1607 | +static void syscall_loop(char *p, unsigned long iterations, |
1608 | + unsigned long zero_size) |
1609 | +{ |
1610 | + for (unsigned long i = 0; i < iterations; i++) { |
1611 | + for (unsigned long j = 0; j < zero_size; j += CACHELINE_SIZE) |
1612 | + load(p + j); |
1613 | + getppid(); |
1614 | + } |
1615 | +} |
1616 | + |
1617 | +int entry_flush_test(void) |
1618 | +{ |
1619 | + char *p; |
1620 | + int repetitions = 10; |
1621 | + int fd, passes = 0, iter, rc = 0; |
1622 | + struct perf_event_read v; |
1623 | + __u64 l1d_misses_total = 0; |
1624 | + unsigned long iterations = 100000, zero_size = 24 * 1024; |
1625 | + unsigned long l1d_misses_expected; |
1626 | + int rfi_flush_orig; |
1627 | + int entry_flush, entry_flush_orig; |
1628 | + |
1629 | + SKIP_IF(geteuid() != 0); |
1630 | + |
1631 | + // The PMU event we use only works on Power7 or later |
1632 | + SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06)); |
1633 | + |
1634 | + if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) { |
1635 | + perror("Unable to read powerpc/rfi_flush debugfs file"); |
1636 | + SKIP_IF(1); |
1637 | + } |
1638 | + |
1639 | + if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) { |
1640 | + perror("Unable to read powerpc/entry_flush debugfs file"); |
1641 | + SKIP_IF(1); |
1642 | + } |
1643 | + |
1644 | + if (rfi_flush_orig != 0) { |
1645 | + if (write_debugfs_file("powerpc/rfi_flush", 0) < 0) { |
1646 | + perror("error writing to powerpc/rfi_flush debugfs file"); |
1647 | + FAIL_IF(1); |
1648 | + } |
1649 | + } |
1650 | + |
1651 | + entry_flush = entry_flush_orig; |
1652 | + |
1653 | + fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); |
1654 | + FAIL_IF(fd < 0); |
1655 | + |
1656 | + p = (char *)memalign(zero_size, CACHELINE_SIZE); |
1657 | + |
1658 | + FAIL_IF(perf_event_enable(fd)); |
1659 | + |
1660 | + // disable L1 prefetching |
1661 | + set_dscr(1); |
1662 | + |
1663 | + iter = repetitions; |
1664 | + |
1665 | + /* |
1666 | + * We expect to see l1d miss for each cacheline access when entry_flush |
1667 | + * is set. Allow a small variation on this. |
1668 | + */ |
1669 | + l1d_misses_expected = iterations * (zero_size / CACHELINE_SIZE - 2); |
1670 | + |
1671 | +again: |
1672 | + FAIL_IF(perf_event_reset(fd)); |
1673 | + |
1674 | + syscall_loop(p, iterations, zero_size); |
1675 | + |
1676 | + FAIL_IF(read(fd, &v, sizeof(v)) != sizeof(v)); |
1677 | + |
1678 | + if (entry_flush && v.l1d_misses >= l1d_misses_expected) |
1679 | + passes++; |
1680 | + else if (!entry_flush && v.l1d_misses < (l1d_misses_expected / 2)) |
1681 | + passes++; |
1682 | + |
1683 | + l1d_misses_total += v.l1d_misses; |
1684 | + |
1685 | + while (--iter) |
1686 | + goto again; |
1687 | + |
1688 | + if (passes < repetitions) { |
1689 | + printf("FAIL (L1D misses with entry_flush=%d: %llu %c %lu) [%d/%d failures]\n", |
1690 | + entry_flush, l1d_misses_total, entry_flush ? '<' : '>', |
1691 | + entry_flush ? repetitions * l1d_misses_expected : |
1692 | + repetitions * l1d_misses_expected / 2, |
1693 | + repetitions - passes, repetitions); |
1694 | + rc = 1; |
1695 | + } else |
1696 | + printf("PASS (L1D misses with entry_flush=%d: %llu %c %lu) [%d/%d pass]\n", |
1697 | + entry_flush, l1d_misses_total, entry_flush ? '>' : '<', |
1698 | + entry_flush ? repetitions * l1d_misses_expected : |
1699 | + repetitions * l1d_misses_expected / 2, |
1700 | + passes, repetitions); |
1701 | + |
1702 | + if (entry_flush == entry_flush_orig) { |
1703 | + entry_flush = !entry_flush_orig; |
1704 | + if (write_debugfs_file("powerpc/entry_flush", entry_flush) < 0) { |
1705 | + perror("error writing to powerpc/entry_flush debugfs file"); |
1706 | + return 1; |
1707 | + } |
1708 | + iter = repetitions; |
1709 | + l1d_misses_total = 0; |
1710 | + passes = 0; |
1711 | + goto again; |
1712 | + } |
1713 | + |
1714 | + perf_event_disable(fd); |
1715 | + close(fd); |
1716 | + |
1717 | + set_dscr(0); |
1718 | + |
1719 | + if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) { |
1720 | + perror("unable to restore original value of powerpc/rfi_flush debugfs file"); |
1721 | + return 1; |
1722 | + } |
1723 | + |
1724 | + if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) { |
1725 | + perror("unable to restore original value of powerpc/entry_flush debugfs file"); |
1726 | + return 1; |
1727 | + } |
1728 | + |
1729 | + return rc; |
1730 | +} |
1731 | + |
1732 | +int main(int argc, char *argv[]) |
1733 | +{ |
1734 | + return test_harness(entry_flush_test, "entry_flush_test"); |
1735 | +} |
1736 | diff --git a/tools/testing/selftests/powerpc/security/rfi_flush.c b/tools/testing/selftests/powerpc/security/rfi_flush.c |
1737 | index 0a7d0afb26b88..533315e68133d 100644 |
1738 | --- a/tools/testing/selftests/powerpc/security/rfi_flush.c |
1739 | +++ b/tools/testing/selftests/powerpc/security/rfi_flush.c |
1740 | @@ -50,16 +50,30 @@ int rfi_flush_test(void) |
1741 | __u64 l1d_misses_total = 0; |
1742 | unsigned long iterations = 100000, zero_size = 24 * 1024; |
1743 | unsigned long l1d_misses_expected; |
1744 | - int rfi_flush_org, rfi_flush; |
1745 | + int rfi_flush_orig, rfi_flush; |
1746 | + int have_entry_flush, entry_flush_orig; |
1747 | |
1748 | SKIP_IF(geteuid() != 0); |
1749 | |
1750 | - if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_org)) { |
1751 | + if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) { |
1752 | perror("Unable to read powerpc/rfi_flush debugfs file"); |
1753 | SKIP_IF(1); |
1754 | } |
1755 | |
1756 | - rfi_flush = rfi_flush_org; |
1757 | + if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) { |
1758 | + have_entry_flush = 0; |
1759 | + } else { |
1760 | + have_entry_flush = 1; |
1761 | + |
1762 | + if (entry_flush_orig != 0) { |
1763 | + if (write_debugfs_file("powerpc/entry_flush", 0) < 0) { |
1764 | + perror("error writing to powerpc/entry_flush debugfs file"); |
1765 | + return 1; |
1766 | + } |
1767 | + } |
1768 | + } |
1769 | + |
1770 | + rfi_flush = rfi_flush_orig; |
1771 | |
1772 | fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); |
1773 | FAIL_IF(fd < 0); |
1774 | @@ -68,6 +82,7 @@ int rfi_flush_test(void) |
1775 | |
1776 | FAIL_IF(perf_event_enable(fd)); |
1777 | |
1778 | + // disable L1 prefetching |
1779 | set_dscr(1); |
1780 | |
1781 | iter = repetitions; |
1782 | @@ -109,8 +124,8 @@ again: |
1783 | repetitions * l1d_misses_expected / 2, |
1784 | passes, repetitions); |
1785 | |
1786 | - if (rfi_flush == rfi_flush_org) { |
1787 | - rfi_flush = !rfi_flush_org; |
1788 | + if (rfi_flush == rfi_flush_orig) { |
1789 | + rfi_flush = !rfi_flush_orig; |
1790 | if (write_debugfs_file("powerpc/rfi_flush", rfi_flush) < 0) { |
1791 | perror("error writing to powerpc/rfi_flush debugfs file"); |
1792 | return 1; |
1793 | @@ -126,11 +141,19 @@ again: |
1794 | |
1795 | set_dscr(0); |
1796 | |
1797 | - if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_org) < 0) { |
1798 | + if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) { |
1799 | perror("unable to restore original value of powerpc/rfi_flush debugfs file"); |
1800 | return 1; |
1801 | } |
1802 | |
1803 | + if (have_entry_flush) { |
1804 | + if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) { |
1805 | + perror("unable to restore original value of powerpc/entry_flush " |
1806 | + "debugfs file"); |
1807 | + return 1; |
1808 | + } |
1809 | + } |
1810 | + |
1811 | return rc; |
1812 | } |
1813 |