Contents of /trunk/kernel-magellan/patches-4.8/0109-4.8.10-all-fixes.patch
Parent Directory | Revision Log
Revision 2849 -
(show annotations)
(download)
Tue Nov 22 13:19:55 2016 UTC (7 years, 10 months ago) by niro
File size: 141674 byte(s)
Tue Nov 22 13:19:55 2016 UTC (7 years, 10 months ago) by niro
File size: 141674 byte(s)
-linux-4.8.10
1 | diff --git a/Makefile b/Makefile |
2 | index c1519ab85258..7cf2b4985703 100644 |
3 | --- a/Makefile |
4 | +++ b/Makefile |
5 | @@ -1,6 +1,6 @@ |
6 | VERSION = 4 |
7 | PATCHLEVEL = 8 |
8 | -SUBLEVEL = 9 |
9 | +SUBLEVEL = 10 |
10 | EXTRAVERSION = |
11 | NAME = Psychotic Stoned Sheep |
12 | |
13 | diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h |
14 | index 37a315d0ddd4..a6847fc05a6d 100644 |
15 | --- a/arch/sparc/include/asm/uaccess_64.h |
16 | +++ b/arch/sparc/include/asm/uaccess_64.h |
17 | @@ -98,7 +98,6 @@ struct exception_table_entry { |
18 | unsigned int insn, fixup; |
19 | }; |
20 | |
21 | -void __ret_efault(void); |
22 | void __retl_efault(void); |
23 | |
24 | /* Uh, these should become the main single-value transfer routines.. |
25 | @@ -205,55 +204,34 @@ int __get_user_bad(void); |
26 | unsigned long __must_check ___copy_from_user(void *to, |
27 | const void __user *from, |
28 | unsigned long size); |
29 | -unsigned long copy_from_user_fixup(void *to, const void __user *from, |
30 | - unsigned long size); |
31 | static inline unsigned long __must_check |
32 | copy_from_user(void *to, const void __user *from, unsigned long size) |
33 | { |
34 | - unsigned long ret; |
35 | - |
36 | check_object_size(to, size, false); |
37 | |
38 | - ret = ___copy_from_user(to, from, size); |
39 | - if (unlikely(ret)) |
40 | - ret = copy_from_user_fixup(to, from, size); |
41 | - |
42 | - return ret; |
43 | + return ___copy_from_user(to, from, size); |
44 | } |
45 | #define __copy_from_user copy_from_user |
46 | |
47 | unsigned long __must_check ___copy_to_user(void __user *to, |
48 | const void *from, |
49 | unsigned long size); |
50 | -unsigned long copy_to_user_fixup(void __user *to, const void *from, |
51 | - unsigned long size); |
52 | static inline unsigned long __must_check |
53 | copy_to_user(void __user *to, const void *from, unsigned long size) |
54 | { |
55 | - unsigned long ret; |
56 | - |
57 | check_object_size(from, size, true); |
58 | |
59 | - ret = ___copy_to_user(to, from, size); |
60 | - if (unlikely(ret)) |
61 | - ret = copy_to_user_fixup(to, from, size); |
62 | - return ret; |
63 | + return ___copy_to_user(to, from, size); |
64 | } |
65 | #define __copy_to_user copy_to_user |
66 | |
67 | unsigned long __must_check ___copy_in_user(void __user *to, |
68 | const void __user *from, |
69 | unsigned long size); |
70 | -unsigned long copy_in_user_fixup(void __user *to, void __user *from, |
71 | - unsigned long size); |
72 | static inline unsigned long __must_check |
73 | copy_in_user(void __user *to, void __user *from, unsigned long size) |
74 | { |
75 | - unsigned long ret = ___copy_in_user(to, from, size); |
76 | - |
77 | - if (unlikely(ret)) |
78 | - ret = copy_in_user_fixup(to, from, size); |
79 | - return ret; |
80 | + return ___copy_in_user(to, from, size); |
81 | } |
82 | #define __copy_in_user copy_in_user |
83 | |
84 | diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S |
85 | index a076b4249e62..5f1f3ae21657 100644 |
86 | --- a/arch/sparc/kernel/head_64.S |
87 | +++ b/arch/sparc/kernel/head_64.S |
88 | @@ -922,47 +922,11 @@ prom_tba: .xword 0 |
89 | tlb_type: .word 0 /* Must NOT end up in BSS */ |
90 | .section ".fixup",#alloc,#execinstr |
91 | |
92 | - .globl __ret_efault, __retl_efault, __ret_one, __retl_one |
93 | -ENTRY(__ret_efault) |
94 | - ret |
95 | - restore %g0, -EFAULT, %o0 |
96 | -ENDPROC(__ret_efault) |
97 | - |
98 | ENTRY(__retl_efault) |
99 | retl |
100 | mov -EFAULT, %o0 |
101 | ENDPROC(__retl_efault) |
102 | |
103 | -ENTRY(__retl_one) |
104 | - retl |
105 | - mov 1, %o0 |
106 | -ENDPROC(__retl_one) |
107 | - |
108 | -ENTRY(__retl_one_fp) |
109 | - VISExitHalf |
110 | - retl |
111 | - mov 1, %o0 |
112 | -ENDPROC(__retl_one_fp) |
113 | - |
114 | -ENTRY(__ret_one_asi) |
115 | - wr %g0, ASI_AIUS, %asi |
116 | - ret |
117 | - restore %g0, 1, %o0 |
118 | -ENDPROC(__ret_one_asi) |
119 | - |
120 | -ENTRY(__retl_one_asi) |
121 | - wr %g0, ASI_AIUS, %asi |
122 | - retl |
123 | - mov 1, %o0 |
124 | -ENDPROC(__retl_one_asi) |
125 | - |
126 | -ENTRY(__retl_one_asi_fp) |
127 | - wr %g0, ASI_AIUS, %asi |
128 | - VISExitHalf |
129 | - retl |
130 | - mov 1, %o0 |
131 | -ENDPROC(__retl_one_asi_fp) |
132 | - |
133 | ENTRY(__retl_o1) |
134 | retl |
135 | mov %o1, %o0 |
136 | diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c |
137 | index 59bbeff55024..07933b9e9ce0 100644 |
138 | --- a/arch/sparc/kernel/jump_label.c |
139 | +++ b/arch/sparc/kernel/jump_label.c |
140 | @@ -13,19 +13,30 @@ |
141 | void arch_jump_label_transform(struct jump_entry *entry, |
142 | enum jump_label_type type) |
143 | { |
144 | - u32 val; |
145 | u32 *insn = (u32 *) (unsigned long) entry->code; |
146 | + u32 val; |
147 | |
148 | if (type == JUMP_LABEL_JMP) { |
149 | s32 off = (s32)entry->target - (s32)entry->code; |
150 | + bool use_v9_branch = false; |
151 | + |
152 | + BUG_ON(off & 3); |
153 | |
154 | #ifdef CONFIG_SPARC64 |
155 | - /* ba,pt %xcc, . + (off << 2) */ |
156 | - val = 0x10680000 | ((u32) off >> 2); |
157 | -#else |
158 | - /* ba . + (off << 2) */ |
159 | - val = 0x10800000 | ((u32) off >> 2); |
160 | + if (off <= 0xfffff && off >= -0x100000) |
161 | + use_v9_branch = true; |
162 | #endif |
163 | + if (use_v9_branch) { |
164 | + /* WDISP19 - target is . + immed << 2 */ |
165 | + /* ba,pt %xcc, . + off */ |
166 | + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff); |
167 | + } else { |
168 | + /* WDISP22 - target is . + immed << 2 */ |
169 | + BUG_ON(off > 0x7fffff); |
170 | + BUG_ON(off < -0x800000); |
171 | + /* ba . + off */ |
172 | + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff); |
173 | + } |
174 | } else { |
175 | val = 0x01000000; |
176 | } |
177 | diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c |
178 | index 9e034f29dcc5..20ffb052fe38 100644 |
179 | --- a/arch/sparc/kernel/sparc_ksyms_64.c |
180 | +++ b/arch/sparc/kernel/sparc_ksyms_64.c |
181 | @@ -27,7 +27,6 @@ EXPORT_SYMBOL(__flushw_user); |
182 | EXPORT_SYMBOL_GPL(real_hard_smp_processor_id); |
183 | |
184 | /* from head_64.S */ |
185 | -EXPORT_SYMBOL(__ret_efault); |
186 | EXPORT_SYMBOL(tlb_type); |
187 | EXPORT_SYMBOL(sun4v_chip_type); |
188 | EXPORT_SYMBOL(prom_root_node); |
189 | diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S |
190 | index b7d0bd6b1406..69a439fa2fc1 100644 |
191 | --- a/arch/sparc/lib/GENcopy_from_user.S |
192 | +++ b/arch/sparc/lib/GENcopy_from_user.S |
193 | @@ -3,11 +3,11 @@ |
194 | * Copyright (C) 2007 David S. Miller (davem@davemloft.net) |
195 | */ |
196 | |
197 | -#define EX_LD(x) \ |
198 | +#define EX_LD(x,y) \ |
199 | 98: x; \ |
200 | .section __ex_table,"a";\ |
201 | .align 4; \ |
202 | - .word 98b, __retl_one; \ |
203 | + .word 98b, y; \ |
204 | .text; \ |
205 | .align 4; |
206 | |
207 | diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S |
208 | index 780550e1afc7..9947427ce354 100644 |
209 | --- a/arch/sparc/lib/GENcopy_to_user.S |
210 | +++ b/arch/sparc/lib/GENcopy_to_user.S |
211 | @@ -3,11 +3,11 @@ |
212 | * Copyright (C) 2007 David S. Miller (davem@davemloft.net) |
213 | */ |
214 | |
215 | -#define EX_ST(x) \ |
216 | +#define EX_ST(x,y) \ |
217 | 98: x; \ |
218 | .section __ex_table,"a";\ |
219 | .align 4; \ |
220 | - .word 98b, __retl_one; \ |
221 | + .word 98b, y; \ |
222 | .text; \ |
223 | .align 4; |
224 | |
225 | diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S |
226 | index 89358ee94851..059ea24ad73d 100644 |
227 | --- a/arch/sparc/lib/GENmemcpy.S |
228 | +++ b/arch/sparc/lib/GENmemcpy.S |
229 | @@ -4,21 +4,18 @@ |
230 | */ |
231 | |
232 | #ifdef __KERNEL__ |
233 | +#include <linux/linkage.h> |
234 | #define GLOBAL_SPARE %g7 |
235 | #else |
236 | #define GLOBAL_SPARE %g5 |
237 | #endif |
238 | |
239 | #ifndef EX_LD |
240 | -#define EX_LD(x) x |
241 | +#define EX_LD(x,y) x |
242 | #endif |
243 | |
244 | #ifndef EX_ST |
245 | -#define EX_ST(x) x |
246 | -#endif |
247 | - |
248 | -#ifndef EX_RETVAL |
249 | -#define EX_RETVAL(x) x |
250 | +#define EX_ST(x,y) x |
251 | #endif |
252 | |
253 | #ifndef LOAD |
254 | @@ -45,6 +42,29 @@ |
255 | .register %g3,#scratch |
256 | |
257 | .text |
258 | + |
259 | +#ifndef EX_RETVAL |
260 | +#define EX_RETVAL(x) x |
261 | +ENTRY(GEN_retl_o4_1) |
262 | + add %o4, %o2, %o4 |
263 | + retl |
264 | + add %o4, 1, %o0 |
265 | +ENDPROC(GEN_retl_o4_1) |
266 | +ENTRY(GEN_retl_g1_8) |
267 | + add %g1, %o2, %g1 |
268 | + retl |
269 | + add %g1, 8, %o0 |
270 | +ENDPROC(GEN_retl_g1_8) |
271 | +ENTRY(GEN_retl_o2_4) |
272 | + retl |
273 | + add %o2, 4, %o0 |
274 | +ENDPROC(GEN_retl_o2_4) |
275 | +ENTRY(GEN_retl_o2_1) |
276 | + retl |
277 | + add %o2, 1, %o0 |
278 | +ENDPROC(GEN_retl_o2_1) |
279 | +#endif |
280 | + |
281 | .align 64 |
282 | |
283 | .globl FUNC_NAME |
284 | @@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
285 | sub %g0, %o4, %o4 |
286 | sub %o2, %o4, %o2 |
287 | 1: subcc %o4, 1, %o4 |
288 | - EX_LD(LOAD(ldub, %o1, %g1)) |
289 | - EX_ST(STORE(stb, %g1, %o0)) |
290 | + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1) |
291 | + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1) |
292 | add %o1, 1, %o1 |
293 | bne,pt %XCC, 1b |
294 | add %o0, 1, %o0 |
295 | @@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
296 | andn %o2, 0x7, %g1 |
297 | sub %o2, %g1, %o2 |
298 | 1: subcc %g1, 0x8, %g1 |
299 | - EX_LD(LOAD(ldx, %o1, %g2)) |
300 | - EX_ST(STORE(stx, %g2, %o0)) |
301 | + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8) |
302 | + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8) |
303 | add %o1, 0x8, %o1 |
304 | bne,pt %XCC, 1b |
305 | add %o0, 0x8, %o0 |
306 | @@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
307 | |
308 | 1: |
309 | subcc %o2, 4, %o2 |
310 | - EX_LD(LOAD(lduw, %o1, %g1)) |
311 | - EX_ST(STORE(stw, %g1, %o1 + %o3)) |
312 | + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4) |
313 | + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4) |
314 | bgu,pt %XCC, 1b |
315 | add %o1, 4, %o1 |
316 | |
317 | @@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
318 | .align 32 |
319 | 90: |
320 | subcc %o2, 1, %o2 |
321 | - EX_LD(LOAD(ldub, %o1, %g1)) |
322 | - EX_ST(STORE(stb, %g1, %o1 + %o3)) |
323 | + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1) |
324 | + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1) |
325 | bgu,pt %XCC, 90b |
326 | add %o1, 1, %o1 |
327 | retl |
328 | diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile |
329 | index 3269b0234093..4f2384a4286a 100644 |
330 | --- a/arch/sparc/lib/Makefile |
331 | +++ b/arch/sparc/lib/Makefile |
332 | @@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o |
333 | lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o |
334 | lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o |
335 | |
336 | -lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o |
337 | +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o |
338 | lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o |
339 | |
340 | obj-$(CONFIG_SPARC64) += iomap.o |
341 | diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S |
342 | index d5242b8c4f94..b79a6998d87c 100644 |
343 | --- a/arch/sparc/lib/NG2copy_from_user.S |
344 | +++ b/arch/sparc/lib/NG2copy_from_user.S |
345 | @@ -3,19 +3,19 @@ |
346 | * Copyright (C) 2007 David S. Miller (davem@davemloft.net) |
347 | */ |
348 | |
349 | -#define EX_LD(x) \ |
350 | +#define EX_LD(x,y) \ |
351 | 98: x; \ |
352 | .section __ex_table,"a";\ |
353 | .align 4; \ |
354 | - .word 98b, __retl_one_asi;\ |
355 | + .word 98b, y; \ |
356 | .text; \ |
357 | .align 4; |
358 | |
359 | -#define EX_LD_FP(x) \ |
360 | +#define EX_LD_FP(x,y) \ |
361 | 98: x; \ |
362 | .section __ex_table,"a";\ |
363 | .align 4; \ |
364 | - .word 98b, __retl_one_asi_fp;\ |
365 | + .word 98b, y##_fp; \ |
366 | .text; \ |
367 | .align 4; |
368 | |
369 | diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S |
370 | index 4e962d993b10..dcec55f254ab 100644 |
371 | --- a/arch/sparc/lib/NG2copy_to_user.S |
372 | +++ b/arch/sparc/lib/NG2copy_to_user.S |
373 | @@ -3,19 +3,19 @@ |
374 | * Copyright (C) 2007 David S. Miller (davem@davemloft.net) |
375 | */ |
376 | |
377 | -#define EX_ST(x) \ |
378 | +#define EX_ST(x,y) \ |
379 | 98: x; \ |
380 | .section __ex_table,"a";\ |
381 | .align 4; \ |
382 | - .word 98b, __retl_one_asi;\ |
383 | + .word 98b, y; \ |
384 | .text; \ |
385 | .align 4; |
386 | |
387 | -#define EX_ST_FP(x) \ |
388 | +#define EX_ST_FP(x,y) \ |
389 | 98: x; \ |
390 | .section __ex_table,"a";\ |
391 | .align 4; \ |
392 | - .word 98b, __retl_one_asi_fp;\ |
393 | + .word 98b, y##_fp; \ |
394 | .text; \ |
395 | .align 4; |
396 | |
397 | diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S |
398 | index d5f585df2f3f..c629dbd121b6 100644 |
399 | --- a/arch/sparc/lib/NG2memcpy.S |
400 | +++ b/arch/sparc/lib/NG2memcpy.S |
401 | @@ -4,6 +4,7 @@ |
402 | */ |
403 | |
404 | #ifdef __KERNEL__ |
405 | +#include <linux/linkage.h> |
406 | #include <asm/visasm.h> |
407 | #include <asm/asi.h> |
408 | #define GLOBAL_SPARE %g7 |
409 | @@ -32,21 +33,17 @@ |
410 | #endif |
411 | |
412 | #ifndef EX_LD |
413 | -#define EX_LD(x) x |
414 | +#define EX_LD(x,y) x |
415 | #endif |
416 | #ifndef EX_LD_FP |
417 | -#define EX_LD_FP(x) x |
418 | +#define EX_LD_FP(x,y) x |
419 | #endif |
420 | |
421 | #ifndef EX_ST |
422 | -#define EX_ST(x) x |
423 | +#define EX_ST(x,y) x |
424 | #endif |
425 | #ifndef EX_ST_FP |
426 | -#define EX_ST_FP(x) x |
427 | -#endif |
428 | - |
429 | -#ifndef EX_RETVAL |
430 | -#define EX_RETVAL(x) x |
431 | +#define EX_ST_FP(x,y) x |
432 | #endif |
433 | |
434 | #ifndef LOAD |
435 | @@ -140,45 +137,110 @@ |
436 | fsrc2 %x6, %f12; \ |
437 | fsrc2 %x7, %f14; |
438 | #define FREG_LOAD_1(base, x0) \ |
439 | - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)) |
440 | + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1) |
441 | #define FREG_LOAD_2(base, x0, x1) \ |
442 | - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
443 | - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); |
444 | + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ |
445 | + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); |
446 | #define FREG_LOAD_3(base, x0, x1, x2) \ |
447 | - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
448 | - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ |
449 | - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); |
450 | + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ |
451 | + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ |
452 | + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); |
453 | #define FREG_LOAD_4(base, x0, x1, x2, x3) \ |
454 | - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
455 | - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ |
456 | - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ |
457 | - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); |
458 | + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ |
459 | + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ |
460 | + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ |
461 | + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); |
462 | #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ |
463 | - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
464 | - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ |
465 | - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ |
466 | - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ |
467 | - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); |
468 | + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ |
469 | + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ |
470 | + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ |
471 | + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ |
472 | + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); |
473 | #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ |
474 | - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
475 | - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ |
476 | - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ |
477 | - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ |
478 | - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ |
479 | - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); |
480 | + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ |
481 | + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ |
482 | + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ |
483 | + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ |
484 | + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ |
485 | + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); |
486 | #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ |
487 | - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
488 | - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ |
489 | - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ |
490 | - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ |
491 | - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ |
492 | - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \ |
493 | - EX_LD_FP(LOAD(ldd, base + 0x30, %x6)); |
494 | + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ |
495 | + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ |
496 | + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ |
497 | + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ |
498 | + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ |
499 | + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \ |
500 | + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1); |
501 | |
502 | .register %g2,#scratch |
503 | .register %g3,#scratch |
504 | |
505 | .text |
506 | +#ifndef EX_RETVAL |
507 | +#define EX_RETVAL(x) x |
508 | +__restore_fp: |
509 | + VISExitHalf |
510 | +__restore_asi: |
511 | + retl |
512 | + wr %g0, ASI_AIUS, %asi |
513 | +ENTRY(NG2_retl_o2) |
514 | + ba,pt %xcc, __restore_asi |
515 | + mov %o2, %o0 |
516 | +ENDPROC(NG2_retl_o2) |
517 | +ENTRY(NG2_retl_o2_plus_1) |
518 | + ba,pt %xcc, __restore_asi |
519 | + add %o2, 1, %o0 |
520 | +ENDPROC(NG2_retl_o2_plus_1) |
521 | +ENTRY(NG2_retl_o2_plus_4) |
522 | + ba,pt %xcc, __restore_asi |
523 | + add %o2, 4, %o0 |
524 | +ENDPROC(NG2_retl_o2_plus_4) |
525 | +ENTRY(NG2_retl_o2_plus_8) |
526 | + ba,pt %xcc, __restore_asi |
527 | + add %o2, 8, %o0 |
528 | +ENDPROC(NG2_retl_o2_plus_8) |
529 | +ENTRY(NG2_retl_o2_plus_o4_plus_1) |
530 | + add %o4, 1, %o4 |
531 | + ba,pt %xcc, __restore_asi |
532 | + add %o2, %o4, %o0 |
533 | +ENDPROC(NG2_retl_o2_plus_o4_plus_1) |
534 | +ENTRY(NG2_retl_o2_plus_o4_plus_8) |
535 | + add %o4, 8, %o4 |
536 | + ba,pt %xcc, __restore_asi |
537 | + add %o2, %o4, %o0 |
538 | +ENDPROC(NG2_retl_o2_plus_o4_plus_8) |
539 | +ENTRY(NG2_retl_o2_plus_o4_plus_16) |
540 | + add %o4, 16, %o4 |
541 | + ba,pt %xcc, __restore_asi |
542 | + add %o2, %o4, %o0 |
543 | +ENDPROC(NG2_retl_o2_plus_o4_plus_16) |
544 | +ENTRY(NG2_retl_o2_plus_g1_fp) |
545 | + ba,pt %xcc, __restore_fp |
546 | + add %o2, %g1, %o0 |
547 | +ENDPROC(NG2_retl_o2_plus_g1_fp) |
548 | +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp) |
549 | + add %g1, 64, %g1 |
550 | + ba,pt %xcc, __restore_fp |
551 | + add %o2, %g1, %o0 |
552 | +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp) |
553 | +ENTRY(NG2_retl_o2_plus_g1_plus_1) |
554 | + add %g1, 1, %g1 |
555 | + ba,pt %xcc, __restore_asi |
556 | + add %o2, %g1, %o0 |
557 | +ENDPROC(NG2_retl_o2_plus_g1_plus_1) |
558 | +ENTRY(NG2_retl_o2_and_7_plus_o4) |
559 | + and %o2, 7, %o2 |
560 | + ba,pt %xcc, __restore_asi |
561 | + add %o2, %o4, %o0 |
562 | +ENDPROC(NG2_retl_o2_and_7_plus_o4) |
563 | +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8) |
564 | + and %o2, 7, %o2 |
565 | + add %o4, 8, %o4 |
566 | + ba,pt %xcc, __restore_asi |
567 | + add %o2, %o4, %o0 |
568 | +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8) |
569 | +#endif |
570 | + |
571 | .align 64 |
572 | |
573 | .globl FUNC_NAME |
574 | @@ -230,8 +292,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
575 | sub %g0, %o4, %o4 ! bytes to align dst |
576 | sub %o2, %o4, %o2 |
577 | 1: subcc %o4, 1, %o4 |
578 | - EX_LD(LOAD(ldub, %o1, %g1)) |
579 | - EX_ST(STORE(stb, %g1, %o0)) |
580 | + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1) |
581 | + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1) |
582 | add %o1, 1, %o1 |
583 | bne,pt %XCC, 1b |
584 | add %o0, 1, %o0 |
585 | @@ -281,11 +343,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
586 | nop |
587 | /* fall through for 0 < low bits < 8 */ |
588 | 110: sub %o4, 64, %g2 |
589 | - EX_LD_FP(LOAD_BLK(%g2, %f0)) |
590 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
591 | - EX_LD_FP(LOAD_BLK(%o4, %f16)) |
592 | + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1) |
593 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
594 | + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) |
595 | FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) |
596 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
597 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) |
598 | FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) |
599 | subcc %g1, 64, %g1 |
600 | add %o4, 64, %o4 |
601 | @@ -296,10 +358,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
602 | |
603 | 120: sub %o4, 56, %g2 |
604 | FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) |
605 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
606 | - EX_LD_FP(LOAD_BLK(%o4, %f16)) |
607 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
608 | + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) |
609 | FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) |
610 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
611 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) |
612 | FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) |
613 | subcc %g1, 64, %g1 |
614 | add %o4, 64, %o4 |
615 | @@ -310,10 +372,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
616 | |
617 | 130: sub %o4, 48, %g2 |
618 | FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) |
619 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
620 | - EX_LD_FP(LOAD_BLK(%o4, %f16)) |
621 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
622 | + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) |
623 | FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) |
624 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
625 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) |
626 | FREG_MOVE_6(f20, f22, f24, f26, f28, f30) |
627 | subcc %g1, 64, %g1 |
628 | add %o4, 64, %o4 |
629 | @@ -324,10 +386,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
630 | |
631 | 140: sub %o4, 40, %g2 |
632 | FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) |
633 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
634 | - EX_LD_FP(LOAD_BLK(%o4, %f16)) |
635 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
636 | + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) |
637 | FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) |
638 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
639 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) |
640 | FREG_MOVE_5(f22, f24, f26, f28, f30) |
641 | subcc %g1, 64, %g1 |
642 | add %o4, 64, %o4 |
643 | @@ -338,10 +400,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
644 | |
645 | 150: sub %o4, 32, %g2 |
646 | FREG_LOAD_4(%g2, f0, f2, f4, f6) |
647 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
648 | - EX_LD_FP(LOAD_BLK(%o4, %f16)) |
649 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
650 | + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) |
651 | FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) |
652 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
653 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) |
654 | FREG_MOVE_4(f24, f26, f28, f30) |
655 | subcc %g1, 64, %g1 |
656 | add %o4, 64, %o4 |
657 | @@ -352,10 +414,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
658 | |
659 | 160: sub %o4, 24, %g2 |
660 | FREG_LOAD_3(%g2, f0, f2, f4) |
661 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
662 | - EX_LD_FP(LOAD_BLK(%o4, %f16)) |
663 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
664 | + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) |
665 | FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) |
666 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
667 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) |
668 | FREG_MOVE_3(f26, f28, f30) |
669 | subcc %g1, 64, %g1 |
670 | add %o4, 64, %o4 |
671 | @@ -366,10 +428,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
672 | |
673 | 170: sub %o4, 16, %g2 |
674 | FREG_LOAD_2(%g2, f0, f2) |
675 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
676 | - EX_LD_FP(LOAD_BLK(%o4, %f16)) |
677 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
678 | + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) |
679 | FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) |
680 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
681 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) |
682 | FREG_MOVE_2(f28, f30) |
683 | subcc %g1, 64, %g1 |
684 | add %o4, 64, %o4 |
685 | @@ -380,10 +442,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
686 | |
687 | 180: sub %o4, 8, %g2 |
688 | FREG_LOAD_1(%g2, f0) |
689 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
690 | - EX_LD_FP(LOAD_BLK(%o4, %f16)) |
691 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
692 | + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) |
693 | FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) |
694 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
695 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) |
696 | FREG_MOVE_1(f30) |
697 | subcc %g1, 64, %g1 |
698 | add %o4, 64, %o4 |
699 | @@ -393,10 +455,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
700 | nop |
701 | |
702 | 190: |
703 | -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
704 | +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) |
705 | subcc %g1, 64, %g1 |
706 | - EX_LD_FP(LOAD_BLK(%o4, %f0)) |
707 | - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
708 | + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64) |
709 | + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64) |
710 | add %o4, 64, %o4 |
711 | bne,pt %xcc, 1b |
712 | LOAD(prefetch, %o4 + 64, #one_read) |
713 | @@ -423,28 +485,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
714 | andn %o2, 0xf, %o4 |
715 | and %o2, 0xf, %o2 |
716 | 1: subcc %o4, 0x10, %o4 |
717 | - EX_LD(LOAD(ldx, %o1, %o5)) |
718 | + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16) |
719 | add %o1, 0x08, %o1 |
720 | - EX_LD(LOAD(ldx, %o1, %g1)) |
721 | + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16) |
722 | sub %o1, 0x08, %o1 |
723 | - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) |
724 | + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16) |
725 | add %o1, 0x8, %o1 |
726 | - EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE)) |
727 | + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8) |
728 | bgu,pt %XCC, 1b |
729 | add %o1, 0x8, %o1 |
730 | 73: andcc %o2, 0x8, %g0 |
731 | be,pt %XCC, 1f |
732 | nop |
733 | sub %o2, 0x8, %o2 |
734 | - EX_LD(LOAD(ldx, %o1, %o5)) |
735 | - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) |
736 | + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8) |
737 | + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8) |
738 | add %o1, 0x8, %o1 |
739 | 1: andcc %o2, 0x4, %g0 |
740 | be,pt %XCC, 1f |
741 | nop |
742 | sub %o2, 0x4, %o2 |
743 | - EX_LD(LOAD(lduw, %o1, %o5)) |
744 | - EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE)) |
745 | + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4) |
746 | + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) |
747 | add %o1, 0x4, %o1 |
748 | 1: cmp %o2, 0 |
749 | be,pt %XCC, 85f |
750 | @@ -460,8 +522,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
751 | sub %o2, %g1, %o2 |
752 | |
753 | 1: subcc %g1, 1, %g1 |
754 | - EX_LD(LOAD(ldub, %o1, %o5)) |
755 | - EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE)) |
756 | + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1) |
757 | + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1) |
758 | bgu,pt %icc, 1b |
759 | add %o1, 1, %o1 |
760 | |
761 | @@ -477,16 +539,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
762 | |
763 | 8: mov 64, GLOBAL_SPARE |
764 | andn %o1, 0x7, %o1 |
765 | - EX_LD(LOAD(ldx, %o1, %g2)) |
766 | + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2) |
767 | sub GLOBAL_SPARE, %g1, GLOBAL_SPARE |
768 | andn %o2, 0x7, %o4 |
769 | sllx %g2, %g1, %g2 |
770 | 1: add %o1, 0x8, %o1 |
771 | - EX_LD(LOAD(ldx, %o1, %g3)) |
772 | + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4) |
773 | subcc %o4, 0x8, %o4 |
774 | srlx %g3, GLOBAL_SPARE, %o5 |
775 | or %o5, %g2, %o5 |
776 | - EX_ST(STORE(stx, %o5, %o0)) |
777 | + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8) |
778 | add %o0, 0x8, %o0 |
779 | bgu,pt %icc, 1b |
780 | sllx %g3, %g1, %g2 |
781 | @@ -506,8 +568,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
782 | |
783 | 1: |
784 | subcc %o2, 4, %o2 |
785 | - EX_LD(LOAD(lduw, %o1, %g1)) |
786 | - EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE)) |
787 | + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4) |
788 | + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) |
789 | bgu,pt %XCC, 1b |
790 | add %o1, 4, %o1 |
791 | |
792 | @@ -517,8 +579,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
793 | .align 32 |
794 | 90: |
795 | subcc %o2, 1, %o2 |
796 | - EX_LD(LOAD(ldub, %o1, %g1)) |
797 | - EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE)) |
798 | + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1) |
799 | + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1) |
800 | bgu,pt %XCC, 90b |
801 | add %o1, 1, %o1 |
802 | retl |
803 | diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S |
804 | index 2e8ee7ad07a9..16a286c1a528 100644 |
805 | --- a/arch/sparc/lib/NG4copy_from_user.S |
806 | +++ b/arch/sparc/lib/NG4copy_from_user.S |
807 | @@ -3,19 +3,19 @@ |
808 | * Copyright (C) 2012 David S. Miller (davem@davemloft.net) |
809 | */ |
810 | |
811 | -#define EX_LD(x) \ |
812 | +#define EX_LD(x, y) \ |
813 | 98: x; \ |
814 | .section __ex_table,"a";\ |
815 | .align 4; \ |
816 | - .word 98b, __retl_one_asi;\ |
817 | + .word 98b, y; \ |
818 | .text; \ |
819 | .align 4; |
820 | |
821 | -#define EX_LD_FP(x) \ |
822 | +#define EX_LD_FP(x,y) \ |
823 | 98: x; \ |
824 | .section __ex_table,"a";\ |
825 | .align 4; \ |
826 | - .word 98b, __retl_one_asi_fp;\ |
827 | + .word 98b, y##_fp; \ |
828 | .text; \ |
829 | .align 4; |
830 | |
831 | diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S |
832 | index be0bf4590df8..6b0276ffc858 100644 |
833 | --- a/arch/sparc/lib/NG4copy_to_user.S |
834 | +++ b/arch/sparc/lib/NG4copy_to_user.S |
835 | @@ -3,19 +3,19 @@ |
836 | * Copyright (C) 2012 David S. Miller (davem@davemloft.net) |
837 | */ |
838 | |
839 | -#define EX_ST(x) \ |
840 | +#define EX_ST(x,y) \ |
841 | 98: x; \ |
842 | .section __ex_table,"a";\ |
843 | .align 4; \ |
844 | - .word 98b, __retl_one_asi;\ |
845 | + .word 98b, y; \ |
846 | .text; \ |
847 | .align 4; |
848 | |
849 | -#define EX_ST_FP(x) \ |
850 | +#define EX_ST_FP(x,y) \ |
851 | 98: x; \ |
852 | .section __ex_table,"a";\ |
853 | .align 4; \ |
854 | - .word 98b, __retl_one_asi_fp;\ |
855 | + .word 98b, y##_fp; \ |
856 | .text; \ |
857 | .align 4; |
858 | |
859 | diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S |
860 | index 8e13ee1f4454..75bb93b1437f 100644 |
861 | --- a/arch/sparc/lib/NG4memcpy.S |
862 | +++ b/arch/sparc/lib/NG4memcpy.S |
863 | @@ -4,6 +4,7 @@ |
864 | */ |
865 | |
866 | #ifdef __KERNEL__ |
867 | +#include <linux/linkage.h> |
868 | #include <asm/visasm.h> |
869 | #include <asm/asi.h> |
870 | #define GLOBAL_SPARE %g7 |
871 | @@ -46,22 +47,19 @@ |
872 | #endif |
873 | |
874 | #ifndef EX_LD |
875 | -#define EX_LD(x) x |
876 | +#define EX_LD(x,y) x |
877 | #endif |
878 | #ifndef EX_LD_FP |
879 | -#define EX_LD_FP(x) x |
880 | +#define EX_LD_FP(x,y) x |
881 | #endif |
882 | |
883 | #ifndef EX_ST |
884 | -#define EX_ST(x) x |
885 | +#define EX_ST(x,y) x |
886 | #endif |
887 | #ifndef EX_ST_FP |
888 | -#define EX_ST_FP(x) x |
889 | +#define EX_ST_FP(x,y) x |
890 | #endif |
891 | |
892 | -#ifndef EX_RETVAL |
893 | -#define EX_RETVAL(x) x |
894 | -#endif |
895 | |
896 | #ifndef LOAD |
897 | #define LOAD(type,addr,dest) type [addr], dest |
898 | @@ -94,6 +92,158 @@ |
899 | .register %g3,#scratch |
900 | |
901 | .text |
902 | +#ifndef EX_RETVAL |
903 | +#define EX_RETVAL(x) x |
904 | +__restore_asi_fp: |
905 | + VISExitHalf |
906 | +__restore_asi: |
907 | + retl |
908 | + wr %g0, ASI_AIUS, %asi |
909 | + |
910 | +ENTRY(NG4_retl_o2) |
911 | + ba,pt %xcc, __restore_asi |
912 | + mov %o2, %o0 |
913 | +ENDPROC(NG4_retl_o2) |
914 | +ENTRY(NG4_retl_o2_plus_1) |
915 | + ba,pt %xcc, __restore_asi |
916 | + add %o2, 1, %o0 |
917 | +ENDPROC(NG4_retl_o2_plus_1) |
918 | +ENTRY(NG4_retl_o2_plus_4) |
919 | + ba,pt %xcc, __restore_asi |
920 | + add %o2, 4, %o0 |
921 | +ENDPROC(NG4_retl_o2_plus_4) |
922 | +ENTRY(NG4_retl_o2_plus_o5) |
923 | + ba,pt %xcc, __restore_asi |
924 | + add %o2, %o5, %o0 |
925 | +ENDPROC(NG4_retl_o2_plus_o5) |
926 | +ENTRY(NG4_retl_o2_plus_o5_plus_4) |
927 | + add %o5, 4, %o5 |
928 | + ba,pt %xcc, __restore_asi |
929 | + add %o2, %o5, %o0 |
930 | +ENDPROC(NG4_retl_o2_plus_o5_plus_4) |
931 | +ENTRY(NG4_retl_o2_plus_o5_plus_8) |
932 | + add %o5, 8, %o5 |
933 | + ba,pt %xcc, __restore_asi |
934 | + add %o2, %o5, %o0 |
935 | +ENDPROC(NG4_retl_o2_plus_o5_plus_8) |
936 | +ENTRY(NG4_retl_o2_plus_o5_plus_16) |
937 | + add %o5, 16, %o5 |
938 | + ba,pt %xcc, __restore_asi |
939 | + add %o2, %o5, %o0 |
940 | +ENDPROC(NG4_retl_o2_plus_o5_plus_16) |
941 | +ENTRY(NG4_retl_o2_plus_o5_plus_24) |
942 | + add %o5, 24, %o5 |
943 | + ba,pt %xcc, __restore_asi |
944 | + add %o2, %o5, %o0 |
945 | +ENDPROC(NG4_retl_o2_plus_o5_plus_24) |
946 | +ENTRY(NG4_retl_o2_plus_o5_plus_32) |
947 | + add %o5, 32, %o5 |
948 | + ba,pt %xcc, __restore_asi |
949 | + add %o2, %o5, %o0 |
950 | +ENDPROC(NG4_retl_o2_plus_o5_plus_32) |
951 | +ENTRY(NG4_retl_o2_plus_g1) |
952 | + ba,pt %xcc, __restore_asi |
953 | + add %o2, %g1, %o0 |
954 | +ENDPROC(NG4_retl_o2_plus_g1) |
955 | +ENTRY(NG4_retl_o2_plus_g1_plus_1) |
956 | + add %g1, 1, %g1 |
957 | + ba,pt %xcc, __restore_asi |
958 | + add %o2, %g1, %o0 |
959 | +ENDPROC(NG4_retl_o2_plus_g1_plus_1) |
960 | +ENTRY(NG4_retl_o2_plus_g1_plus_8) |
961 | + add %g1, 8, %g1 |
962 | + ba,pt %xcc, __restore_asi |
963 | + add %o2, %g1, %o0 |
964 | +ENDPROC(NG4_retl_o2_plus_g1_plus_8) |
965 | +ENTRY(NG4_retl_o2_plus_o4) |
966 | + ba,pt %xcc, __restore_asi |
967 | + add %o2, %o4, %o0 |
968 | +ENDPROC(NG4_retl_o2_plus_o4) |
969 | +ENTRY(NG4_retl_o2_plus_o4_plus_8) |
970 | + add %o4, 8, %o4 |
971 | + ba,pt %xcc, __restore_asi |
972 | + add %o2, %o4, %o0 |
973 | +ENDPROC(NG4_retl_o2_plus_o4_plus_8) |
974 | +ENTRY(NG4_retl_o2_plus_o4_plus_16) |
975 | + add %o4, 16, %o4 |
976 | + ba,pt %xcc, __restore_asi |
977 | + add %o2, %o4, %o0 |
978 | +ENDPROC(NG4_retl_o2_plus_o4_plus_16) |
979 | +ENTRY(NG4_retl_o2_plus_o4_plus_24) |
980 | + add %o4, 24, %o4 |
981 | + ba,pt %xcc, __restore_asi |
982 | + add %o2, %o4, %o0 |
983 | +ENDPROC(NG4_retl_o2_plus_o4_plus_24) |
984 | +ENTRY(NG4_retl_o2_plus_o4_plus_32) |
985 | + add %o4, 32, %o4 |
986 | + ba,pt %xcc, __restore_asi |
987 | + add %o2, %o4, %o0 |
988 | +ENDPROC(NG4_retl_o2_plus_o4_plus_32) |
989 | +ENTRY(NG4_retl_o2_plus_o4_plus_40) |
990 | + add %o4, 40, %o4 |
991 | + ba,pt %xcc, __restore_asi |
992 | + add %o2, %o4, %o0 |
993 | +ENDPROC(NG4_retl_o2_plus_o4_plus_40) |
994 | +ENTRY(NG4_retl_o2_plus_o4_plus_48) |
995 | + add %o4, 48, %o4 |
996 | + ba,pt %xcc, __restore_asi |
997 | + add %o2, %o4, %o0 |
998 | +ENDPROC(NG4_retl_o2_plus_o4_plus_48) |
999 | +ENTRY(NG4_retl_o2_plus_o4_plus_56) |
1000 | + add %o4, 56, %o4 |
1001 | + ba,pt %xcc, __restore_asi |
1002 | + add %o2, %o4, %o0 |
1003 | +ENDPROC(NG4_retl_o2_plus_o4_plus_56) |
1004 | +ENTRY(NG4_retl_o2_plus_o4_plus_64) |
1005 | + add %o4, 64, %o4 |
1006 | + ba,pt %xcc, __restore_asi |
1007 | + add %o2, %o4, %o0 |
1008 | +ENDPROC(NG4_retl_o2_plus_o4_plus_64) |
1009 | +ENTRY(NG4_retl_o2_plus_o4_fp) |
1010 | + ba,pt %xcc, __restore_asi_fp |
1011 | + add %o2, %o4, %o0 |
1012 | +ENDPROC(NG4_retl_o2_plus_o4_fp) |
1013 | +ENTRY(NG4_retl_o2_plus_o4_plus_8_fp) |
1014 | + add %o4, 8, %o4 |
1015 | + ba,pt %xcc, __restore_asi_fp |
1016 | + add %o2, %o4, %o0 |
1017 | +ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp) |
1018 | +ENTRY(NG4_retl_o2_plus_o4_plus_16_fp) |
1019 | + add %o4, 16, %o4 |
1020 | + ba,pt %xcc, __restore_asi_fp |
1021 | + add %o2, %o4, %o0 |
1022 | +ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp) |
1023 | +ENTRY(NG4_retl_o2_plus_o4_plus_24_fp) |
1024 | + add %o4, 24, %o4 |
1025 | + ba,pt %xcc, __restore_asi_fp |
1026 | + add %o2, %o4, %o0 |
1027 | +ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp) |
1028 | +ENTRY(NG4_retl_o2_plus_o4_plus_32_fp) |
1029 | + add %o4, 32, %o4 |
1030 | + ba,pt %xcc, __restore_asi_fp |
1031 | + add %o2, %o4, %o0 |
1032 | +ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp) |
1033 | +ENTRY(NG4_retl_o2_plus_o4_plus_40_fp) |
1034 | + add %o4, 40, %o4 |
1035 | + ba,pt %xcc, __restore_asi_fp |
1036 | + add %o2, %o4, %o0 |
1037 | +ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp) |
1038 | +ENTRY(NG4_retl_o2_plus_o4_plus_48_fp) |
1039 | + add %o4, 48, %o4 |
1040 | + ba,pt %xcc, __restore_asi_fp |
1041 | + add %o2, %o4, %o0 |
1042 | +ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp) |
1043 | +ENTRY(NG4_retl_o2_plus_o4_plus_56_fp) |
1044 | + add %o4, 56, %o4 |
1045 | + ba,pt %xcc, __restore_asi_fp |
1046 | + add %o2, %o4, %o0 |
1047 | +ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp) |
1048 | +ENTRY(NG4_retl_o2_plus_o4_plus_64_fp) |
1049 | + add %o4, 64, %o4 |
1050 | + ba,pt %xcc, __restore_asi_fp |
1051 | + add %o2, %o4, %o0 |
1052 | +ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp) |
1053 | +#endif |
1054 | .align 64 |
1055 | |
1056 | .globl FUNC_NAME |
1057 | @@ -124,12 +274,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1058 | brz,pt %g1, 51f |
1059 | sub %o2, %g1, %o2 |
1060 | |
1061 | -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) |
1062 | + |
1063 | +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) |
1064 | add %o1, 1, %o1 |
1065 | subcc %g1, 1, %g1 |
1066 | add %o0, 1, %o0 |
1067 | bne,pt %icc, 1b |
1068 | - EX_ST(STORE(stb, %g2, %o0 - 0x01)) |
1069 | + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1) |
1070 | |
1071 | 51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong) |
1072 | LOAD(prefetch, %o1 + 0x080, #n_reads_strong) |
1073 | @@ -154,43 +305,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1074 | brz,pt %g1, .Llarge_aligned |
1075 | sub %o2, %g1, %o2 |
1076 | |
1077 | -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2)) |
1078 | +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) |
1079 | add %o1, 8, %o1 |
1080 | subcc %g1, 8, %g1 |
1081 | add %o0, 8, %o0 |
1082 | bne,pt %icc, 1b |
1083 | - EX_ST(STORE(stx, %g2, %o0 - 0x08)) |
1084 | + EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8) |
1085 | |
1086 | .Llarge_aligned: |
1087 | /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */ |
1088 | andn %o2, 0x3f, %o4 |
1089 | sub %o2, %o4, %o2 |
1090 | |
1091 | -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) |
1092 | +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4) |
1093 | add %o1, 0x40, %o1 |
1094 | - EX_LD(LOAD(ldx, %o1 - 0x38, %g2)) |
1095 | + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4) |
1096 | subcc %o4, 0x40, %o4 |
1097 | - EX_LD(LOAD(ldx, %o1 - 0x30, %g3)) |
1098 | - EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE)) |
1099 | - EX_LD(LOAD(ldx, %o1 - 0x20, %o5)) |
1100 | - EX_ST(STORE_INIT(%g1, %o0)) |
1101 | + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64) |
1102 | + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64) |
1103 | + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64) |
1104 | + EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64) |
1105 | add %o0, 0x08, %o0 |
1106 | - EX_ST(STORE_INIT(%g2, %o0)) |
1107 | + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56) |
1108 | add %o0, 0x08, %o0 |
1109 | - EX_LD(LOAD(ldx, %o1 - 0x18, %g2)) |
1110 | - EX_ST(STORE_INIT(%g3, %o0)) |
1111 | + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48) |
1112 | + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48) |
1113 | add %o0, 0x08, %o0 |
1114 | - EX_LD(LOAD(ldx, %o1 - 0x10, %g3)) |
1115 | - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) |
1116 | + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40) |
1117 | + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40) |
1118 | add %o0, 0x08, %o0 |
1119 | - EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE)) |
1120 | - EX_ST(STORE_INIT(%o5, %o0)) |
1121 | + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32) |
1122 | + EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32) |
1123 | add %o0, 0x08, %o0 |
1124 | - EX_ST(STORE_INIT(%g2, %o0)) |
1125 | + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24) |
1126 | add %o0, 0x08, %o0 |
1127 | - EX_ST(STORE_INIT(%g3, %o0)) |
1128 | + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16) |
1129 | add %o0, 0x08, %o0 |
1130 | - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) |
1131 | + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8) |
1132 | add %o0, 0x08, %o0 |
1133 | bne,pt %icc, 1b |
1134 | LOAD(prefetch, %o1 + 0x200, #n_reads_strong) |
1135 | @@ -216,17 +367,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1136 | sub %o2, %o4, %o2 |
1137 | alignaddr %o1, %g0, %g1 |
1138 | add %o1, %o4, %o1 |
1139 | - EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0)) |
1140 | -1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2)) |
1141 | + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4) |
1142 | +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4) |
1143 | subcc %o4, 0x40, %o4 |
1144 | - EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4)) |
1145 | - EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6)) |
1146 | - EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8)) |
1147 | - EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10)) |
1148 | - EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12)) |
1149 | - EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14)) |
1150 | + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64) |
1151 | + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64) |
1152 | + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64) |
1153 | + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64) |
1154 | + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64) |
1155 | + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64) |
1156 | faligndata %f0, %f2, %f16 |
1157 | - EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0)) |
1158 | + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64) |
1159 | faligndata %f2, %f4, %f18 |
1160 | add %g1, 0x40, %g1 |
1161 | faligndata %f4, %f6, %f20 |
1162 | @@ -235,14 +386,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1163 | faligndata %f10, %f12, %f26 |
1164 | faligndata %f12, %f14, %f28 |
1165 | faligndata %f14, %f0, %f30 |
1166 | - EX_ST_FP(STORE(std, %f16, %o0 + 0x00)) |
1167 | - EX_ST_FP(STORE(std, %f18, %o0 + 0x08)) |
1168 | - EX_ST_FP(STORE(std, %f20, %o0 + 0x10)) |
1169 | - EX_ST_FP(STORE(std, %f22, %o0 + 0x18)) |
1170 | - EX_ST_FP(STORE(std, %f24, %o0 + 0x20)) |
1171 | - EX_ST_FP(STORE(std, %f26, %o0 + 0x28)) |
1172 | - EX_ST_FP(STORE(std, %f28, %o0 + 0x30)) |
1173 | - EX_ST_FP(STORE(std, %f30, %o0 + 0x38)) |
1174 | + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64) |
1175 | + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56) |
1176 | + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48) |
1177 | + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40) |
1178 | + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32) |
1179 | + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24) |
1180 | + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16) |
1181 | + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8) |
1182 | add %o0, 0x40, %o0 |
1183 | bne,pt %icc, 1b |
1184 | LOAD(prefetch, %g1 + 0x200, #n_reads_strong) |
1185 | @@ -270,37 +421,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1186 | andncc %o2, 0x20 - 1, %o5 |
1187 | be,pn %icc, 2f |
1188 | sub %o2, %o5, %o2 |
1189 | -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) |
1190 | - EX_LD(LOAD(ldx, %o1 + 0x08, %g2)) |
1191 | - EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE)) |
1192 | - EX_LD(LOAD(ldx, %o1 + 0x18, %o4)) |
1193 | +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) |
1194 | + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5) |
1195 | + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5) |
1196 | + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5) |
1197 | add %o1, 0x20, %o1 |
1198 | subcc %o5, 0x20, %o5 |
1199 | - EX_ST(STORE(stx, %g1, %o0 + 0x00)) |
1200 | - EX_ST(STORE(stx, %g2, %o0 + 0x08)) |
1201 | - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10)) |
1202 | - EX_ST(STORE(stx, %o4, %o0 + 0x18)) |
1203 | + EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32) |
1204 | + EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24) |
1205 | + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24) |
1206 | + EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8) |
1207 | bne,pt %icc, 1b |
1208 | add %o0, 0x20, %o0 |
1209 | 2: andcc %o2, 0x18, %o5 |
1210 | be,pt %icc, 3f |
1211 | sub %o2, %o5, %o2 |
1212 | -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) |
1213 | + |
1214 | +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) |
1215 | add %o1, 0x08, %o1 |
1216 | add %o0, 0x08, %o0 |
1217 | subcc %o5, 0x08, %o5 |
1218 | bne,pt %icc, 1b |
1219 | - EX_ST(STORE(stx, %g1, %o0 - 0x08)) |
1220 | + EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8) |
1221 | 3: brz,pt %o2, .Lexit |
1222 | cmp %o2, 0x04 |
1223 | bl,pn %icc, .Ltiny |
1224 | nop |
1225 | - EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) |
1226 | + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2) |
1227 | add %o1, 0x04, %o1 |
1228 | add %o0, 0x04, %o0 |
1229 | subcc %o2, 0x04, %o2 |
1230 | bne,pn %icc, .Ltiny |
1231 | - EX_ST(STORE(stw, %g1, %o0 - 0x04)) |
1232 | + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4) |
1233 | ba,a,pt %icc, .Lexit |
1234 | .Lmedium_unaligned: |
1235 | /* First get dest 8 byte aligned. */ |
1236 | @@ -309,12 +461,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1237 | brz,pt %g1, 2f |
1238 | sub %o2, %g1, %o2 |
1239 | |
1240 | -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) |
1241 | +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) |
1242 | add %o1, 1, %o1 |
1243 | subcc %g1, 1, %g1 |
1244 | add %o0, 1, %o0 |
1245 | bne,pt %icc, 1b |
1246 | - EX_ST(STORE(stb, %g2, %o0 - 0x01)) |
1247 | + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1) |
1248 | 2: |
1249 | and %o1, 0x7, %g1 |
1250 | brz,pn %g1, .Lmedium_noprefetch |
1251 | @@ -322,16 +474,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1252 | mov 64, %g2 |
1253 | sub %g2, %g1, %g2 |
1254 | andn %o1, 0x7, %o1 |
1255 | - EX_LD(LOAD(ldx, %o1 + 0x00, %o4)) |
1256 | + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2) |
1257 | sllx %o4, %g1, %o4 |
1258 | andn %o2, 0x08 - 1, %o5 |
1259 | sub %o2, %o5, %o2 |
1260 | -1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3)) |
1261 | +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5) |
1262 | add %o1, 0x08, %o1 |
1263 | subcc %o5, 0x08, %o5 |
1264 | srlx %g3, %g2, GLOBAL_SPARE |
1265 | or GLOBAL_SPARE, %o4, GLOBAL_SPARE |
1266 | - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00)) |
1267 | + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8) |
1268 | add %o0, 0x08, %o0 |
1269 | bne,pt %icc, 1b |
1270 | sllx %g3, %g1, %o4 |
1271 | @@ -342,17 +494,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1272 | ba,pt %icc, .Lsmall_unaligned |
1273 | |
1274 | .Ltiny: |
1275 | - EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) |
1276 | + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2) |
1277 | subcc %o2, 1, %o2 |
1278 | be,pn %icc, .Lexit |
1279 | - EX_ST(STORE(stb, %g1, %o0 + 0x00)) |
1280 | - EX_LD(LOAD(ldub, %o1 + 0x01, %g1)) |
1281 | + EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1) |
1282 | + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2) |
1283 | subcc %o2, 1, %o2 |
1284 | be,pn %icc, .Lexit |
1285 | - EX_ST(STORE(stb, %g1, %o0 + 0x01)) |
1286 | - EX_LD(LOAD(ldub, %o1 + 0x02, %g1)) |
1287 | + EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1) |
1288 | + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2) |
1289 | ba,pt %icc, .Lexit |
1290 | - EX_ST(STORE(stb, %g1, %o0 + 0x02)) |
1291 | + EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2) |
1292 | |
1293 | .Lsmall: |
1294 | andcc %g2, 0x3, %g0 |
1295 | @@ -360,22 +512,22 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
1296 | andn %o2, 0x4 - 1, %o5 |
1297 | sub %o2, %o5, %o2 |
1298 | 1: |
1299 | - EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) |
1300 | + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) |
1301 | add %o1, 0x04, %o1 |
1302 | subcc %o5, 0x04, %o5 |
1303 | add %o0, 0x04, %o0 |
1304 | bne,pt %icc, 1b |
1305 | - EX_ST(STORE(stw, %g1, %o0 - 0x04)) |
1306 | + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4) |
1307 | brz,pt %o2, .Lexit |
1308 | nop |
1309 | ba,a,pt %icc, .Ltiny |
1310 | |
1311 | .Lsmall_unaligned: |
1312 | -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) |
1313 | +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2) |
1314 | add %o1, 1, %o1 |
1315 | add %o0, 1, %o0 |
1316 | subcc %o2, 1, %o2 |
1317 | bne,pt %icc, 1b |
1318 | - EX_ST(STORE(stb, %g1, %o0 - 0x01)) |
1319 | + EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1) |
1320 | ba,a,pt %icc, .Lexit |
1321 | .size FUNC_NAME, .-FUNC_NAME |
1322 | diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S |
1323 | index 5d1e4d1ac21e..9cd42fcbc781 100644 |
1324 | --- a/arch/sparc/lib/NGcopy_from_user.S |
1325 | +++ b/arch/sparc/lib/NGcopy_from_user.S |
1326 | @@ -3,11 +3,11 @@ |
1327 | * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) |
1328 | */ |
1329 | |
1330 | -#define EX_LD(x) \ |
1331 | +#define EX_LD(x,y) \ |
1332 | 98: x; \ |
1333 | .section __ex_table,"a";\ |
1334 | .align 4; \ |
1335 | - .word 98b, __ret_one_asi;\ |
1336 | + .word 98b, y; \ |
1337 | .text; \ |
1338 | .align 4; |
1339 | |
1340 | diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S |
1341 | index ff630dcb273c..5c358afd464e 100644 |
1342 | --- a/arch/sparc/lib/NGcopy_to_user.S |
1343 | +++ b/arch/sparc/lib/NGcopy_to_user.S |
1344 | @@ -3,11 +3,11 @@ |
1345 | * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) |
1346 | */ |
1347 | |
1348 | -#define EX_ST(x) \ |
1349 | +#define EX_ST(x,y) \ |
1350 | 98: x; \ |
1351 | .section __ex_table,"a";\ |
1352 | .align 4; \ |
1353 | - .word 98b, __ret_one_asi;\ |
1354 | + .word 98b, y; \ |
1355 | .text; \ |
1356 | .align 4; |
1357 | |
1358 | diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S |
1359 | index 96a14caf6966..d88c4ed50a00 100644 |
1360 | --- a/arch/sparc/lib/NGmemcpy.S |
1361 | +++ b/arch/sparc/lib/NGmemcpy.S |
1362 | @@ -4,6 +4,7 @@ |
1363 | */ |
1364 | |
1365 | #ifdef __KERNEL__ |
1366 | +#include <linux/linkage.h> |
1367 | #include <asm/asi.h> |
1368 | #include <asm/thread_info.h> |
1369 | #define GLOBAL_SPARE %g7 |
1370 | @@ -27,15 +28,11 @@ |
1371 | #endif |
1372 | |
1373 | #ifndef EX_LD |
1374 | -#define EX_LD(x) x |
1375 | +#define EX_LD(x,y) x |
1376 | #endif |
1377 | |
1378 | #ifndef EX_ST |
1379 | -#define EX_ST(x) x |
1380 | -#endif |
1381 | - |
1382 | -#ifndef EX_RETVAL |
1383 | -#define EX_RETVAL(x) x |
1384 | +#define EX_ST(x,y) x |
1385 | #endif |
1386 | |
1387 | #ifndef LOAD |
1388 | @@ -79,6 +76,92 @@ |
1389 | .register %g3,#scratch |
1390 | |
1391 | .text |
1392 | +#ifndef EX_RETVAL |
1393 | +#define EX_RETVAL(x) x |
1394 | +__restore_asi: |
1395 | + ret |
1396 | + wr %g0, ASI_AIUS, %asi |
1397 | + restore |
1398 | +ENTRY(NG_ret_i2_plus_i4_plus_1) |
1399 | + ba,pt %xcc, __restore_asi |
1400 | + add %i2, %i5, %i0 |
1401 | +ENDPROC(NG_ret_i2_plus_i4_plus_1) |
1402 | +ENTRY(NG_ret_i2_plus_g1) |
1403 | + ba,pt %xcc, __restore_asi |
1404 | + add %i2, %g1, %i0 |
1405 | +ENDPROC(NG_ret_i2_plus_g1) |
1406 | +ENTRY(NG_ret_i2_plus_g1_minus_8) |
1407 | + sub %g1, 8, %g1 |
1408 | + ba,pt %xcc, __restore_asi |
1409 | + add %i2, %g1, %i0 |
1410 | +ENDPROC(NG_ret_i2_plus_g1_minus_8) |
1411 | +ENTRY(NG_ret_i2_plus_g1_minus_16) |
1412 | + sub %g1, 16, %g1 |
1413 | + ba,pt %xcc, __restore_asi |
1414 | + add %i2, %g1, %i0 |
1415 | +ENDPROC(NG_ret_i2_plus_g1_minus_16) |
1416 | +ENTRY(NG_ret_i2_plus_g1_minus_24) |
1417 | + sub %g1, 24, %g1 |
1418 | + ba,pt %xcc, __restore_asi |
1419 | + add %i2, %g1, %i0 |
1420 | +ENDPROC(NG_ret_i2_plus_g1_minus_24) |
1421 | +ENTRY(NG_ret_i2_plus_g1_minus_32) |
1422 | + sub %g1, 32, %g1 |
1423 | + ba,pt %xcc, __restore_asi |
1424 | + add %i2, %g1, %i0 |
1425 | +ENDPROC(NG_ret_i2_plus_g1_minus_32) |
1426 | +ENTRY(NG_ret_i2_plus_g1_minus_40) |
1427 | + sub %g1, 40, %g1 |
1428 | + ba,pt %xcc, __restore_asi |
1429 | + add %i2, %g1, %i0 |
1430 | +ENDPROC(NG_ret_i2_plus_g1_minus_40) |
1431 | +ENTRY(NG_ret_i2_plus_g1_minus_48) |
1432 | + sub %g1, 48, %g1 |
1433 | + ba,pt %xcc, __restore_asi |
1434 | + add %i2, %g1, %i0 |
1435 | +ENDPROC(NG_ret_i2_plus_g1_minus_48) |
1436 | +ENTRY(NG_ret_i2_plus_g1_minus_56) |
1437 | + sub %g1, 56, %g1 |
1438 | + ba,pt %xcc, __restore_asi |
1439 | + add %i2, %g1, %i0 |
1440 | +ENDPROC(NG_ret_i2_plus_g1_minus_56) |
1441 | +ENTRY(NG_ret_i2_plus_i4) |
1442 | + ba,pt %xcc, __restore_asi |
1443 | + add %i2, %i4, %i0 |
1444 | +ENDPROC(NG_ret_i2_plus_i4) |
1445 | +ENTRY(NG_ret_i2_plus_i4_minus_8) |
1446 | + sub %i4, 8, %i4 |
1447 | + ba,pt %xcc, __restore_asi |
1448 | + add %i2, %i4, %i0 |
1449 | +ENDPROC(NG_ret_i2_plus_i4_minus_8) |
1450 | +ENTRY(NG_ret_i2_plus_8) |
1451 | + ba,pt %xcc, __restore_asi |
1452 | + add %i2, 8, %i0 |
1453 | +ENDPROC(NG_ret_i2_plus_8) |
1454 | +ENTRY(NG_ret_i2_plus_4) |
1455 | + ba,pt %xcc, __restore_asi |
1456 | + add %i2, 4, %i0 |
1457 | +ENDPROC(NG_ret_i2_plus_4) |
1458 | +ENTRY(NG_ret_i2_plus_1) |
1459 | + ba,pt %xcc, __restore_asi |
1460 | + add %i2, 1, %i0 |
1461 | +ENDPROC(NG_ret_i2_plus_1) |
1462 | +ENTRY(NG_ret_i2_plus_g1_plus_1) |
1463 | + add %g1, 1, %g1 |
1464 | + ba,pt %xcc, __restore_asi |
1465 | + add %i2, %g1, %i0 |
1466 | +ENDPROC(NG_ret_i2_plus_g1_plus_1) |
1467 | +ENTRY(NG_ret_i2) |
1468 | + ba,pt %xcc, __restore_asi |
1469 | + mov %i2, %i0 |
1470 | +ENDPROC(NG_ret_i2) |
1471 | +ENTRY(NG_ret_i2_and_7_plus_i4) |
1472 | + and %i2, 7, %i2 |
1473 | + ba,pt %xcc, __restore_asi |
1474 | + add %i2, %i4, %i0 |
1475 | +ENDPROC(NG_ret_i2_and_7_plus_i4) |
1476 | +#endif |
1477 | + |
1478 | .align 64 |
1479 | |
1480 | .globl FUNC_NAME |
1481 | @@ -126,8 +209,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1482 | sub %g0, %i4, %i4 ! bytes to align dst |
1483 | sub %i2, %i4, %i2 |
1484 | 1: subcc %i4, 1, %i4 |
1485 | - EX_LD(LOAD(ldub, %i1, %g1)) |
1486 | - EX_ST(STORE(stb, %g1, %o0)) |
1487 | + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1) |
1488 | + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1) |
1489 | add %i1, 1, %i1 |
1490 | bne,pt %XCC, 1b |
1491 | add %o0, 1, %o0 |
1492 | @@ -160,7 +243,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1493 | and %i4, 0x7, GLOBAL_SPARE |
1494 | sll GLOBAL_SPARE, 3, GLOBAL_SPARE |
1495 | mov 64, %i5 |
1496 | - EX_LD(LOAD_TWIN(%i1, %g2, %g3)) |
1497 | + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1) |
1498 | sub %i5, GLOBAL_SPARE, %i5 |
1499 | mov 16, %o4 |
1500 | mov 32, %o5 |
1501 | @@ -178,31 +261,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1502 | srlx WORD3, PRE_SHIFT, TMP; \ |
1503 | or WORD2, TMP, WORD2; |
1504 | |
1505 | -8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) |
1506 | +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) |
1507 | MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) |
1508 | LOAD(prefetch, %i1 + %i3, #one_read) |
1509 | |
1510 | - EX_ST(STORE_INIT(%g2, %o0 + 0x00)) |
1511 | - EX_ST(STORE_INIT(%g3, %o0 + 0x08)) |
1512 | + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1) |
1513 | + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) |
1514 | |
1515 | - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) |
1516 | + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) |
1517 | MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) |
1518 | |
1519 | - EX_ST(STORE_INIT(%o2, %o0 + 0x10)) |
1520 | - EX_ST(STORE_INIT(%o3, %o0 + 0x18)) |
1521 | + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) |
1522 | + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) |
1523 | |
1524 | - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) |
1525 | + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) |
1526 | MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) |
1527 | |
1528 | - EX_ST(STORE_INIT(%g2, %o0 + 0x20)) |
1529 | - EX_ST(STORE_INIT(%g3, %o0 + 0x28)) |
1530 | + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) |
1531 | + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) |
1532 | |
1533 | - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) |
1534 | + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) |
1535 | add %i1, 64, %i1 |
1536 | MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) |
1537 | |
1538 | - EX_ST(STORE_INIT(%o2, %o0 + 0x30)) |
1539 | - EX_ST(STORE_INIT(%o3, %o0 + 0x38)) |
1540 | + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) |
1541 | + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) |
1542 | |
1543 | subcc %g1, 64, %g1 |
1544 | bne,pt %XCC, 8b |
1545 | @@ -211,31 +294,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1546 | ba,pt %XCC, 60f |
1547 | add %i1, %i4, %i1 |
1548 | |
1549 | -9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) |
1550 | +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) |
1551 | MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) |
1552 | LOAD(prefetch, %i1 + %i3, #one_read) |
1553 | |
1554 | - EX_ST(STORE_INIT(%g3, %o0 + 0x00)) |
1555 | - EX_ST(STORE_INIT(%o2, %o0 + 0x08)) |
1556 | + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1) |
1557 | + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) |
1558 | |
1559 | - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) |
1560 | + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) |
1561 | MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) |
1562 | |
1563 | - EX_ST(STORE_INIT(%o3, %o0 + 0x10)) |
1564 | - EX_ST(STORE_INIT(%g2, %o0 + 0x18)) |
1565 | + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) |
1566 | + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) |
1567 | |
1568 | - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) |
1569 | + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) |
1570 | MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) |
1571 | |
1572 | - EX_ST(STORE_INIT(%g3, %o0 + 0x20)) |
1573 | - EX_ST(STORE_INIT(%o2, %o0 + 0x28)) |
1574 | + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) |
1575 | + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) |
1576 | |
1577 | - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) |
1578 | + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) |
1579 | add %i1, 64, %i1 |
1580 | MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) |
1581 | |
1582 | - EX_ST(STORE_INIT(%o3, %o0 + 0x30)) |
1583 | - EX_ST(STORE_INIT(%g2, %o0 + 0x38)) |
1584 | + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) |
1585 | + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) |
1586 | |
1587 | subcc %g1, 64, %g1 |
1588 | bne,pt %XCC, 9b |
1589 | @@ -249,25 +332,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1590 | * one twin load ahead, then add 8 back into source when |
1591 | * we finish the loop. |
1592 | */ |
1593 | - EX_LD(LOAD_TWIN(%i1, %o4, %o5)) |
1594 | + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1) |
1595 | mov 16, %o7 |
1596 | mov 32, %g2 |
1597 | mov 48, %g3 |
1598 | mov 64, %o1 |
1599 | -1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) |
1600 | +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) |
1601 | LOAD(prefetch, %i1 + %o1, #one_read) |
1602 | - EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line |
1603 | - EX_ST(STORE_INIT(%o2, %o0 + 0x08)) |
1604 | - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) |
1605 | - EX_ST(STORE_INIT(%o3, %o0 + 0x10)) |
1606 | - EX_ST(STORE_INIT(%o4, %o0 + 0x18)) |
1607 | - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) |
1608 | - EX_ST(STORE_INIT(%o5, %o0 + 0x20)) |
1609 | - EX_ST(STORE_INIT(%o2, %o0 + 0x28)) |
1610 | - EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5)) |
1611 | + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line |
1612 | + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) |
1613 | + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) |
1614 | + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) |
1615 | + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) |
1616 | + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) |
1617 | + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) |
1618 | + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) |
1619 | + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48) |
1620 | add %i1, 64, %i1 |
1621 | - EX_ST(STORE_INIT(%o3, %o0 + 0x30)) |
1622 | - EX_ST(STORE_INIT(%o4, %o0 + 0x38)) |
1623 | + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) |
1624 | + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) |
1625 | subcc %g1, 64, %g1 |
1626 | bne,pt %XCC, 1b |
1627 | add %o0, 64, %o0 |
1628 | @@ -282,20 +365,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1629 | mov 32, %g2 |
1630 | mov 48, %g3 |
1631 | mov 64, %o1 |
1632 | -1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5)) |
1633 | - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) |
1634 | +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1) |
1635 | + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) |
1636 | LOAD(prefetch, %i1 + %o1, #one_read) |
1637 | - EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line |
1638 | - EX_ST(STORE_INIT(%o5, %o0 + 0x08)) |
1639 | - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) |
1640 | - EX_ST(STORE_INIT(%o2, %o0 + 0x10)) |
1641 | - EX_ST(STORE_INIT(%o3, %o0 + 0x18)) |
1642 | - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) |
1643 | + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line |
1644 | + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) |
1645 | + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) |
1646 | + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) |
1647 | + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) |
1648 | + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) |
1649 | add %i1, 64, %i1 |
1650 | - EX_ST(STORE_INIT(%o4, %o0 + 0x20)) |
1651 | - EX_ST(STORE_INIT(%o5, %o0 + 0x28)) |
1652 | - EX_ST(STORE_INIT(%o2, %o0 + 0x30)) |
1653 | - EX_ST(STORE_INIT(%o3, %o0 + 0x38)) |
1654 | + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) |
1655 | + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) |
1656 | + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) |
1657 | + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) |
1658 | subcc %g1, 64, %g1 |
1659 | bne,pt %XCC, 1b |
1660 | add %o0, 64, %o0 |
1661 | @@ -321,28 +404,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1662 | andn %i2, 0xf, %i4 |
1663 | and %i2, 0xf, %i2 |
1664 | 1: subcc %i4, 0x10, %i4 |
1665 | - EX_LD(LOAD(ldx, %i1, %o4)) |
1666 | + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4) |
1667 | add %i1, 0x08, %i1 |
1668 | - EX_LD(LOAD(ldx, %i1, %g1)) |
1669 | + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4) |
1670 | sub %i1, 0x08, %i1 |
1671 | - EX_ST(STORE(stx, %o4, %i1 + %i3)) |
1672 | + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4) |
1673 | add %i1, 0x8, %i1 |
1674 | - EX_ST(STORE(stx, %g1, %i1 + %i3)) |
1675 | + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8) |
1676 | bgu,pt %XCC, 1b |
1677 | add %i1, 0x8, %i1 |
1678 | 73: andcc %i2, 0x8, %g0 |
1679 | be,pt %XCC, 1f |
1680 | nop |
1681 | sub %i2, 0x8, %i2 |
1682 | - EX_LD(LOAD(ldx, %i1, %o4)) |
1683 | - EX_ST(STORE(stx, %o4, %i1 + %i3)) |
1684 | + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8) |
1685 | + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8) |
1686 | add %i1, 0x8, %i1 |
1687 | 1: andcc %i2, 0x4, %g0 |
1688 | be,pt %XCC, 1f |
1689 | nop |
1690 | sub %i2, 0x4, %i2 |
1691 | - EX_LD(LOAD(lduw, %i1, %i5)) |
1692 | - EX_ST(STORE(stw, %i5, %i1 + %i3)) |
1693 | + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4) |
1694 | + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4) |
1695 | add %i1, 0x4, %i1 |
1696 | 1: cmp %i2, 0 |
1697 | be,pt %XCC, 85f |
1698 | @@ -358,8 +441,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1699 | sub %i2, %g1, %i2 |
1700 | |
1701 | 1: subcc %g1, 1, %g1 |
1702 | - EX_LD(LOAD(ldub, %i1, %i5)) |
1703 | - EX_ST(STORE(stb, %i5, %i1 + %i3)) |
1704 | + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1) |
1705 | + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1) |
1706 | bgu,pt %icc, 1b |
1707 | add %i1, 1, %i1 |
1708 | |
1709 | @@ -375,16 +458,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1710 | |
1711 | 8: mov 64, %i3 |
1712 | andn %i1, 0x7, %i1 |
1713 | - EX_LD(LOAD(ldx, %i1, %g2)) |
1714 | + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2) |
1715 | sub %i3, %g1, %i3 |
1716 | andn %i2, 0x7, %i4 |
1717 | sllx %g2, %g1, %g2 |
1718 | 1: add %i1, 0x8, %i1 |
1719 | - EX_LD(LOAD(ldx, %i1, %g3)) |
1720 | + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4) |
1721 | subcc %i4, 0x8, %i4 |
1722 | srlx %g3, %i3, %i5 |
1723 | or %i5, %g2, %i5 |
1724 | - EX_ST(STORE(stx, %i5, %o0)) |
1725 | + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4) |
1726 | add %o0, 0x8, %o0 |
1727 | bgu,pt %icc, 1b |
1728 | sllx %g3, %g1, %g2 |
1729 | @@ -404,8 +487,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1730 | |
1731 | 1: |
1732 | subcc %i2, 4, %i2 |
1733 | - EX_LD(LOAD(lduw, %i1, %g1)) |
1734 | - EX_ST(STORE(stw, %g1, %i1 + %i3)) |
1735 | + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4) |
1736 | + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4) |
1737 | bgu,pt %XCC, 1b |
1738 | add %i1, 4, %i1 |
1739 | |
1740 | @@ -415,8 +498,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ |
1741 | .align 32 |
1742 | 90: |
1743 | subcc %i2, 1, %i2 |
1744 | - EX_LD(LOAD(ldub, %i1, %g1)) |
1745 | - EX_ST(STORE(stb, %g1, %i1 + %i3)) |
1746 | + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1) |
1747 | + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1) |
1748 | bgu,pt %XCC, 90b |
1749 | add %i1, 1, %i1 |
1750 | ret |
1751 | diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S |
1752 | index ecc5692fa2b4..bb6ff73229e3 100644 |
1753 | --- a/arch/sparc/lib/U1copy_from_user.S |
1754 | +++ b/arch/sparc/lib/U1copy_from_user.S |
1755 | @@ -3,19 +3,19 @@ |
1756 | * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) |
1757 | */ |
1758 | |
1759 | -#define EX_LD(x) \ |
1760 | +#define EX_LD(x,y) \ |
1761 | 98: x; \ |
1762 | .section __ex_table,"a";\ |
1763 | .align 4; \ |
1764 | - .word 98b, __retl_one; \ |
1765 | + .word 98b, y; \ |
1766 | .text; \ |
1767 | .align 4; |
1768 | |
1769 | -#define EX_LD_FP(x) \ |
1770 | +#define EX_LD_FP(x,y) \ |
1771 | 98: x; \ |
1772 | .section __ex_table,"a";\ |
1773 | .align 4; \ |
1774 | - .word 98b, __retl_one_fp;\ |
1775 | + .word 98b, y; \ |
1776 | .text; \ |
1777 | .align 4; |
1778 | |
1779 | diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S |
1780 | index 9eea392e44d4..ed92ce739558 100644 |
1781 | --- a/arch/sparc/lib/U1copy_to_user.S |
1782 | +++ b/arch/sparc/lib/U1copy_to_user.S |
1783 | @@ -3,19 +3,19 @@ |
1784 | * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) |
1785 | */ |
1786 | |
1787 | -#define EX_ST(x) \ |
1788 | +#define EX_ST(x,y) \ |
1789 | 98: x; \ |
1790 | .section __ex_table,"a";\ |
1791 | .align 4; \ |
1792 | - .word 98b, __retl_one; \ |
1793 | + .word 98b, y; \ |
1794 | .text; \ |
1795 | .align 4; |
1796 | |
1797 | -#define EX_ST_FP(x) \ |
1798 | +#define EX_ST_FP(x,y) \ |
1799 | 98: x; \ |
1800 | .section __ex_table,"a";\ |
1801 | .align 4; \ |
1802 | - .word 98b, __retl_one_fp;\ |
1803 | + .word 98b, y; \ |
1804 | .text; \ |
1805 | .align 4; |
1806 | |
1807 | diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S |
1808 | index 3e6209ebb7d7..f30d2ab2c371 100644 |
1809 | --- a/arch/sparc/lib/U1memcpy.S |
1810 | +++ b/arch/sparc/lib/U1memcpy.S |
1811 | @@ -5,6 +5,7 @@ |
1812 | */ |
1813 | |
1814 | #ifdef __KERNEL__ |
1815 | +#include <linux/linkage.h> |
1816 | #include <asm/visasm.h> |
1817 | #include <asm/asi.h> |
1818 | #define GLOBAL_SPARE g7 |
1819 | @@ -23,21 +24,17 @@ |
1820 | #endif |
1821 | |
1822 | #ifndef EX_LD |
1823 | -#define EX_LD(x) x |
1824 | +#define EX_LD(x,y) x |
1825 | #endif |
1826 | #ifndef EX_LD_FP |
1827 | -#define EX_LD_FP(x) x |
1828 | +#define EX_LD_FP(x,y) x |
1829 | #endif |
1830 | |
1831 | #ifndef EX_ST |
1832 | -#define EX_ST(x) x |
1833 | +#define EX_ST(x,y) x |
1834 | #endif |
1835 | #ifndef EX_ST_FP |
1836 | -#define EX_ST_FP(x) x |
1837 | -#endif |
1838 | - |
1839 | -#ifndef EX_RETVAL |
1840 | -#define EX_RETVAL(x) x |
1841 | +#define EX_ST_FP(x,y) x |
1842 | #endif |
1843 | |
1844 | #ifndef LOAD |
1845 | @@ -78,53 +75,169 @@ |
1846 | faligndata %f7, %f8, %f60; \ |
1847 | faligndata %f8, %f9, %f62; |
1848 | |
1849 | -#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \ |
1850 | - EX_LD_FP(LOAD_BLK(%src, %fdest)); \ |
1851 | - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ |
1852 | - add %src, 0x40, %src; \ |
1853 | - subcc %len, 0x40, %len; \ |
1854 | - be,pn %xcc, jmptgt; \ |
1855 | - add %dest, 0x40, %dest; \ |
1856 | - |
1857 | -#define LOOP_CHUNK1(src, dest, len, branch_dest) \ |
1858 | - MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest) |
1859 | -#define LOOP_CHUNK2(src, dest, len, branch_dest) \ |
1860 | - MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest) |
1861 | -#define LOOP_CHUNK3(src, dest, len, branch_dest) \ |
1862 | - MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest) |
1863 | +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \ |
1864 | + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \ |
1865 | + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ |
1866 | + add %src, 0x40, %src; \ |
1867 | + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \ |
1868 | + be,pn %xcc, jmptgt; \ |
1869 | + add %dest, 0x40, %dest; \ |
1870 | + |
1871 | +#define LOOP_CHUNK1(src, dest, branch_dest) \ |
1872 | + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest) |
1873 | +#define LOOP_CHUNK2(src, dest, branch_dest) \ |
1874 | + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest) |
1875 | +#define LOOP_CHUNK3(src, dest, branch_dest) \ |
1876 | + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest) |
1877 | |
1878 | #define DO_SYNC membar #Sync; |
1879 | #define STORE_SYNC(dest, fsrc) \ |
1880 | - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ |
1881 | + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ |
1882 | add %dest, 0x40, %dest; \ |
1883 | DO_SYNC |
1884 | |
1885 | #define STORE_JUMP(dest, fsrc, target) \ |
1886 | - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ |
1887 | + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \ |
1888 | add %dest, 0x40, %dest; \ |
1889 | ba,pt %xcc, target; \ |
1890 | nop; |
1891 | |
1892 | -#define FINISH_VISCHUNK(dest, f0, f1, left) \ |
1893 | - subcc %left, 8, %left;\ |
1894 | - bl,pn %xcc, 95f; \ |
1895 | - faligndata %f0, %f1, %f48; \ |
1896 | - EX_ST_FP(STORE(std, %f48, %dest)); \ |
1897 | +#define FINISH_VISCHUNK(dest, f0, f1) \ |
1898 | + subcc %g3, 8, %g3; \ |
1899 | + bl,pn %xcc, 95f; \ |
1900 | + faligndata %f0, %f1, %f48; \ |
1901 | + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \ |
1902 | add %dest, 8, %dest; |
1903 | |
1904 | -#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ |
1905 | - subcc %left, 8, %left; \ |
1906 | - bl,pn %xcc, 95f; \ |
1907 | +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ |
1908 | + subcc %g3, 8, %g3; \ |
1909 | + bl,pn %xcc, 95f; \ |
1910 | fsrc2 %f0, %f1; |
1911 | |
1912 | -#define UNEVEN_VISCHUNK(dest, f0, f1, left) \ |
1913 | - UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ |
1914 | +#define UNEVEN_VISCHUNK(dest, f0, f1) \ |
1915 | + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ |
1916 | ba,a,pt %xcc, 93f; |
1917 | |
1918 | .register %g2,#scratch |
1919 | .register %g3,#scratch |
1920 | |
1921 | .text |
1922 | +#ifndef EX_RETVAL |
1923 | +#define EX_RETVAL(x) x |
1924 | +ENTRY(U1_g1_1_fp) |
1925 | + VISExitHalf |
1926 | + add %g1, 1, %g1 |
1927 | + add %g1, %g2, %g1 |
1928 | + retl |
1929 | + add %g1, %o2, %o0 |
1930 | +ENDPROC(U1_g1_1_fp) |
1931 | +ENTRY(U1_g2_0_fp) |
1932 | + VISExitHalf |
1933 | + retl |
1934 | + add %g2, %o2, %o0 |
1935 | +ENDPROC(U1_g2_0_fp) |
1936 | +ENTRY(U1_g2_8_fp) |
1937 | + VISExitHalf |
1938 | + add %g2, 8, %g2 |
1939 | + retl |
1940 | + add %g2, %o2, %o0 |
1941 | +ENDPROC(U1_g2_8_fp) |
1942 | +ENTRY(U1_gs_0_fp) |
1943 | + VISExitHalf |
1944 | + add %GLOBAL_SPARE, %g3, %o0 |
1945 | + retl |
1946 | + add %o0, %o2, %o0 |
1947 | +ENDPROC(U1_gs_0_fp) |
1948 | +ENTRY(U1_gs_80_fp) |
1949 | + VISExitHalf |
1950 | + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE |
1951 | + add %GLOBAL_SPARE, %g3, %o0 |
1952 | + retl |
1953 | + add %o0, %o2, %o0 |
1954 | +ENDPROC(U1_gs_80_fp) |
1955 | +ENTRY(U1_gs_40_fp) |
1956 | + VISExitHalf |
1957 | + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE |
1958 | + add %GLOBAL_SPARE, %g3, %o0 |
1959 | + retl |
1960 | + add %o0, %o2, %o0 |
1961 | +ENDPROC(U1_gs_40_fp) |
1962 | +ENTRY(U1_g3_0_fp) |
1963 | + VISExitHalf |
1964 | + retl |
1965 | + add %g3, %o2, %o0 |
1966 | +ENDPROC(U1_g3_0_fp) |
1967 | +ENTRY(U1_g3_8_fp) |
1968 | + VISExitHalf |
1969 | + add %g3, 8, %g3 |
1970 | + retl |
1971 | + add %g3, %o2, %o0 |
1972 | +ENDPROC(U1_g3_8_fp) |
1973 | +ENTRY(U1_o2_0_fp) |
1974 | + VISExitHalf |
1975 | + retl |
1976 | + mov %o2, %o0 |
1977 | +ENDPROC(U1_o2_0_fp) |
1978 | +ENTRY(U1_o2_1_fp) |
1979 | + VISExitHalf |
1980 | + retl |
1981 | + add %o2, 1, %o0 |
1982 | +ENDPROC(U1_o2_1_fp) |
1983 | +ENTRY(U1_gs_0) |
1984 | + VISExitHalf |
1985 | + retl |
1986 | + add %GLOBAL_SPARE, %o2, %o0 |
1987 | +ENDPROC(U1_gs_0) |
1988 | +ENTRY(U1_gs_8) |
1989 | + VISExitHalf |
1990 | + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE |
1991 | + retl |
1992 | + add %GLOBAL_SPARE, 0x8, %o0 |
1993 | +ENDPROC(U1_gs_8) |
1994 | +ENTRY(U1_gs_10) |
1995 | + VISExitHalf |
1996 | + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE |
1997 | + retl |
1998 | + add %GLOBAL_SPARE, 0x10, %o0 |
1999 | +ENDPROC(U1_gs_10) |
2000 | +ENTRY(U1_o2_0) |
2001 | + retl |
2002 | + mov %o2, %o0 |
2003 | +ENDPROC(U1_o2_0) |
2004 | +ENTRY(U1_o2_8) |
2005 | + retl |
2006 | + add %o2, 8, %o0 |
2007 | +ENDPROC(U1_o2_8) |
2008 | +ENTRY(U1_o2_4) |
2009 | + retl |
2010 | + add %o2, 4, %o0 |
2011 | +ENDPROC(U1_o2_4) |
2012 | +ENTRY(U1_o2_1) |
2013 | + retl |
2014 | + add %o2, 1, %o0 |
2015 | +ENDPROC(U1_o2_1) |
2016 | +ENTRY(U1_g1_0) |
2017 | + retl |
2018 | + add %g1, %o2, %o0 |
2019 | +ENDPROC(U1_g1_0) |
2020 | +ENTRY(U1_g1_1) |
2021 | + add %g1, 1, %g1 |
2022 | + retl |
2023 | + add %g1, %o2, %o0 |
2024 | +ENDPROC(U1_g1_1) |
2025 | +ENTRY(U1_gs_0_o2_adj) |
2026 | + and %o2, 7, %o2 |
2027 | + retl |
2028 | + add %GLOBAL_SPARE, %o2, %o0 |
2029 | +ENDPROC(U1_gs_0_o2_adj) |
2030 | +ENTRY(U1_gs_8_o2_adj) |
2031 | + and %o2, 7, %o2 |
2032 | + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE |
2033 | + retl |
2034 | + add %GLOBAL_SPARE, %o2, %o0 |
2035 | +ENDPROC(U1_gs_8_o2_adj) |
2036 | +#endif |
2037 | + |
2038 | .align 64 |
2039 | |
2040 | .globl FUNC_NAME |
2041 | @@ -166,8 +279,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2042 | and %g2, 0x38, %g2 |
2043 | |
2044 | 1: subcc %g1, 0x1, %g1 |
2045 | - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3)) |
2046 | - EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE)) |
2047 | + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp) |
2048 | + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp) |
2049 | bgu,pt %XCC, 1b |
2050 | add %o1, 0x1, %o1 |
2051 | |
2052 | @@ -178,20 +291,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2053 | be,pt %icc, 3f |
2054 | alignaddr %o1, %g0, %o1 |
2055 | |
2056 | - EX_LD_FP(LOAD(ldd, %o1, %f4)) |
2057 | -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6)) |
2058 | + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp) |
2059 | +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp) |
2060 | add %o1, 0x8, %o1 |
2061 | subcc %g2, 0x8, %g2 |
2062 | faligndata %f4, %f6, %f0 |
2063 | - EX_ST_FP(STORE(std, %f0, %o0)) |
2064 | + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) |
2065 | be,pn %icc, 3f |
2066 | add %o0, 0x8, %o0 |
2067 | |
2068 | - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4)) |
2069 | + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp) |
2070 | add %o1, 0x8, %o1 |
2071 | subcc %g2, 0x8, %g2 |
2072 | faligndata %f6, %f4, %f0 |
2073 | - EX_ST_FP(STORE(std, %f0, %o0)) |
2074 | + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) |
2075 | bne,pt %icc, 1b |
2076 | add %o0, 0x8, %o0 |
2077 | |
2078 | @@ -214,13 +327,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2079 | add %g1, %GLOBAL_SPARE, %g1 |
2080 | subcc %o2, %g3, %o2 |
2081 | |
2082 | - EX_LD_FP(LOAD_BLK(%o1, %f0)) |
2083 | + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp) |
2084 | add %o1, 0x40, %o1 |
2085 | add %g1, %g3, %g1 |
2086 | - EX_LD_FP(LOAD_BLK(%o1, %f16)) |
2087 | + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp) |
2088 | add %o1, 0x40, %o1 |
2089 | sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE |
2090 | - EX_LD_FP(LOAD_BLK(%o1, %f32)) |
2091 | + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp) |
2092 | add %o1, 0x40, %o1 |
2093 | |
2094 | /* There are 8 instances of the unrolled loop, |
2095 | @@ -240,11 +353,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2096 | |
2097 | .align 64 |
2098 | 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) |
2099 | - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) |
2100 | + LOOP_CHUNK1(o1, o0, 1f) |
2101 | FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) |
2102 | - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) |
2103 | + LOOP_CHUNK2(o1, o0, 2f) |
2104 | FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) |
2105 | - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) |
2106 | + LOOP_CHUNK3(o1, o0, 3f) |
2107 | ba,pt %xcc, 1b+4 |
2108 | faligndata %f0, %f2, %f48 |
2109 | 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) |
2110 | @@ -261,11 +374,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2111 | STORE_JUMP(o0, f48, 56f) |
2112 | |
2113 | 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) |
2114 | - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) |
2115 | + LOOP_CHUNK1(o1, o0, 1f) |
2116 | FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) |
2117 | - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) |
2118 | + LOOP_CHUNK2(o1, o0, 2f) |
2119 | FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) |
2120 | - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) |
2121 | + LOOP_CHUNK3(o1, o0, 3f) |
2122 | ba,pt %xcc, 1b+4 |
2123 | faligndata %f2, %f4, %f48 |
2124 | 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) |
2125 | @@ -282,11 +395,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2126 | STORE_JUMP(o0, f48, 57f) |
2127 | |
2128 | 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) |
2129 | - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) |
2130 | + LOOP_CHUNK1(o1, o0, 1f) |
2131 | FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) |
2132 | - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) |
2133 | + LOOP_CHUNK2(o1, o0, 2f) |
2134 | FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) |
2135 | - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) |
2136 | + LOOP_CHUNK3(o1, o0, 3f) |
2137 | ba,pt %xcc, 1b+4 |
2138 | faligndata %f4, %f6, %f48 |
2139 | 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) |
2140 | @@ -303,11 +416,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2141 | STORE_JUMP(o0, f48, 58f) |
2142 | |
2143 | 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) |
2144 | - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) |
2145 | + LOOP_CHUNK1(o1, o0, 1f) |
2146 | FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) |
2147 | - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) |
2148 | + LOOP_CHUNK2(o1, o0, 2f) |
2149 | FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) |
2150 | - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) |
2151 | + LOOP_CHUNK3(o1, o0, 3f) |
2152 | ba,pt %xcc, 1b+4 |
2153 | faligndata %f6, %f8, %f48 |
2154 | 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) |
2155 | @@ -324,11 +437,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2156 | STORE_JUMP(o0, f48, 59f) |
2157 | |
2158 | 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) |
2159 | - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) |
2160 | + LOOP_CHUNK1(o1, o0, 1f) |
2161 | FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) |
2162 | - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) |
2163 | + LOOP_CHUNK2(o1, o0, 2f) |
2164 | FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) |
2165 | - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) |
2166 | + LOOP_CHUNK3(o1, o0, 3f) |
2167 | ba,pt %xcc, 1b+4 |
2168 | faligndata %f8, %f10, %f48 |
2169 | 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) |
2170 | @@ -345,11 +458,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2171 | STORE_JUMP(o0, f48, 60f) |
2172 | |
2173 | 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) |
2174 | - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) |
2175 | + LOOP_CHUNK1(o1, o0, 1f) |
2176 | FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) |
2177 | - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) |
2178 | + LOOP_CHUNK2(o1, o0, 2f) |
2179 | FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) |
2180 | - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) |
2181 | + LOOP_CHUNK3(o1, o0, 3f) |
2182 | ba,pt %xcc, 1b+4 |
2183 | faligndata %f10, %f12, %f48 |
2184 | 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) |
2185 | @@ -366,11 +479,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2186 | STORE_JUMP(o0, f48, 61f) |
2187 | |
2188 | 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) |
2189 | - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) |
2190 | + LOOP_CHUNK1(o1, o0, 1f) |
2191 | FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) |
2192 | - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) |
2193 | + LOOP_CHUNK2(o1, o0, 2f) |
2194 | FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) |
2195 | - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) |
2196 | + LOOP_CHUNK3(o1, o0, 3f) |
2197 | ba,pt %xcc, 1b+4 |
2198 | faligndata %f12, %f14, %f48 |
2199 | 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) |
2200 | @@ -387,11 +500,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2201 | STORE_JUMP(o0, f48, 62f) |
2202 | |
2203 | 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) |
2204 | - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) |
2205 | + LOOP_CHUNK1(o1, o0, 1f) |
2206 | FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) |
2207 | - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) |
2208 | + LOOP_CHUNK2(o1, o0, 2f) |
2209 | FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) |
2210 | - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) |
2211 | + LOOP_CHUNK3(o1, o0, 3f) |
2212 | ba,pt %xcc, 1b+4 |
2213 | faligndata %f14, %f16, %f48 |
2214 | 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) |
2215 | @@ -407,53 +520,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2216 | FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) |
2217 | STORE_JUMP(o0, f48, 63f) |
2218 | |
2219 | -40: FINISH_VISCHUNK(o0, f0, f2, g3) |
2220 | -41: FINISH_VISCHUNK(o0, f2, f4, g3) |
2221 | -42: FINISH_VISCHUNK(o0, f4, f6, g3) |
2222 | -43: FINISH_VISCHUNK(o0, f6, f8, g3) |
2223 | -44: FINISH_VISCHUNK(o0, f8, f10, g3) |
2224 | -45: FINISH_VISCHUNK(o0, f10, f12, g3) |
2225 | -46: FINISH_VISCHUNK(o0, f12, f14, g3) |
2226 | -47: UNEVEN_VISCHUNK(o0, f14, f0, g3) |
2227 | -48: FINISH_VISCHUNK(o0, f16, f18, g3) |
2228 | -49: FINISH_VISCHUNK(o0, f18, f20, g3) |
2229 | -50: FINISH_VISCHUNK(o0, f20, f22, g3) |
2230 | -51: FINISH_VISCHUNK(o0, f22, f24, g3) |
2231 | -52: FINISH_VISCHUNK(o0, f24, f26, g3) |
2232 | -53: FINISH_VISCHUNK(o0, f26, f28, g3) |
2233 | -54: FINISH_VISCHUNK(o0, f28, f30, g3) |
2234 | -55: UNEVEN_VISCHUNK(o0, f30, f0, g3) |
2235 | -56: FINISH_VISCHUNK(o0, f32, f34, g3) |
2236 | -57: FINISH_VISCHUNK(o0, f34, f36, g3) |
2237 | -58: FINISH_VISCHUNK(o0, f36, f38, g3) |
2238 | -59: FINISH_VISCHUNK(o0, f38, f40, g3) |
2239 | -60: FINISH_VISCHUNK(o0, f40, f42, g3) |
2240 | -61: FINISH_VISCHUNK(o0, f42, f44, g3) |
2241 | -62: FINISH_VISCHUNK(o0, f44, f46, g3) |
2242 | -63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3) |
2243 | - |
2244 | -93: EX_LD_FP(LOAD(ldd, %o1, %f2)) |
2245 | +40: FINISH_VISCHUNK(o0, f0, f2) |
2246 | +41: FINISH_VISCHUNK(o0, f2, f4) |
2247 | +42: FINISH_VISCHUNK(o0, f4, f6) |
2248 | +43: FINISH_VISCHUNK(o0, f6, f8) |
2249 | +44: FINISH_VISCHUNK(o0, f8, f10) |
2250 | +45: FINISH_VISCHUNK(o0, f10, f12) |
2251 | +46: FINISH_VISCHUNK(o0, f12, f14) |
2252 | +47: UNEVEN_VISCHUNK(o0, f14, f0) |
2253 | +48: FINISH_VISCHUNK(o0, f16, f18) |
2254 | +49: FINISH_VISCHUNK(o0, f18, f20) |
2255 | +50: FINISH_VISCHUNK(o0, f20, f22) |
2256 | +51: FINISH_VISCHUNK(o0, f22, f24) |
2257 | +52: FINISH_VISCHUNK(o0, f24, f26) |
2258 | +53: FINISH_VISCHUNK(o0, f26, f28) |
2259 | +54: FINISH_VISCHUNK(o0, f28, f30) |
2260 | +55: UNEVEN_VISCHUNK(o0, f30, f0) |
2261 | +56: FINISH_VISCHUNK(o0, f32, f34) |
2262 | +57: FINISH_VISCHUNK(o0, f34, f36) |
2263 | +58: FINISH_VISCHUNK(o0, f36, f38) |
2264 | +59: FINISH_VISCHUNK(o0, f38, f40) |
2265 | +60: FINISH_VISCHUNK(o0, f40, f42) |
2266 | +61: FINISH_VISCHUNK(o0, f42, f44) |
2267 | +62: FINISH_VISCHUNK(o0, f44, f46) |
2268 | +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0) |
2269 | + |
2270 | +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp) |
2271 | add %o1, 8, %o1 |
2272 | subcc %g3, 8, %g3 |
2273 | faligndata %f0, %f2, %f8 |
2274 | - EX_ST_FP(STORE(std, %f8, %o0)) |
2275 | + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) |
2276 | bl,pn %xcc, 95f |
2277 | add %o0, 8, %o0 |
2278 | - EX_LD_FP(LOAD(ldd, %o1, %f0)) |
2279 | + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp) |
2280 | add %o1, 8, %o1 |
2281 | subcc %g3, 8, %g3 |
2282 | faligndata %f2, %f0, %f8 |
2283 | - EX_ST_FP(STORE(std, %f8, %o0)) |
2284 | + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) |
2285 | bge,pt %xcc, 93b |
2286 | add %o0, 8, %o0 |
2287 | |
2288 | 95: brz,pt %o2, 2f |
2289 | mov %g1, %o1 |
2290 | |
2291 | -1: EX_LD_FP(LOAD(ldub, %o1, %o3)) |
2292 | +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp) |
2293 | add %o1, 1, %o1 |
2294 | subcc %o2, 1, %o2 |
2295 | - EX_ST_FP(STORE(stb, %o3, %o0)) |
2296 | + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp) |
2297 | bne,pt %xcc, 1b |
2298 | add %o0, 1, %o0 |
2299 | |
2300 | @@ -469,27 +582,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2301 | |
2302 | 72: andn %o2, 0xf, %GLOBAL_SPARE |
2303 | and %o2, 0xf, %o2 |
2304 | -1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) |
2305 | - EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) |
2306 | +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0) |
2307 | + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0) |
2308 | subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE |
2309 | - EX_ST(STORE(stx, %o5, %o1 + %o3)) |
2310 | + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10) |
2311 | add %o1, 0x8, %o1 |
2312 | - EX_ST(STORE(stx, %g1, %o1 + %o3)) |
2313 | + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8) |
2314 | bgu,pt %XCC, 1b |
2315 | add %o1, 0x8, %o1 |
2316 | 73: andcc %o2, 0x8, %g0 |
2317 | be,pt %XCC, 1f |
2318 | nop |
2319 | - EX_LD(LOAD(ldx, %o1, %o5)) |
2320 | + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0) |
2321 | sub %o2, 0x8, %o2 |
2322 | - EX_ST(STORE(stx, %o5, %o1 + %o3)) |
2323 | + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8) |
2324 | add %o1, 0x8, %o1 |
2325 | 1: andcc %o2, 0x4, %g0 |
2326 | be,pt %XCC, 1f |
2327 | nop |
2328 | - EX_LD(LOAD(lduw, %o1, %o5)) |
2329 | + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0) |
2330 | sub %o2, 0x4, %o2 |
2331 | - EX_ST(STORE(stw, %o5, %o1 + %o3)) |
2332 | + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4) |
2333 | add %o1, 0x4, %o1 |
2334 | 1: cmp %o2, 0 |
2335 | be,pt %XCC, 85f |
2336 | @@ -503,9 +616,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2337 | sub %g0, %g1, %g1 |
2338 | sub %o2, %g1, %o2 |
2339 | |
2340 | -1: EX_LD(LOAD(ldub, %o1, %o5)) |
2341 | +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0) |
2342 | subcc %g1, 1, %g1 |
2343 | - EX_ST(STORE(stb, %o5, %o1 + %o3)) |
2344 | + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1) |
2345 | bgu,pt %icc, 1b |
2346 | add %o1, 1, %o1 |
2347 | |
2348 | @@ -521,16 +634,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2349 | |
2350 | 8: mov 64, %o3 |
2351 | andn %o1, 0x7, %o1 |
2352 | - EX_LD(LOAD(ldx, %o1, %g2)) |
2353 | + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0) |
2354 | sub %o3, %g1, %o3 |
2355 | andn %o2, 0x7, %GLOBAL_SPARE |
2356 | sllx %g2, %g1, %g2 |
2357 | -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) |
2358 | +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj) |
2359 | subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE |
2360 | add %o1, 0x8, %o1 |
2361 | srlx %g3, %o3, %o5 |
2362 | or %o5, %g2, %o5 |
2363 | - EX_ST(STORE(stx, %o5, %o0)) |
2364 | + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj) |
2365 | add %o0, 0x8, %o0 |
2366 | bgu,pt %icc, 1b |
2367 | sllx %g3, %g1, %g2 |
2368 | @@ -548,9 +661,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2369 | bne,pn %XCC, 90f |
2370 | sub %o0, %o1, %o3 |
2371 | |
2372 | -1: EX_LD(LOAD(lduw, %o1, %g1)) |
2373 | +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0) |
2374 | subcc %o2, 4, %o2 |
2375 | - EX_ST(STORE(stw, %g1, %o1 + %o3)) |
2376 | + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4) |
2377 | bgu,pt %XCC, 1b |
2378 | add %o1, 4, %o1 |
2379 | |
2380 | @@ -558,9 +671,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2381 | mov EX_RETVAL(%o4), %o0 |
2382 | |
2383 | .align 32 |
2384 | -90: EX_LD(LOAD(ldub, %o1, %g1)) |
2385 | +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0) |
2386 | subcc %o2, 1, %o2 |
2387 | - EX_ST(STORE(stb, %g1, %o1 + %o3)) |
2388 | + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1) |
2389 | bgu,pt %XCC, 90b |
2390 | add %o1, 1, %o1 |
2391 | retl |
2392 | diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S |
2393 | index 88ad73d86fe4..db73010a1af8 100644 |
2394 | --- a/arch/sparc/lib/U3copy_from_user.S |
2395 | +++ b/arch/sparc/lib/U3copy_from_user.S |
2396 | @@ -3,19 +3,19 @@ |
2397 | * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) |
2398 | */ |
2399 | |
2400 | -#define EX_LD(x) \ |
2401 | +#define EX_LD(x,y) \ |
2402 | 98: x; \ |
2403 | .section __ex_table,"a";\ |
2404 | .align 4; \ |
2405 | - .word 98b, __retl_one; \ |
2406 | + .word 98b, y; \ |
2407 | .text; \ |
2408 | .align 4; |
2409 | |
2410 | -#define EX_LD_FP(x) \ |
2411 | +#define EX_LD_FP(x,y) \ |
2412 | 98: x; \ |
2413 | .section __ex_table,"a";\ |
2414 | .align 4; \ |
2415 | - .word 98b, __retl_one_fp;\ |
2416 | + .word 98b, y##_fp; \ |
2417 | .text; \ |
2418 | .align 4; |
2419 | |
2420 | diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S |
2421 | index 845139d75537..c4ee858e352a 100644 |
2422 | --- a/arch/sparc/lib/U3copy_to_user.S |
2423 | +++ b/arch/sparc/lib/U3copy_to_user.S |
2424 | @@ -3,19 +3,19 @@ |
2425 | * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) |
2426 | */ |
2427 | |
2428 | -#define EX_ST(x) \ |
2429 | +#define EX_ST(x,y) \ |
2430 | 98: x; \ |
2431 | .section __ex_table,"a";\ |
2432 | .align 4; \ |
2433 | - .word 98b, __retl_one; \ |
2434 | + .word 98b, y; \ |
2435 | .text; \ |
2436 | .align 4; |
2437 | |
2438 | -#define EX_ST_FP(x) \ |
2439 | +#define EX_ST_FP(x,y) \ |
2440 | 98: x; \ |
2441 | .section __ex_table,"a";\ |
2442 | .align 4; \ |
2443 | - .word 98b, __retl_one_fp;\ |
2444 | + .word 98b, y##_fp; \ |
2445 | .text; \ |
2446 | .align 4; |
2447 | |
2448 | diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S |
2449 | index 491ee69e4995..54f98706b03b 100644 |
2450 | --- a/arch/sparc/lib/U3memcpy.S |
2451 | +++ b/arch/sparc/lib/U3memcpy.S |
2452 | @@ -4,6 +4,7 @@ |
2453 | */ |
2454 | |
2455 | #ifdef __KERNEL__ |
2456 | +#include <linux/linkage.h> |
2457 | #include <asm/visasm.h> |
2458 | #include <asm/asi.h> |
2459 | #define GLOBAL_SPARE %g7 |
2460 | @@ -22,21 +23,17 @@ |
2461 | #endif |
2462 | |
2463 | #ifndef EX_LD |
2464 | -#define EX_LD(x) x |
2465 | +#define EX_LD(x,y) x |
2466 | #endif |
2467 | #ifndef EX_LD_FP |
2468 | -#define EX_LD_FP(x) x |
2469 | +#define EX_LD_FP(x,y) x |
2470 | #endif |
2471 | |
2472 | #ifndef EX_ST |
2473 | -#define EX_ST(x) x |
2474 | +#define EX_ST(x,y) x |
2475 | #endif |
2476 | #ifndef EX_ST_FP |
2477 | -#define EX_ST_FP(x) x |
2478 | -#endif |
2479 | - |
2480 | -#ifndef EX_RETVAL |
2481 | -#define EX_RETVAL(x) x |
2482 | +#define EX_ST_FP(x,y) x |
2483 | #endif |
2484 | |
2485 | #ifndef LOAD |
2486 | @@ -77,6 +74,87 @@ |
2487 | */ |
2488 | |
2489 | .text |
2490 | +#ifndef EX_RETVAL |
2491 | +#define EX_RETVAL(x) x |
2492 | +__restore_fp: |
2493 | + VISExitHalf |
2494 | + retl |
2495 | + nop |
2496 | +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) |
2497 | + add %g1, 1, %g1 |
2498 | + add %g2, %g1, %g2 |
2499 | + ba,pt %xcc, __restore_fp |
2500 | + add %o2, %g2, %o0 |
2501 | +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) |
2502 | +ENTRY(U3_retl_o2_plus_g2_fp) |
2503 | + ba,pt %xcc, __restore_fp |
2504 | + add %o2, %g2, %o0 |
2505 | +ENDPROC(U3_retl_o2_plus_g2_fp) |
2506 | +ENTRY(U3_retl_o2_plus_g2_plus_8_fp) |
2507 | + add %g2, 8, %g2 |
2508 | + ba,pt %xcc, __restore_fp |
2509 | + add %o2, %g2, %o0 |
2510 | +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp) |
2511 | +ENTRY(U3_retl_o2) |
2512 | + retl |
2513 | + mov %o2, %o0 |
2514 | +ENDPROC(U3_retl_o2) |
2515 | +ENTRY(U3_retl_o2_plus_1) |
2516 | + retl |
2517 | + add %o2, 1, %o0 |
2518 | +ENDPROC(U3_retl_o2_plus_1) |
2519 | +ENTRY(U3_retl_o2_plus_4) |
2520 | + retl |
2521 | + add %o2, 4, %o0 |
2522 | +ENDPROC(U3_retl_o2_plus_4) |
2523 | +ENTRY(U3_retl_o2_plus_8) |
2524 | + retl |
2525 | + add %o2, 8, %o0 |
2526 | +ENDPROC(U3_retl_o2_plus_8) |
2527 | +ENTRY(U3_retl_o2_plus_g1_plus_1) |
2528 | + add %g1, 1, %g1 |
2529 | + retl |
2530 | + add %o2, %g1, %o0 |
2531 | +ENDPROC(U3_retl_o2_plus_g1_plus_1) |
2532 | +ENTRY(U3_retl_o2_fp) |
2533 | + ba,pt %xcc, __restore_fp |
2534 | + mov %o2, %o0 |
2535 | +ENDPROC(U3_retl_o2_fp) |
2536 | +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) |
2537 | + sll %o3, 6, %o3 |
2538 | + add %o3, 0x80, %o3 |
2539 | + ba,pt %xcc, __restore_fp |
2540 | + add %o2, %o3, %o0 |
2541 | +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) |
2542 | +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) |
2543 | + sll %o3, 6, %o3 |
2544 | + add %o3, 0x40, %o3 |
2545 | + ba,pt %xcc, __restore_fp |
2546 | + add %o2, %o3, %o0 |
2547 | +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) |
2548 | +ENTRY(U3_retl_o2_plus_GS_plus_0x10) |
2549 | + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE |
2550 | + retl |
2551 | + add %o2, GLOBAL_SPARE, %o0 |
2552 | +ENDPROC(U3_retl_o2_plus_GS_plus_0x10) |
2553 | +ENTRY(U3_retl_o2_plus_GS_plus_0x08) |
2554 | + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE |
2555 | + retl |
2556 | + add %o2, GLOBAL_SPARE, %o0 |
2557 | +ENDPROC(U3_retl_o2_plus_GS_plus_0x08) |
2558 | +ENTRY(U3_retl_o2_and_7_plus_GS) |
2559 | + and %o2, 7, %o2 |
2560 | + retl |
2561 | + add %o2, GLOBAL_SPARE, %o2 |
2562 | +ENDPROC(U3_retl_o2_and_7_plus_GS) |
2563 | +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8) |
2564 | + add GLOBAL_SPARE, 8, GLOBAL_SPARE |
2565 | + and %o2, 7, %o2 |
2566 | + retl |
2567 | + add %o2, GLOBAL_SPARE, %o2 |
2568 | +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8) |
2569 | +#endif |
2570 | + |
2571 | .align 64 |
2572 | |
2573 | /* The cheetah's flexible spine, oversized liver, enlarged heart, |
2574 | @@ -126,8 +204,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2575 | and %g2, 0x38, %g2 |
2576 | |
2577 | 1: subcc %g1, 0x1, %g1 |
2578 | - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3)) |
2579 | - EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE)) |
2580 | + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1) |
2581 | + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1) |
2582 | bgu,pt %XCC, 1b |
2583 | add %o1, 0x1, %o1 |
2584 | |
2585 | @@ -138,20 +216,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2586 | be,pt %icc, 3f |
2587 | alignaddr %o1, %g0, %o1 |
2588 | |
2589 | - EX_LD_FP(LOAD(ldd, %o1, %f4)) |
2590 | -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6)) |
2591 | + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2) |
2592 | +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2) |
2593 | add %o1, 0x8, %o1 |
2594 | subcc %g2, 0x8, %g2 |
2595 | faligndata %f4, %f6, %f0 |
2596 | - EX_ST_FP(STORE(std, %f0, %o0)) |
2597 | + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8) |
2598 | be,pn %icc, 3f |
2599 | add %o0, 0x8, %o0 |
2600 | |
2601 | - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4)) |
2602 | + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2) |
2603 | add %o1, 0x8, %o1 |
2604 | subcc %g2, 0x8, %g2 |
2605 | faligndata %f6, %f4, %f2 |
2606 | - EX_ST_FP(STORE(std, %f2, %o0)) |
2607 | + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8) |
2608 | bne,pt %icc, 1b |
2609 | add %o0, 0x8, %o0 |
2610 | |
2611 | @@ -161,25 +239,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2612 | LOAD(prefetch, %o1 + 0x080, #one_read) |
2613 | LOAD(prefetch, %o1 + 0x0c0, #one_read) |
2614 | LOAD(prefetch, %o1 + 0x100, #one_read) |
2615 | - EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0)) |
2616 | + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2) |
2617 | LOAD(prefetch, %o1 + 0x140, #one_read) |
2618 | - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) |
2619 | + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2) |
2620 | LOAD(prefetch, %o1 + 0x180, #one_read) |
2621 | - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) |
2622 | + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2) |
2623 | LOAD(prefetch, %o1 + 0x1c0, #one_read) |
2624 | faligndata %f0, %f2, %f16 |
2625 | - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) |
2626 | + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2) |
2627 | faligndata %f2, %f4, %f18 |
2628 | - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) |
2629 | + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2) |
2630 | faligndata %f4, %f6, %f20 |
2631 | - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) |
2632 | + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2) |
2633 | faligndata %f6, %f8, %f22 |
2634 | |
2635 | - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) |
2636 | + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2) |
2637 | faligndata %f8, %f10, %f24 |
2638 | - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) |
2639 | + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2) |
2640 | faligndata %f10, %f12, %f26 |
2641 | - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) |
2642 | + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2) |
2643 | |
2644 | subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE |
2645 | add %o1, 0x40, %o1 |
2646 | @@ -190,26 +268,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2647 | |
2648 | .align 64 |
2649 | 1: |
2650 | - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) |
2651 | + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) |
2652 | faligndata %f12, %f14, %f28 |
2653 | - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) |
2654 | + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) |
2655 | faligndata %f14, %f0, %f30 |
2656 | - EX_ST_FP(STORE_BLK(%f16, %o0)) |
2657 | - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) |
2658 | + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) |
2659 | + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2660 | faligndata %f0, %f2, %f16 |
2661 | add %o0, 0x40, %o0 |
2662 | |
2663 | - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) |
2664 | + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2665 | faligndata %f2, %f4, %f18 |
2666 | - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) |
2667 | + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2668 | faligndata %f4, %f6, %f20 |
2669 | - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) |
2670 | + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2671 | subcc %o3, 0x01, %o3 |
2672 | faligndata %f6, %f8, %f22 |
2673 | - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) |
2674 | + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80) |
2675 | |
2676 | faligndata %f8, %f10, %f24 |
2677 | - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) |
2678 | + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80) |
2679 | LOAD(prefetch, %o1 + 0x1c0, #one_read) |
2680 | faligndata %f10, %f12, %f26 |
2681 | bg,pt %XCC, 1b |
2682 | @@ -217,29 +295,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2683 | |
2684 | /* Finally we copy the last full 64-byte block. */ |
2685 | 2: |
2686 | - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) |
2687 | + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) |
2688 | faligndata %f12, %f14, %f28 |
2689 | - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) |
2690 | + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) |
2691 | faligndata %f14, %f0, %f30 |
2692 | - EX_ST_FP(STORE_BLK(%f16, %o0)) |
2693 | - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) |
2694 | + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) |
2695 | + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2696 | faligndata %f0, %f2, %f16 |
2697 | - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) |
2698 | + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2699 | faligndata %f2, %f4, %f18 |
2700 | - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) |
2701 | + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2702 | faligndata %f4, %f6, %f20 |
2703 | - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) |
2704 | + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2705 | faligndata %f6, %f8, %f22 |
2706 | - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) |
2707 | + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2708 | faligndata %f8, %f10, %f24 |
2709 | cmp %g1, 0 |
2710 | be,pt %XCC, 1f |
2711 | add %o0, 0x40, %o0 |
2712 | - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) |
2713 | + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2714 | 1: faligndata %f10, %f12, %f26 |
2715 | faligndata %f12, %f14, %f28 |
2716 | faligndata %f14, %f0, %f30 |
2717 | - EX_ST_FP(STORE_BLK(%f16, %o0)) |
2718 | + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40) |
2719 | add %o0, 0x40, %o0 |
2720 | add %o1, 0x40, %o1 |
2721 | membar #Sync |
2722 | @@ -259,20 +337,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2723 | |
2724 | sub %o2, %g2, %o2 |
2725 | be,a,pt %XCC, 1f |
2726 | - EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0)) |
2727 | + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2) |
2728 | |
2729 | -1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2)) |
2730 | +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2) |
2731 | add %o1, 0x8, %o1 |
2732 | subcc %g2, 0x8, %g2 |
2733 | faligndata %f0, %f2, %f8 |
2734 | - EX_ST_FP(STORE(std, %f8, %o0)) |
2735 | + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) |
2736 | be,pn %XCC, 2f |
2737 | add %o0, 0x8, %o0 |
2738 | - EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0)) |
2739 | + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2) |
2740 | add %o1, 0x8, %o1 |
2741 | subcc %g2, 0x8, %g2 |
2742 | faligndata %f2, %f0, %f8 |
2743 | - EX_ST_FP(STORE(std, %f8, %o0)) |
2744 | + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) |
2745 | bne,pn %XCC, 1b |
2746 | add %o0, 0x8, %o0 |
2747 | |
2748 | @@ -292,30 +370,33 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2749 | andcc %o2, 0x8, %g0 |
2750 | be,pt %icc, 1f |
2751 | nop |
2752 | - EX_LD(LOAD(ldx, %o1, %o5)) |
2753 | - EX_ST(STORE(stx, %o5, %o1 + %o3)) |
2754 | + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2) |
2755 | + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2) |
2756 | add %o1, 0x8, %o1 |
2757 | + sub %o2, 8, %o2 |
2758 | |
2759 | 1: andcc %o2, 0x4, %g0 |
2760 | be,pt %icc, 1f |
2761 | nop |
2762 | - EX_LD(LOAD(lduw, %o1, %o5)) |
2763 | - EX_ST(STORE(stw, %o5, %o1 + %o3)) |
2764 | + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2) |
2765 | + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2) |
2766 | add %o1, 0x4, %o1 |
2767 | + sub %o2, 4, %o2 |
2768 | |
2769 | 1: andcc %o2, 0x2, %g0 |
2770 | be,pt %icc, 1f |
2771 | nop |
2772 | - EX_LD(LOAD(lduh, %o1, %o5)) |
2773 | - EX_ST(STORE(sth, %o5, %o1 + %o3)) |
2774 | + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2) |
2775 | + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2) |
2776 | add %o1, 0x2, %o1 |
2777 | + sub %o2, 2, %o2 |
2778 | |
2779 | 1: andcc %o2, 0x1, %g0 |
2780 | be,pt %icc, 85f |
2781 | nop |
2782 | - EX_LD(LOAD(ldub, %o1, %o5)) |
2783 | + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2) |
2784 | ba,pt %xcc, 85f |
2785 | - EX_ST(STORE(stb, %o5, %o1 + %o3)) |
2786 | + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2) |
2787 | |
2788 | .align 64 |
2789 | 70: /* 16 < len <= 64 */ |
2790 | @@ -326,26 +407,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2791 | andn %o2, 0xf, GLOBAL_SPARE |
2792 | and %o2, 0xf, %o2 |
2793 | 1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE |
2794 | - EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) |
2795 | - EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) |
2796 | - EX_ST(STORE(stx, %o5, %o1 + %o3)) |
2797 | + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10) |
2798 | + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10) |
2799 | + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10) |
2800 | add %o1, 0x8, %o1 |
2801 | - EX_ST(STORE(stx, %g1, %o1 + %o3)) |
2802 | + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08) |
2803 | bgu,pt %XCC, 1b |
2804 | add %o1, 0x8, %o1 |
2805 | 73: andcc %o2, 0x8, %g0 |
2806 | be,pt %XCC, 1f |
2807 | nop |
2808 | sub %o2, 0x8, %o2 |
2809 | - EX_LD(LOAD(ldx, %o1, %o5)) |
2810 | - EX_ST(STORE(stx, %o5, %o1 + %o3)) |
2811 | + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8) |
2812 | + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8) |
2813 | add %o1, 0x8, %o1 |
2814 | 1: andcc %o2, 0x4, %g0 |
2815 | be,pt %XCC, 1f |
2816 | nop |
2817 | sub %o2, 0x4, %o2 |
2818 | - EX_LD(LOAD(lduw, %o1, %o5)) |
2819 | - EX_ST(STORE(stw, %o5, %o1 + %o3)) |
2820 | + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4) |
2821 | + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4) |
2822 | add %o1, 0x4, %o1 |
2823 | 1: cmp %o2, 0 |
2824 | be,pt %XCC, 85f |
2825 | @@ -361,8 +442,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2826 | sub %o2, %g1, %o2 |
2827 | |
2828 | 1: subcc %g1, 1, %g1 |
2829 | - EX_LD(LOAD(ldub, %o1, %o5)) |
2830 | - EX_ST(STORE(stb, %o5, %o1 + %o3)) |
2831 | + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1) |
2832 | + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1) |
2833 | bgu,pt %icc, 1b |
2834 | add %o1, 1, %o1 |
2835 | |
2836 | @@ -378,16 +459,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2837 | |
2838 | 8: mov 64, %o3 |
2839 | andn %o1, 0x7, %o1 |
2840 | - EX_LD(LOAD(ldx, %o1, %g2)) |
2841 | + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2) |
2842 | sub %o3, %g1, %o3 |
2843 | andn %o2, 0x7, GLOBAL_SPARE |
2844 | sllx %g2, %g1, %g2 |
2845 | -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) |
2846 | +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS) |
2847 | subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE |
2848 | add %o1, 0x8, %o1 |
2849 | srlx %g3, %o3, %o5 |
2850 | or %o5, %g2, %o5 |
2851 | - EX_ST(STORE(stx, %o5, %o0)) |
2852 | + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8) |
2853 | add %o0, 0x8, %o0 |
2854 | bgu,pt %icc, 1b |
2855 | sllx %g3, %g1, %g2 |
2856 | @@ -407,8 +488,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2857 | |
2858 | 1: |
2859 | subcc %o2, 4, %o2 |
2860 | - EX_LD(LOAD(lduw, %o1, %g1)) |
2861 | - EX_ST(STORE(stw, %g1, %o1 + %o3)) |
2862 | + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4) |
2863 | + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4) |
2864 | bgu,pt %XCC, 1b |
2865 | add %o1, 4, %o1 |
2866 | |
2867 | @@ -418,8 +499,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ |
2868 | .align 32 |
2869 | 90: |
2870 | subcc %o2, 1, %o2 |
2871 | - EX_LD(LOAD(ldub, %o1, %g1)) |
2872 | - EX_ST(STORE(stb, %g1, %o1 + %o3)) |
2873 | + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1) |
2874 | + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1) |
2875 | bgu,pt %XCC, 90b |
2876 | add %o1, 1, %o1 |
2877 | retl |
2878 | diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S |
2879 | index 302c0e60dc2c..4c89b486fa0d 100644 |
2880 | --- a/arch/sparc/lib/copy_in_user.S |
2881 | +++ b/arch/sparc/lib/copy_in_user.S |
2882 | @@ -8,18 +8,33 @@ |
2883 | |
2884 | #define XCC xcc |
2885 | |
2886 | -#define EX(x,y) \ |
2887 | +#define EX(x,y,z) \ |
2888 | 98: x,y; \ |
2889 | .section __ex_table,"a";\ |
2890 | .align 4; \ |
2891 | - .word 98b, __retl_one; \ |
2892 | + .word 98b, z; \ |
2893 | .text; \ |
2894 | .align 4; |
2895 | |
2896 | +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8) |
2897 | +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4) |
2898 | +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1) |
2899 | + |
2900 | .register %g2,#scratch |
2901 | .register %g3,#scratch |
2902 | |
2903 | .text |
2904 | +__retl_o4_plus_8: |
2905 | + add %o4, %o2, %o4 |
2906 | + retl |
2907 | + add %o4, 8, %o0 |
2908 | +__retl_o2_plus_4: |
2909 | + retl |
2910 | + add %o2, 4, %o0 |
2911 | +__retl_o2_plus_1: |
2912 | + retl |
2913 | + add %o2, 1, %o0 |
2914 | + |
2915 | .align 32 |
2916 | |
2917 | /* Don't try to get too fancy here, just nice and |
2918 | @@ -44,8 +59,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ |
2919 | andn %o2, 0x7, %o4 |
2920 | and %o2, 0x7, %o2 |
2921 | 1: subcc %o4, 0x8, %o4 |
2922 | - EX(ldxa [%o1] %asi, %o5) |
2923 | - EX(stxa %o5, [%o0] %asi) |
2924 | + EX_O4(ldxa [%o1] %asi, %o5) |
2925 | + EX_O4(stxa %o5, [%o0] %asi) |
2926 | add %o1, 0x8, %o1 |
2927 | bgu,pt %XCC, 1b |
2928 | add %o0, 0x8, %o0 |
2929 | @@ -53,8 +68,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ |
2930 | be,pt %XCC, 1f |
2931 | nop |
2932 | sub %o2, 0x4, %o2 |
2933 | - EX(lduwa [%o1] %asi, %o5) |
2934 | - EX(stwa %o5, [%o0] %asi) |
2935 | + EX_O2_4(lduwa [%o1] %asi, %o5) |
2936 | + EX_O2_4(stwa %o5, [%o0] %asi) |
2937 | add %o1, 0x4, %o1 |
2938 | add %o0, 0x4, %o0 |
2939 | 1: cmp %o2, 0 |
2940 | @@ -70,8 +85,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ |
2941 | |
2942 | 82: |
2943 | subcc %o2, 4, %o2 |
2944 | - EX(lduwa [%o1] %asi, %g1) |
2945 | - EX(stwa %g1, [%o0] %asi) |
2946 | + EX_O2_4(lduwa [%o1] %asi, %g1) |
2947 | + EX_O2_4(stwa %g1, [%o0] %asi) |
2948 | add %o1, 4, %o1 |
2949 | bgu,pt %XCC, 82b |
2950 | add %o0, 4, %o0 |
2951 | @@ -82,8 +97,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ |
2952 | .align 32 |
2953 | 90: |
2954 | subcc %o2, 1, %o2 |
2955 | - EX(lduba [%o1] %asi, %g1) |
2956 | - EX(stba %g1, [%o0] %asi) |
2957 | + EX_O2_1(lduba [%o1] %asi, %g1) |
2958 | + EX_O2_1(stba %g1, [%o0] %asi) |
2959 | add %o1, 1, %o1 |
2960 | bgu,pt %XCC, 90b |
2961 | add %o0, 1, %o0 |
2962 | diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c |
2963 | deleted file mode 100644 |
2964 | index ac96ae236709..000000000000 |
2965 | --- a/arch/sparc/lib/user_fixup.c |
2966 | +++ /dev/null |
2967 | @@ -1,71 +0,0 @@ |
2968 | -/* user_fixup.c: Fix up user copy faults. |
2969 | - * |
2970 | - * Copyright (C) 2004 David S. Miller <davem@redhat.com> |
2971 | - */ |
2972 | - |
2973 | -#include <linux/compiler.h> |
2974 | -#include <linux/kernel.h> |
2975 | -#include <linux/string.h> |
2976 | -#include <linux/errno.h> |
2977 | -#include <linux/module.h> |
2978 | - |
2979 | -#include <asm/uaccess.h> |
2980 | - |
2981 | -/* Calculating the exact fault address when using |
2982 | - * block loads and stores can be very complicated. |
2983 | - * |
2984 | - * Instead of trying to be clever and handling all |
2985 | - * of the cases, just fix things up simply here. |
2986 | - */ |
2987 | - |
2988 | -static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset) |
2989 | -{ |
2990 | - unsigned long fault_addr = current_thread_info()->fault_address; |
2991 | - unsigned long end = start + size; |
2992 | - |
2993 | - if (fault_addr < start || fault_addr >= end) { |
2994 | - *offset = 0; |
2995 | - } else { |
2996 | - *offset = fault_addr - start; |
2997 | - size = end - fault_addr; |
2998 | - } |
2999 | - return size; |
3000 | -} |
3001 | - |
3002 | -unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size) |
3003 | -{ |
3004 | - unsigned long offset; |
3005 | - |
3006 | - size = compute_size((unsigned long) from, size, &offset); |
3007 | - if (likely(size)) |
3008 | - memset(to + offset, 0, size); |
3009 | - |
3010 | - return size; |
3011 | -} |
3012 | -EXPORT_SYMBOL(copy_from_user_fixup); |
3013 | - |
3014 | -unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size) |
3015 | -{ |
3016 | - unsigned long offset; |
3017 | - |
3018 | - return compute_size((unsigned long) to, size, &offset); |
3019 | -} |
3020 | -EXPORT_SYMBOL(copy_to_user_fixup); |
3021 | - |
3022 | -unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size) |
3023 | -{ |
3024 | - unsigned long fault_addr = current_thread_info()->fault_address; |
3025 | - unsigned long start = (unsigned long) to; |
3026 | - unsigned long end = start + size; |
3027 | - |
3028 | - if (fault_addr >= start && fault_addr < end) |
3029 | - return end - fault_addr; |
3030 | - |
3031 | - start = (unsigned long) from; |
3032 | - end = start + size; |
3033 | - if (fault_addr >= start && fault_addr < end) |
3034 | - return end - fault_addr; |
3035 | - |
3036 | - return size; |
3037 | -} |
3038 | -EXPORT_SYMBOL(copy_in_user_fixup); |
3039 | diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c |
3040 | index f2b77112e9d8..e20fbbafb0b0 100644 |
3041 | --- a/arch/sparc/mm/tsb.c |
3042 | +++ b/arch/sparc/mm/tsb.c |
3043 | @@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr) |
3044 | return (tag == (vaddr >> 22)); |
3045 | } |
3046 | |
3047 | +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end) |
3048 | +{ |
3049 | + unsigned long idx; |
3050 | + |
3051 | + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) { |
3052 | + struct tsb *ent = &swapper_tsb[idx]; |
3053 | + unsigned long match = idx << 13; |
3054 | + |
3055 | + match |= (ent->tag << 22); |
3056 | + if (match >= start && match < end) |
3057 | + ent->tag = (1UL << TSB_TAG_INVALID_BIT); |
3058 | + } |
3059 | +} |
3060 | + |
3061 | /* TSB flushes need only occur on the processor initiating the address |
3062 | * space modification, not on each cpu the address space has run on. |
3063 | * Only the TLB flush needs that treatment. |
3064 | @@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) |
3065 | { |
3066 | unsigned long v; |
3067 | |
3068 | + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES) |
3069 | + return flush_tsb_kernel_range_scan(start, end); |
3070 | + |
3071 | for (v = start; v < end; v += PAGE_SIZE) { |
3072 | unsigned long hash = tsb_hash(v, PAGE_SHIFT, |
3073 | KERNEL_TSB_NENTRIES); |
3074 | diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S |
3075 | index b4f4733abc6e..5d2fd6cd3189 100644 |
3076 | --- a/arch/sparc/mm/ultra.S |
3077 | +++ b/arch/sparc/mm/ultra.S |
3078 | @@ -30,7 +30,7 @@ |
3079 | .text |
3080 | .align 32 |
3081 | .globl __flush_tlb_mm |
3082 | -__flush_tlb_mm: /* 18 insns */ |
3083 | +__flush_tlb_mm: /* 19 insns */ |
3084 | /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ |
3085 | ldxa [%o1] ASI_DMMU, %g2 |
3086 | cmp %g2, %o0 |
3087 | @@ -81,7 +81,7 @@ __flush_tlb_page: /* 22 insns */ |
3088 | |
3089 | .align 32 |
3090 | .globl __flush_tlb_pending |
3091 | -__flush_tlb_pending: /* 26 insns */ |
3092 | +__flush_tlb_pending: /* 27 insns */ |
3093 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
3094 | rdpr %pstate, %g7 |
3095 | sllx %o1, 3, %o1 |
3096 | @@ -113,12 +113,14 @@ __flush_tlb_pending: /* 26 insns */ |
3097 | |
3098 | .align 32 |
3099 | .globl __flush_tlb_kernel_range |
3100 | -__flush_tlb_kernel_range: /* 16 insns */ |
3101 | +__flush_tlb_kernel_range: /* 31 insns */ |
3102 | /* %o0=start, %o1=end */ |
3103 | cmp %o0, %o1 |
3104 | be,pn %xcc, 2f |
3105 | + sub %o1, %o0, %o3 |
3106 | + srlx %o3, 18, %o4 |
3107 | + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow |
3108 | sethi %hi(PAGE_SIZE), %o4 |
3109 | - sub %o1, %o0, %o3 |
3110 | sub %o3, %o4, %o3 |
3111 | or %o0, 0x20, %o0 ! Nucleus |
3112 | 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP |
3113 | @@ -131,6 +133,41 @@ __flush_tlb_kernel_range: /* 16 insns */ |
3114 | retl |
3115 | nop |
3116 | nop |
3117 | + nop |
3118 | + nop |
3119 | + nop |
3120 | + nop |
3121 | + nop |
3122 | + nop |
3123 | + nop |
3124 | + nop |
3125 | + nop |
3126 | + nop |
3127 | + nop |
3128 | + nop |
3129 | + nop |
3130 | + |
3131 | +__spitfire_flush_tlb_kernel_range_slow: |
3132 | + mov 63 * 8, %o4 |
3133 | +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 |
3134 | + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ |
3135 | + bne,pn %xcc, 2f |
3136 | + mov TLB_TAG_ACCESS, %o3 |
3137 | + stxa %g0, [%o3] ASI_IMMU |
3138 | + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS |
3139 | + membar #Sync |
3140 | +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 |
3141 | + andcc %o3, 0x40, %g0 |
3142 | + bne,pn %xcc, 2f |
3143 | + mov TLB_TAG_ACCESS, %o3 |
3144 | + stxa %g0, [%o3] ASI_DMMU |
3145 | + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS |
3146 | + membar #Sync |
3147 | +2: sub %o4, 8, %o4 |
3148 | + brgez,pt %o4, 1b |
3149 | + nop |
3150 | + retl |
3151 | + nop |
3152 | |
3153 | __spitfire_flush_tlb_mm_slow: |
3154 | rdpr %pstate, %g1 |
3155 | @@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */ |
3156 | retl |
3157 | wrpr %g7, 0x0, %pstate |
3158 | |
3159 | +__cheetah_flush_tlb_kernel_range: /* 31 insns */ |
3160 | + /* %o0=start, %o1=end */ |
3161 | + cmp %o0, %o1 |
3162 | + be,pn %xcc, 2f |
3163 | + sub %o1, %o0, %o3 |
3164 | + srlx %o3, 18, %o4 |
3165 | + brnz,pn %o4, 3f |
3166 | + sethi %hi(PAGE_SIZE), %o4 |
3167 | + sub %o3, %o4, %o3 |
3168 | + or %o0, 0x20, %o0 ! Nucleus |
3169 | +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP |
3170 | + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP |
3171 | + membar #Sync |
3172 | + brnz,pt %o3, 1b |
3173 | + sub %o3, %o4, %o3 |
3174 | +2: sethi %hi(KERNBASE), %o3 |
3175 | + flush %o3 |
3176 | + retl |
3177 | + nop |
3178 | +3: mov 0x80, %o4 |
3179 | + stxa %g0, [%o4] ASI_DMMU_DEMAP |
3180 | + membar #Sync |
3181 | + stxa %g0, [%o4] ASI_IMMU_DEMAP |
3182 | + membar #Sync |
3183 | + retl |
3184 | + nop |
3185 | + nop |
3186 | + nop |
3187 | + nop |
3188 | + nop |
3189 | + nop |
3190 | + nop |
3191 | + nop |
3192 | + |
3193 | #ifdef DCACHE_ALIASING_POSSIBLE |
3194 | __cheetah_flush_dcache_page: /* 11 insns */ |
3195 | sethi %hi(PAGE_OFFSET), %g1 |
3196 | @@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error: |
3197 | ret |
3198 | restore |
3199 | |
3200 | -__hypervisor_flush_tlb_mm: /* 10 insns */ |
3201 | +__hypervisor_flush_tlb_mm: /* 19 insns */ |
3202 | mov %o0, %o2 /* ARG2: mmu context */ |
3203 | mov 0, %o0 /* ARG0: CPU lists unimplemented */ |
3204 | mov 0, %o1 /* ARG1: CPU lists unimplemented */ |
3205 | mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
3206 | mov HV_FAST_MMU_DEMAP_CTX, %o5 |
3207 | ta HV_FAST_TRAP |
3208 | - brnz,pn %o0, __hypervisor_tlb_tl0_error |
3209 | + brnz,pn %o0, 1f |
3210 | mov HV_FAST_MMU_DEMAP_CTX, %o1 |
3211 | retl |
3212 | nop |
3213 | +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5 |
3214 | + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 |
3215 | + nop |
3216 | + nop |
3217 | + nop |
3218 | + nop |
3219 | + nop |
3220 | + nop |
3221 | + nop |
3222 | |
3223 | -__hypervisor_flush_tlb_page: /* 11 insns */ |
3224 | +__hypervisor_flush_tlb_page: /* 22 insns */ |
3225 | /* %o0 = context, %o1 = vaddr */ |
3226 | mov %o0, %g2 |
3227 | mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ |
3228 | @@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */ |
3229 | srlx %o0, PAGE_SHIFT, %o0 |
3230 | sllx %o0, PAGE_SHIFT, %o0 |
3231 | ta HV_MMU_UNMAP_ADDR_TRAP |
3232 | - brnz,pn %o0, __hypervisor_tlb_tl0_error |
3233 | + brnz,pn %o0, 1f |
3234 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
3235 | retl |
3236 | nop |
3237 | +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
3238 | + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 |
3239 | + nop |
3240 | + nop |
3241 | + nop |
3242 | + nop |
3243 | + nop |
3244 | + nop |
3245 | + nop |
3246 | + nop |
3247 | + nop |
3248 | |
3249 | -__hypervisor_flush_tlb_pending: /* 16 insns */ |
3250 | +__hypervisor_flush_tlb_pending: /* 27 insns */ |
3251 | /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ |
3252 | sllx %o1, 3, %g1 |
3253 | mov %o2, %g2 |
3254 | @@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */ |
3255 | srlx %o0, PAGE_SHIFT, %o0 |
3256 | sllx %o0, PAGE_SHIFT, %o0 |
3257 | ta HV_MMU_UNMAP_ADDR_TRAP |
3258 | - brnz,pn %o0, __hypervisor_tlb_tl0_error |
3259 | + brnz,pn %o0, 1f |
3260 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
3261 | brnz,pt %g1, 1b |
3262 | nop |
3263 | retl |
3264 | nop |
3265 | +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
3266 | + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 |
3267 | + nop |
3268 | + nop |
3269 | + nop |
3270 | + nop |
3271 | + nop |
3272 | + nop |
3273 | + nop |
3274 | + nop |
3275 | + nop |
3276 | |
3277 | -__hypervisor_flush_tlb_kernel_range: /* 16 insns */ |
3278 | +__hypervisor_flush_tlb_kernel_range: /* 31 insns */ |
3279 | /* %o0=start, %o1=end */ |
3280 | cmp %o0, %o1 |
3281 | be,pn %xcc, 2f |
3282 | - sethi %hi(PAGE_SIZE), %g3 |
3283 | - mov %o0, %g1 |
3284 | - sub %o1, %g1, %g2 |
3285 | + sub %o1, %o0, %g2 |
3286 | + srlx %g2, 18, %g3 |
3287 | + brnz,pn %g3, 4f |
3288 | + mov %o0, %g1 |
3289 | + sethi %hi(PAGE_SIZE), %g3 |
3290 | sub %g2, %g3, %g2 |
3291 | 1: add %g1, %g2, %o0 /* ARG0: virtual address */ |
3292 | mov 0, %o1 /* ARG1: mmu context */ |
3293 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
3294 | ta HV_MMU_UNMAP_ADDR_TRAP |
3295 | - brnz,pn %o0, __hypervisor_tlb_tl0_error |
3296 | + brnz,pn %o0, 3f |
3297 | mov HV_MMU_UNMAP_ADDR_TRAP, %o1 |
3298 | brnz,pt %g2, 1b |
3299 | sub %g2, %g3, %g2 |
3300 | 2: retl |
3301 | nop |
3302 | +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 |
3303 | + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 |
3304 | + nop |
3305 | +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */ |
3306 | + mov 0, %o1 /* ARG1: CPU lists unimplemented */ |
3307 | + mov 0, %o2 /* ARG2: mmu context == nucleus */ |
3308 | + mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
3309 | + mov HV_FAST_MMU_DEMAP_CTX, %o5 |
3310 | + ta HV_FAST_TRAP |
3311 | + brnz,pn %o0, 3b |
3312 | + mov HV_FAST_MMU_DEMAP_CTX, %o1 |
3313 | + retl |
3314 | + nop |
3315 | |
3316 | #ifdef DCACHE_ALIASING_POSSIBLE |
3317 | /* XXX Niagara and friends have an 8K cache, so no aliasing is |
3318 | @@ -394,43 +511,6 @@ tlb_patch_one: |
3319 | retl |
3320 | nop |
3321 | |
3322 | - .globl cheetah_patch_cachetlbops |
3323 | -cheetah_patch_cachetlbops: |
3324 | - save %sp, -128, %sp |
3325 | - |
3326 | - sethi %hi(__flush_tlb_mm), %o0 |
3327 | - or %o0, %lo(__flush_tlb_mm), %o0 |
3328 | - sethi %hi(__cheetah_flush_tlb_mm), %o1 |
3329 | - or %o1, %lo(__cheetah_flush_tlb_mm), %o1 |
3330 | - call tlb_patch_one |
3331 | - mov 19, %o2 |
3332 | - |
3333 | - sethi %hi(__flush_tlb_page), %o0 |
3334 | - or %o0, %lo(__flush_tlb_page), %o0 |
3335 | - sethi %hi(__cheetah_flush_tlb_page), %o1 |
3336 | - or %o1, %lo(__cheetah_flush_tlb_page), %o1 |
3337 | - call tlb_patch_one |
3338 | - mov 22, %o2 |
3339 | - |
3340 | - sethi %hi(__flush_tlb_pending), %o0 |
3341 | - or %o0, %lo(__flush_tlb_pending), %o0 |
3342 | - sethi %hi(__cheetah_flush_tlb_pending), %o1 |
3343 | - or %o1, %lo(__cheetah_flush_tlb_pending), %o1 |
3344 | - call tlb_patch_one |
3345 | - mov 27, %o2 |
3346 | - |
3347 | -#ifdef DCACHE_ALIASING_POSSIBLE |
3348 | - sethi %hi(__flush_dcache_page), %o0 |
3349 | - or %o0, %lo(__flush_dcache_page), %o0 |
3350 | - sethi %hi(__cheetah_flush_dcache_page), %o1 |
3351 | - or %o1, %lo(__cheetah_flush_dcache_page), %o1 |
3352 | - call tlb_patch_one |
3353 | - mov 11, %o2 |
3354 | -#endif /* DCACHE_ALIASING_POSSIBLE */ |
3355 | - |
3356 | - ret |
3357 | - restore |
3358 | - |
3359 | #ifdef CONFIG_SMP |
3360 | /* These are all called by the slaves of a cross call, at |
3361 | * trap level 1, with interrupts fully disabled. |
3362 | @@ -447,7 +527,7 @@ cheetah_patch_cachetlbops: |
3363 | */ |
3364 | .align 32 |
3365 | .globl xcall_flush_tlb_mm |
3366 | -xcall_flush_tlb_mm: /* 21 insns */ |
3367 | +xcall_flush_tlb_mm: /* 24 insns */ |
3368 | mov PRIMARY_CONTEXT, %g2 |
3369 | ldxa [%g2] ASI_DMMU, %g3 |
3370 | srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 |
3371 | @@ -469,9 +549,12 @@ xcall_flush_tlb_mm: /* 21 insns */ |
3372 | nop |
3373 | nop |
3374 | nop |
3375 | + nop |
3376 | + nop |
3377 | + nop |
3378 | |
3379 | .globl xcall_flush_tlb_page |
3380 | -xcall_flush_tlb_page: /* 17 insns */ |
3381 | +xcall_flush_tlb_page: /* 20 insns */ |
3382 | /* %g5=context, %g1=vaddr */ |
3383 | mov PRIMARY_CONTEXT, %g4 |
3384 | ldxa [%g4] ASI_DMMU, %g2 |
3385 | @@ -490,15 +573,20 @@ xcall_flush_tlb_page: /* 17 insns */ |
3386 | retry |
3387 | nop |
3388 | nop |
3389 | + nop |
3390 | + nop |
3391 | + nop |
3392 | |
3393 | .globl xcall_flush_tlb_kernel_range |
3394 | -xcall_flush_tlb_kernel_range: /* 25 insns */ |
3395 | +xcall_flush_tlb_kernel_range: /* 44 insns */ |
3396 | sethi %hi(PAGE_SIZE - 1), %g2 |
3397 | or %g2, %lo(PAGE_SIZE - 1), %g2 |
3398 | andn %g1, %g2, %g1 |
3399 | andn %g7, %g2, %g7 |
3400 | sub %g7, %g1, %g3 |
3401 | - add %g2, 1, %g2 |
3402 | + srlx %g3, 18, %g2 |
3403 | + brnz,pn %g2, 2f |
3404 | + add %g2, 1, %g2 |
3405 | sub %g3, %g2, %g3 |
3406 | or %g1, 0x20, %g1 ! Nucleus |
3407 | 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP |
3408 | @@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */ |
3409 | brnz,pt %g3, 1b |
3410 | sub %g3, %g2, %g3 |
3411 | retry |
3412 | - nop |
3413 | - nop |
3414 | +2: mov 63 * 8, %g1 |
3415 | +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 |
3416 | + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ |
3417 | + bne,pn %xcc, 2f |
3418 | + mov TLB_TAG_ACCESS, %g2 |
3419 | + stxa %g0, [%g2] ASI_IMMU |
3420 | + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS |
3421 | + membar #Sync |
3422 | +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 |
3423 | + andcc %g2, 0x40, %g0 |
3424 | + bne,pn %xcc, 2f |
3425 | + mov TLB_TAG_ACCESS, %g2 |
3426 | + stxa %g0, [%g2] ASI_DMMU |
3427 | + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS |
3428 | + membar #Sync |
3429 | +2: sub %g1, 8, %g1 |
3430 | + brgez,pt %g1, 1b |
3431 | + nop |
3432 | + retry |
3433 | nop |
3434 | nop |
3435 | nop |
3436 | @@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4: |
3437 | |
3438 | retry |
3439 | |
3440 | +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ |
3441 | + sethi %hi(PAGE_SIZE - 1), %g2 |
3442 | + or %g2, %lo(PAGE_SIZE - 1), %g2 |
3443 | + andn %g1, %g2, %g1 |
3444 | + andn %g7, %g2, %g7 |
3445 | + sub %g7, %g1, %g3 |
3446 | + srlx %g3, 18, %g2 |
3447 | + brnz,pn %g2, 2f |
3448 | + add %g2, 1, %g2 |
3449 | + sub %g3, %g2, %g3 |
3450 | + or %g1, 0x20, %g1 ! Nucleus |
3451 | +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP |
3452 | + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP |
3453 | + membar #Sync |
3454 | + brnz,pt %g3, 1b |
3455 | + sub %g3, %g2, %g3 |
3456 | + retry |
3457 | +2: mov 0x80, %g2 |
3458 | + stxa %g0, [%g2] ASI_DMMU_DEMAP |
3459 | + membar #Sync |
3460 | + stxa %g0, [%g2] ASI_IMMU_DEMAP |
3461 | + membar #Sync |
3462 | + retry |
3463 | + nop |
3464 | + nop |
3465 | + nop |
3466 | + nop |
3467 | + nop |
3468 | + nop |
3469 | + nop |
3470 | + nop |
3471 | + nop |
3472 | + nop |
3473 | + nop |
3474 | + nop |
3475 | + nop |
3476 | + nop |
3477 | + nop |
3478 | + nop |
3479 | + nop |
3480 | + nop |
3481 | + nop |
3482 | + nop |
3483 | + nop |
3484 | + nop |
3485 | + |
3486 | #ifdef DCACHE_ALIASING_POSSIBLE |
3487 | .align 32 |
3488 | .globl xcall_flush_dcache_page_cheetah |
3489 | @@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error: |
3490 | ba,a,pt %xcc, rtrap |
3491 | |
3492 | .globl __hypervisor_xcall_flush_tlb_mm |
3493 | -__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ |
3494 | +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */ |
3495 | /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ |
3496 | mov %o0, %g2 |
3497 | mov %o1, %g3 |
3498 | @@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ |
3499 | mov HV_FAST_MMU_DEMAP_CTX, %o5 |
3500 | ta HV_FAST_TRAP |
3501 | mov HV_FAST_MMU_DEMAP_CTX, %g6 |
3502 | - brnz,pn %o0, __hypervisor_tlb_xcall_error |
3503 | + brnz,pn %o0, 1f |
3504 | mov %o0, %g5 |
3505 | mov %g2, %o0 |
3506 | mov %g3, %o1 |
3507 | @@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ |
3508 | mov %g7, %o5 |
3509 | membar #Sync |
3510 | retry |
3511 | +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
3512 | + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 |
3513 | + nop |
3514 | |
3515 | .globl __hypervisor_xcall_flush_tlb_page |
3516 | -__hypervisor_xcall_flush_tlb_page: /* 17 insns */ |
3517 | +__hypervisor_xcall_flush_tlb_page: /* 20 insns */ |
3518 | /* %g5=ctx, %g1=vaddr */ |
3519 | mov %o0, %g2 |
3520 | mov %o1, %g3 |
3521 | @@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */ |
3522 | sllx %o0, PAGE_SHIFT, %o0 |
3523 | ta HV_MMU_UNMAP_ADDR_TRAP |
3524 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
3525 | - brnz,a,pn %o0, __hypervisor_tlb_xcall_error |
3526 | + brnz,a,pn %o0, 1f |
3527 | mov %o0, %g5 |
3528 | mov %g2, %o0 |
3529 | mov %g3, %o1 |
3530 | mov %g4, %o2 |
3531 | membar #Sync |
3532 | retry |
3533 | +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
3534 | + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 |
3535 | + nop |
3536 | |
3537 | .globl __hypervisor_xcall_flush_tlb_kernel_range |
3538 | -__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ |
3539 | +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ |
3540 | /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ |
3541 | sethi %hi(PAGE_SIZE - 1), %g2 |
3542 | or %g2, %lo(PAGE_SIZE - 1), %g2 |
3543 | andn %g1, %g2, %g1 |
3544 | andn %g7, %g2, %g7 |
3545 | sub %g7, %g1, %g3 |
3546 | + srlx %g3, 18, %g7 |
3547 | add %g2, 1, %g2 |
3548 | sub %g3, %g2, %g3 |
3549 | mov %o0, %g2 |
3550 | mov %o1, %g4 |
3551 | - mov %o2, %g7 |
3552 | + brnz,pn %g7, 2f |
3553 | + mov %o2, %g7 |
3554 | 1: add %g1, %g3, %o0 /* ARG0: virtual address */ |
3555 | mov 0, %o1 /* ARG1: mmu context */ |
3556 | mov HV_MMU_ALL, %o2 /* ARG2: flags */ |
3557 | ta HV_MMU_UNMAP_ADDR_TRAP |
3558 | mov HV_MMU_UNMAP_ADDR_TRAP, %g6 |
3559 | - brnz,pn %o0, __hypervisor_tlb_xcall_error |
3560 | + brnz,pn %o0, 1f |
3561 | mov %o0, %g5 |
3562 | sethi %hi(PAGE_SIZE), %o2 |
3563 | brnz,pt %g3, 1b |
3564 | sub %g3, %o2, %g3 |
3565 | - mov %g2, %o0 |
3566 | +5: mov %g2, %o0 |
3567 | mov %g4, %o1 |
3568 | mov %g7, %o2 |
3569 | membar #Sync |
3570 | retry |
3571 | +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 |
3572 | + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 |
3573 | + nop |
3574 | +2: mov %o3, %g1 |
3575 | + mov %o5, %g3 |
3576 | + mov 0, %o0 /* ARG0: CPU lists unimplemented */ |
3577 | + mov 0, %o1 /* ARG1: CPU lists unimplemented */ |
3578 | + mov 0, %o2 /* ARG2: mmu context == nucleus */ |
3579 | + mov HV_MMU_ALL, %o3 /* ARG3: flags */ |
3580 | + mov HV_FAST_MMU_DEMAP_CTX, %o5 |
3581 | + ta HV_FAST_TRAP |
3582 | + mov %g1, %o3 |
3583 | + brz,pt %o0, 5b |
3584 | + mov %g3, %o5 |
3585 | + mov HV_FAST_MMU_DEMAP_CTX, %g6 |
3586 | + ba,pt %xcc, 1b |
3587 | + clr %g5 |
3588 | |
3589 | /* These just get rescheduled to PIL vectors. */ |
3590 | .globl xcall_call_function |
3591 | @@ -809,6 +985,58 @@ xcall_kgdb_capture: |
3592 | |
3593 | #endif /* CONFIG_SMP */ |
3594 | |
3595 | + .globl cheetah_patch_cachetlbops |
3596 | +cheetah_patch_cachetlbops: |
3597 | + save %sp, -128, %sp |
3598 | + |
3599 | + sethi %hi(__flush_tlb_mm), %o0 |
3600 | + or %o0, %lo(__flush_tlb_mm), %o0 |
3601 | + sethi %hi(__cheetah_flush_tlb_mm), %o1 |
3602 | + or %o1, %lo(__cheetah_flush_tlb_mm), %o1 |
3603 | + call tlb_patch_one |
3604 | + mov 19, %o2 |
3605 | + |
3606 | + sethi %hi(__flush_tlb_page), %o0 |
3607 | + or %o0, %lo(__flush_tlb_page), %o0 |
3608 | + sethi %hi(__cheetah_flush_tlb_page), %o1 |
3609 | + or %o1, %lo(__cheetah_flush_tlb_page), %o1 |
3610 | + call tlb_patch_one |
3611 | + mov 22, %o2 |
3612 | + |
3613 | + sethi %hi(__flush_tlb_pending), %o0 |
3614 | + or %o0, %lo(__flush_tlb_pending), %o0 |
3615 | + sethi %hi(__cheetah_flush_tlb_pending), %o1 |
3616 | + or %o1, %lo(__cheetah_flush_tlb_pending), %o1 |
3617 | + call tlb_patch_one |
3618 | + mov 27, %o2 |
3619 | + |
3620 | + sethi %hi(__flush_tlb_kernel_range), %o0 |
3621 | + or %o0, %lo(__flush_tlb_kernel_range), %o0 |
3622 | + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 |
3623 | + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 |
3624 | + call tlb_patch_one |
3625 | + mov 31, %o2 |
3626 | + |
3627 | +#ifdef DCACHE_ALIASING_POSSIBLE |
3628 | + sethi %hi(__flush_dcache_page), %o0 |
3629 | + or %o0, %lo(__flush_dcache_page), %o0 |
3630 | + sethi %hi(__cheetah_flush_dcache_page), %o1 |
3631 | + or %o1, %lo(__cheetah_flush_dcache_page), %o1 |
3632 | + call tlb_patch_one |
3633 | + mov 11, %o2 |
3634 | +#endif /* DCACHE_ALIASING_POSSIBLE */ |
3635 | + |
3636 | +#ifdef CONFIG_SMP |
3637 | + sethi %hi(xcall_flush_tlb_kernel_range), %o0 |
3638 | + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 |
3639 | + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 |
3640 | + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 |
3641 | + call tlb_patch_one |
3642 | + mov 44, %o2 |
3643 | +#endif /* CONFIG_SMP */ |
3644 | + |
3645 | + ret |
3646 | + restore |
3647 | |
3648 | .globl hypervisor_patch_cachetlbops |
3649 | hypervisor_patch_cachetlbops: |
3650 | @@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops: |
3651 | sethi %hi(__hypervisor_flush_tlb_mm), %o1 |
3652 | or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 |
3653 | call tlb_patch_one |
3654 | - mov 10, %o2 |
3655 | + mov 19, %o2 |
3656 | |
3657 | sethi %hi(__flush_tlb_page), %o0 |
3658 | or %o0, %lo(__flush_tlb_page), %o0 |
3659 | sethi %hi(__hypervisor_flush_tlb_page), %o1 |
3660 | or %o1, %lo(__hypervisor_flush_tlb_page), %o1 |
3661 | call tlb_patch_one |
3662 | - mov 11, %o2 |
3663 | + mov 22, %o2 |
3664 | |
3665 | sethi %hi(__flush_tlb_pending), %o0 |
3666 | or %o0, %lo(__flush_tlb_pending), %o0 |
3667 | sethi %hi(__hypervisor_flush_tlb_pending), %o1 |
3668 | or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 |
3669 | call tlb_patch_one |
3670 | - mov 16, %o2 |
3671 | + mov 27, %o2 |
3672 | |
3673 | sethi %hi(__flush_tlb_kernel_range), %o0 |
3674 | or %o0, %lo(__flush_tlb_kernel_range), %o0 |
3675 | sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 |
3676 | or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 |
3677 | call tlb_patch_one |
3678 | - mov 16, %o2 |
3679 | + mov 31, %o2 |
3680 | |
3681 | #ifdef DCACHE_ALIASING_POSSIBLE |
3682 | sethi %hi(__flush_dcache_page), %o0 |
3683 | @@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops: |
3684 | sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 |
3685 | or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 |
3686 | call tlb_patch_one |
3687 | - mov 21, %o2 |
3688 | + mov 24, %o2 |
3689 | |
3690 | sethi %hi(xcall_flush_tlb_page), %o0 |
3691 | or %o0, %lo(xcall_flush_tlb_page), %o0 |
3692 | sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 |
3693 | or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 |
3694 | call tlb_patch_one |
3695 | - mov 17, %o2 |
3696 | + mov 20, %o2 |
3697 | |
3698 | sethi %hi(xcall_flush_tlb_kernel_range), %o0 |
3699 | or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 |
3700 | sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 |
3701 | or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 |
3702 | call tlb_patch_one |
3703 | - mov 25, %o2 |
3704 | + mov 44, %o2 |
3705 | #endif /* CONFIG_SMP */ |
3706 | |
3707 | ret |
3708 | diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c |
3709 | index c4751ece76f6..45e87c9cc828 100644 |
3710 | --- a/drivers/net/ethernet/broadcom/bgmac.c |
3711 | +++ b/drivers/net/ethernet/broadcom/bgmac.c |
3712 | @@ -307,6 +307,10 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac, |
3713 | u32 ctl; |
3714 | |
3715 | ctl = bgmac_read(bgmac, ring->mmio_base + BGMAC_DMA_RX_CTL); |
3716 | + |
3717 | + /* preserve ONLY bits 16-17 from current hardware value */ |
3718 | + ctl &= BGMAC_DMA_RX_ADDREXT_MASK; |
3719 | + |
3720 | if (bgmac->feature_flags & BGMAC_FEAT_RX_MASK_SETUP) { |
3721 | ctl &= ~BGMAC_DMA_RX_BL_MASK; |
3722 | ctl |= BGMAC_DMA_RX_BL_128 << BGMAC_DMA_RX_BL_SHIFT; |
3723 | @@ -317,7 +321,6 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac, |
3724 | ctl &= ~BGMAC_DMA_RX_PT_MASK; |
3725 | ctl |= BGMAC_DMA_RX_PT_1 << BGMAC_DMA_RX_PT_SHIFT; |
3726 | } |
3727 | - ctl &= BGMAC_DMA_RX_ADDREXT_MASK; |
3728 | ctl |= BGMAC_DMA_RX_ENABLE; |
3729 | ctl |= BGMAC_DMA_RX_PARITY_DISABLE; |
3730 | ctl |= BGMAC_DMA_RX_OVERFLOW_CONT; |
3731 | diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c |
3732 | index 505ceaf451e2..2c850a92ab15 100644 |
3733 | --- a/drivers/net/ethernet/broadcom/bnx2.c |
3734 | +++ b/drivers/net/ethernet/broadcom/bnx2.c |
3735 | @@ -49,6 +49,7 @@ |
3736 | #include <linux/firmware.h> |
3737 | #include <linux/log2.h> |
3738 | #include <linux/aer.h> |
3739 | +#include <linux/crash_dump.h> |
3740 | |
3741 | #if defined(CONFIG_CNIC) || defined(CONFIG_CNIC_MODULE) |
3742 | #define BCM_CNIC 1 |
3743 | @@ -4759,15 +4760,16 @@ bnx2_setup_msix_tbl(struct bnx2 *bp) |
3744 | BNX2_WR(bp, BNX2_PCI_GRC_WINDOW3_ADDR, BNX2_MSIX_PBA_ADDR); |
3745 | } |
3746 | |
3747 | -static int |
3748 | -bnx2_reset_chip(struct bnx2 *bp, u32 reset_code) |
3749 | +static void |
3750 | +bnx2_wait_dma_complete(struct bnx2 *bp) |
3751 | { |
3752 | u32 val; |
3753 | - int i, rc = 0; |
3754 | - u8 old_port; |
3755 | + int i; |
3756 | |
3757 | - /* Wait for the current PCI transaction to complete before |
3758 | - * issuing a reset. */ |
3759 | + /* |
3760 | + * Wait for the current PCI transaction to complete before |
3761 | + * issuing a reset. |
3762 | + */ |
3763 | if ((BNX2_CHIP(bp) == BNX2_CHIP_5706) || |
3764 | (BNX2_CHIP(bp) == BNX2_CHIP_5708)) { |
3765 | BNX2_WR(bp, BNX2_MISC_ENABLE_CLR_BITS, |
3766 | @@ -4791,6 +4793,21 @@ bnx2_reset_chip(struct bnx2 *bp, u32 reset_code) |
3767 | } |
3768 | } |
3769 | |
3770 | + return; |
3771 | +} |
3772 | + |
3773 | + |
3774 | +static int |
3775 | +bnx2_reset_chip(struct bnx2 *bp, u32 reset_code) |
3776 | +{ |
3777 | + u32 val; |
3778 | + int i, rc = 0; |
3779 | + u8 old_port; |
3780 | + |
3781 | + /* Wait for the current PCI transaction to complete before |
3782 | + * issuing a reset. */ |
3783 | + bnx2_wait_dma_complete(bp); |
3784 | + |
3785 | /* Wait for the firmware to tell us it is ok to issue a reset. */ |
3786 | bnx2_fw_sync(bp, BNX2_DRV_MSG_DATA_WAIT0 | reset_code, 1, 1); |
3787 | |
3788 | @@ -6356,6 +6373,10 @@ bnx2_open(struct net_device *dev) |
3789 | struct bnx2 *bp = netdev_priv(dev); |
3790 | int rc; |
3791 | |
3792 | + rc = bnx2_request_firmware(bp); |
3793 | + if (rc < 0) |
3794 | + goto out; |
3795 | + |
3796 | netif_carrier_off(dev); |
3797 | |
3798 | bnx2_disable_int(bp); |
3799 | @@ -6424,6 +6445,7 @@ open_err: |
3800 | bnx2_free_irq(bp); |
3801 | bnx2_free_mem(bp); |
3802 | bnx2_del_napi(bp); |
3803 | + bnx2_release_firmware(bp); |
3804 | goto out; |
3805 | } |
3806 | |
3807 | @@ -8570,12 +8592,15 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) |
3808 | |
3809 | pci_set_drvdata(pdev, dev); |
3810 | |
3811 | - rc = bnx2_request_firmware(bp); |
3812 | - if (rc < 0) |
3813 | - goto error; |
3814 | - |
3815 | + /* |
3816 | + * In-flight DMA from 1st kernel could continue going in kdump kernel. |
3817 | + * New io-page table has been created before bnx2 does reset at open stage. |
3818 | + * We have to wait for the in-flight DMA to complete to avoid it look up |
3819 | + * into the newly created io-page table. |
3820 | + */ |
3821 | + if (is_kdump_kernel()) |
3822 | + bnx2_wait_dma_complete(bp); |
3823 | |
3824 | - bnx2_reset_chip(bp, BNX2_DRV_MSG_CODE_RESET); |
3825 | memcpy(dev->dev_addr, bp->mac_addr, ETH_ALEN); |
3826 | |
3827 | dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG | |
3828 | @@ -8608,7 +8633,6 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) |
3829 | return 0; |
3830 | |
3831 | error: |
3832 | - bnx2_release_firmware(bp); |
3833 | pci_iounmap(pdev, bp->regview); |
3834 | pci_release_regions(pdev); |
3835 | pci_disable_device(pdev); |
3836 | diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c |
3837 | index d48873bcbddf..5cdc96bdd444 100644 |
3838 | --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c |
3839 | +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c |
3840 | @@ -231,7 +231,7 @@ mlxsw_sp_span_entry_create(struct mlxsw_sp_port *port) |
3841 | |
3842 | span_entry->used = true; |
3843 | span_entry->id = index; |
3844 | - span_entry->ref_count = 0; |
3845 | + span_entry->ref_count = 1; |
3846 | span_entry->local_port = local_port; |
3847 | return span_entry; |
3848 | } |
3849 | @@ -268,6 +268,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port) |
3850 | |
3851 | span_entry = mlxsw_sp_span_entry_find(port); |
3852 | if (span_entry) { |
3853 | + /* Already exists, just take a reference */ |
3854 | span_entry->ref_count++; |
3855 | return span_entry; |
3856 | } |
3857 | @@ -278,6 +279,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port) |
3858 | static int mlxsw_sp_span_entry_put(struct mlxsw_sp *mlxsw_sp, |
3859 | struct mlxsw_sp_span_entry *span_entry) |
3860 | { |
3861 | + WARN_ON(!span_entry->ref_count); |
3862 | if (--span_entry->ref_count == 0) |
3863 | mlxsw_sp_span_entry_destroy(mlxsw_sp, span_entry); |
3864 | return 0; |
3865 | diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c |
3866 | index 3f5c51da6d3e..62514b9bf988 100644 |
3867 | --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c |
3868 | +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c |
3869 | @@ -777,6 +777,26 @@ static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, |
3870 | } |
3871 | } |
3872 | |
3873 | +static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) |
3874 | +{ |
3875 | + u8 num_rec, last_rec_index, num_entries; |
3876 | + |
3877 | + num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); |
3878 | + last_rec_index = num_rec - 1; |
3879 | + |
3880 | + if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) |
3881 | + return false; |
3882 | + if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == |
3883 | + MLXSW_REG_RAUHTD_TYPE_IPV6) |
3884 | + return true; |
3885 | + |
3886 | + num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, |
3887 | + last_rec_index); |
3888 | + if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) |
3889 | + return true; |
3890 | + return false; |
3891 | +} |
3892 | + |
3893 | static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) |
3894 | { |
3895 | char *rauhtd_pl; |
3896 | @@ -803,7 +823,7 @@ static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) |
3897 | for (i = 0; i < num_rec; i++) |
3898 | mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, |
3899 | i); |
3900 | - } while (num_rec); |
3901 | + } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); |
3902 | rtnl_unlock(); |
3903 | |
3904 | kfree(rauhtd_pl); |
3905 | diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c |
3906 | index 4c8c60af7985..fe9e7b1979b8 100644 |
3907 | --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c |
3908 | +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c |
3909 | @@ -871,6 +871,13 @@ static int stmmac_init_phy(struct net_device *dev) |
3910 | return -ENODEV; |
3911 | } |
3912 | |
3913 | + /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid |
3914 | + * subsequent PHY polling, make sure we force a link transition if |
3915 | + * we have a UP/DOWN/UP transition |
3916 | + */ |
3917 | + if (phydev->is_pseudo_fixed_link) |
3918 | + phydev->irq = PHY_POLL; |
3919 | + |
3920 | pr_debug("stmmac_init_phy: %s: attached to PHY (UID 0x%x)" |
3921 | " Link = %d\n", dev->name, phydev->phy_id, phydev->link); |
3922 | |
3923 | diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c |
3924 | index 5c8429f23a89..3a5530d0511b 100644 |
3925 | --- a/drivers/usb/gadget/function/f_fs.c |
3926 | +++ b/drivers/usb/gadget/function/f_fs.c |
3927 | @@ -133,8 +133,60 @@ struct ffs_epfile { |
3928 | /* |
3929 | * Buffer for holding data from partial reads which may happen since |
3930 | * we’re rounding user read requests to a multiple of a max packet size. |
3931 | + * |
3932 | + * The pointer is initialised with NULL value and may be set by |
3933 | + * __ffs_epfile_read_data function to point to a temporary buffer. |
3934 | + * |
3935 | + * In normal operation, calls to __ffs_epfile_read_buffered will consume |
3936 | + * data from said buffer and eventually free it. Importantly, while the |
3937 | + * function is using the buffer, it sets the pointer to NULL. This is |
3938 | + * all right since __ffs_epfile_read_data and __ffs_epfile_read_buffered |
3939 | + * can never run concurrently (they are synchronised by epfile->mutex) |
3940 | + * so the latter will not assign a new value to the pointer. |
3941 | + * |
3942 | + * Meanwhile ffs_func_eps_disable frees the buffer (if the pointer is |
3943 | + * valid) and sets the pointer to READ_BUFFER_DROP value. This special |
3944 | + * value is crux of the synchronisation between ffs_func_eps_disable and |
3945 | + * __ffs_epfile_read_data. |
3946 | + * |
3947 | + * Once __ffs_epfile_read_data is about to finish it will try to set the |
3948 | + * pointer back to its old value (as described above), but seeing as the |
3949 | + * pointer is not-NULL (namely READ_BUFFER_DROP) it will instead free |
3950 | + * the buffer. |
3951 | + * |
3952 | + * == State transitions == |
3953 | + * |
3954 | + * • ptr == NULL: (initial state) |
3955 | + * â—¦ __ffs_epfile_read_buffer_free: go to ptr == DROP |
3956 | + * â—¦ __ffs_epfile_read_buffered: nop |
3957 | + * â—¦ __ffs_epfile_read_data allocates temp buffer: go to ptr == buf |
3958 | + * ◦ reading finishes: n/a, not in ‘and reading’ state |
3959 | + * • ptr == DROP: |
3960 | + * â—¦ __ffs_epfile_read_buffer_free: nop |
3961 | + * â—¦ __ffs_epfile_read_buffered: go to ptr == NULL |
3962 | + * â—¦ __ffs_epfile_read_data allocates temp buffer: free buf, nop |
3963 | + * ◦ reading finishes: n/a, not in ‘and reading’ state |
3964 | + * • ptr == buf: |
3965 | + * â—¦ __ffs_epfile_read_buffer_free: free buf, go to ptr == DROP |
3966 | + * â—¦ __ffs_epfile_read_buffered: go to ptr == NULL and reading |
3967 | + * â—¦ __ffs_epfile_read_data: n/a, __ffs_epfile_read_buffered |
3968 | + * is always called first |
3969 | + * ◦ reading finishes: n/a, not in ‘and reading’ state |
3970 | + * • ptr == NULL and reading: |
3971 | + * â—¦ __ffs_epfile_read_buffer_free: go to ptr == DROP and reading |
3972 | + * â—¦ __ffs_epfile_read_buffered: n/a, mutex is held |
3973 | + * â—¦ __ffs_epfile_read_data: n/a, mutex is held |
3974 | + * ◦ reading finishes and … |
3975 | + * … all data read: free buf, go to ptr == NULL |
3976 | + * … otherwise: go to ptr == buf and reading |
3977 | + * • ptr == DROP and reading: |
3978 | + * â—¦ __ffs_epfile_read_buffer_free: nop |
3979 | + * â—¦ __ffs_epfile_read_buffered: n/a, mutex is held |
3980 | + * â—¦ __ffs_epfile_read_data: n/a, mutex is held |
3981 | + * â—¦ reading finishes: free buf, go to ptr == DROP |
3982 | */ |
3983 | - struct ffs_buffer *read_buffer; /* P: epfile->mutex */ |
3984 | + struct ffs_buffer *read_buffer; |
3985 | +#define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN)) |
3986 | |
3987 | char name[5]; |
3988 | |
3989 | @@ -733,25 +785,47 @@ static void ffs_epfile_async_io_complete(struct usb_ep *_ep, |
3990 | schedule_work(&io_data->work); |
3991 | } |
3992 | |
3993 | +static void __ffs_epfile_read_buffer_free(struct ffs_epfile *epfile) |
3994 | +{ |
3995 | + /* |
3996 | + * See comment in struct ffs_epfile for full read_buffer pointer |
3997 | + * synchronisation story. |
3998 | + */ |
3999 | + struct ffs_buffer *buf = xchg(&epfile->read_buffer, READ_BUFFER_DROP); |
4000 | + if (buf && buf != READ_BUFFER_DROP) |
4001 | + kfree(buf); |
4002 | +} |
4003 | + |
4004 | /* Assumes epfile->mutex is held. */ |
4005 | static ssize_t __ffs_epfile_read_buffered(struct ffs_epfile *epfile, |
4006 | struct iov_iter *iter) |
4007 | { |
4008 | - struct ffs_buffer *buf = epfile->read_buffer; |
4009 | + /* |
4010 | + * Null out epfile->read_buffer so ffs_func_eps_disable does not free |
4011 | + * the buffer while we are using it. See comment in struct ffs_epfile |
4012 | + * for full read_buffer pointer synchronisation story. |
4013 | + */ |
4014 | + struct ffs_buffer *buf = xchg(&epfile->read_buffer, NULL); |
4015 | ssize_t ret; |
4016 | - if (!buf) |
4017 | + if (!buf || buf == READ_BUFFER_DROP) |
4018 | return 0; |
4019 | |
4020 | ret = copy_to_iter(buf->data, buf->length, iter); |
4021 | if (buf->length == ret) { |
4022 | kfree(buf); |
4023 | - epfile->read_buffer = NULL; |
4024 | - } else if (unlikely(iov_iter_count(iter))) { |
4025 | + return ret; |
4026 | + } |
4027 | + |
4028 | + if (unlikely(iov_iter_count(iter))) { |
4029 | ret = -EFAULT; |
4030 | } else { |
4031 | buf->length -= ret; |
4032 | buf->data += ret; |
4033 | } |
4034 | + |
4035 | + if (cmpxchg(&epfile->read_buffer, NULL, buf)) |
4036 | + kfree(buf); |
4037 | + |
4038 | return ret; |
4039 | } |
4040 | |
4041 | @@ -780,7 +854,15 @@ static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile, |
4042 | buf->length = data_len; |
4043 | buf->data = buf->storage; |
4044 | memcpy(buf->storage, data + ret, data_len); |
4045 | - epfile->read_buffer = buf; |
4046 | + |
4047 | + /* |
4048 | + * At this point read_buffer is NULL or READ_BUFFER_DROP (if |
4049 | + * ffs_func_eps_disable has been called in the meanwhile). See comment |
4050 | + * in struct ffs_epfile for full read_buffer pointer synchronisation |
4051 | + * story. |
4052 | + */ |
4053 | + if (unlikely(cmpxchg(&epfile->read_buffer, NULL, buf))) |
4054 | + kfree(buf); |
4055 | |
4056 | return ret; |
4057 | } |
4058 | @@ -1094,8 +1176,7 @@ ffs_epfile_release(struct inode *inode, struct file *file) |
4059 | |
4060 | ENTER(); |
4061 | |
4062 | - kfree(epfile->read_buffer); |
4063 | - epfile->read_buffer = NULL; |
4064 | + __ffs_epfile_read_buffer_free(epfile); |
4065 | ffs_data_closed(epfile->ffs); |
4066 | |
4067 | return 0; |
4068 | @@ -1721,24 +1802,20 @@ static void ffs_func_eps_disable(struct ffs_function *func) |
4069 | unsigned count = func->ffs->eps_count; |
4070 | unsigned long flags; |
4071 | |
4072 | + spin_lock_irqsave(&func->ffs->eps_lock, flags); |
4073 | do { |
4074 | - if (epfile) |
4075 | - mutex_lock(&epfile->mutex); |
4076 | - spin_lock_irqsave(&func->ffs->eps_lock, flags); |
4077 | /* pending requests get nuked */ |
4078 | if (likely(ep->ep)) |
4079 | usb_ep_disable(ep->ep); |
4080 | ++ep; |
4081 | - spin_unlock_irqrestore(&func->ffs->eps_lock, flags); |
4082 | |
4083 | if (epfile) { |
4084 | epfile->ep = NULL; |
4085 | - kfree(epfile->read_buffer); |
4086 | - epfile->read_buffer = NULL; |
4087 | - mutex_unlock(&epfile->mutex); |
4088 | + __ffs_epfile_read_buffer_free(epfile); |
4089 | ++epfile; |
4090 | } |
4091 | } while (--count); |
4092 | + spin_unlock_irqrestore(&func->ffs->eps_lock, flags); |
4093 | } |
4094 | |
4095 | static int ffs_func_eps_enable(struct ffs_function *func) |
4096 | diff --git a/include/net/ip.h b/include/net/ip.h |
4097 | index 156b0c11b524..0ccf6daf6f56 100644 |
4098 | --- a/include/net/ip.h |
4099 | +++ b/include/net/ip.h |
4100 | @@ -47,7 +47,6 @@ struct inet_skb_parm { |
4101 | #define IPSKB_REROUTED BIT(4) |
4102 | #define IPSKB_DOREDIRECT BIT(5) |
4103 | #define IPSKB_FRAG_PMTU BIT(6) |
4104 | -#define IPSKB_FRAG_SEGS BIT(7) |
4105 | |
4106 | u16 frag_max_size; |
4107 | }; |
4108 | diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h |
4109 | index 43a5a0e4524c..b01d5d1d7439 100644 |
4110 | --- a/include/net/ip6_tunnel.h |
4111 | +++ b/include/net/ip6_tunnel.h |
4112 | @@ -145,6 +145,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, |
4113 | { |
4114 | int pkt_len, err; |
4115 | |
4116 | + memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); |
4117 | pkt_len = skb->len - skb_inner_network_offset(skb); |
4118 | err = ip6_local_out(dev_net(skb_dst(skb)->dev), sk, skb); |
4119 | if (unlikely(net_xmit_eval(err))) |
4120 | diff --git a/include/net/sock.h b/include/net/sock.h |
4121 | index 8741988e6880..c26eab962ec7 100644 |
4122 | --- a/include/net/sock.h |
4123 | +++ b/include/net/sock.h |
4124 | @@ -1587,11 +1587,11 @@ static inline void sock_put(struct sock *sk) |
4125 | void sock_gen_put(struct sock *sk); |
4126 | |
4127 | int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested, |
4128 | - unsigned int trim_cap); |
4129 | + unsigned int trim_cap, bool refcounted); |
4130 | static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb, |
4131 | const int nested) |
4132 | { |
4133 | - return __sk_receive_skb(sk, skb, nested, 1); |
4134 | + return __sk_receive_skb(sk, skb, nested, 1, true); |
4135 | } |
4136 | |
4137 | static inline void sk_tx_queue_set(struct sock *sk, int tx_queue) |
4138 | diff --git a/include/net/tcp.h b/include/net/tcp.h |
4139 | index 7717302cab91..0de698940793 100644 |
4140 | --- a/include/net/tcp.h |
4141 | +++ b/include/net/tcp.h |
4142 | @@ -1164,6 +1164,7 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp) |
4143 | } |
4144 | |
4145 | bool tcp_prequeue(struct sock *sk, struct sk_buff *skb); |
4146 | +int tcp_filter(struct sock *sk, struct sk_buff *skb); |
4147 | |
4148 | #undef STATE_TRACE |
4149 | |
4150 | diff --git a/include/uapi/linux/atm_zatm.h b/include/uapi/linux/atm_zatm.h |
4151 | index 5cd4d4d2dd1d..9c9c6ad55f14 100644 |
4152 | --- a/include/uapi/linux/atm_zatm.h |
4153 | +++ b/include/uapi/linux/atm_zatm.h |
4154 | @@ -14,7 +14,6 @@ |
4155 | |
4156 | #include <linux/atmapi.h> |
4157 | #include <linux/atmioc.h> |
4158 | -#include <linux/time.h> |
4159 | |
4160 | #define ZATM_GETPOOL _IOW('a',ATMIOC_SARPRV+1,struct atmif_sioc) |
4161 | /* get pool statistics */ |
4162 | diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c |
4163 | index 570eeca7bdfa..ad1bc67aff1b 100644 |
4164 | --- a/kernel/bpf/hashtab.c |
4165 | +++ b/kernel/bpf/hashtab.c |
4166 | @@ -687,7 +687,8 @@ static void delete_all_elements(struct bpf_htab *htab) |
4167 | |
4168 | hlist_for_each_entry_safe(l, n, head, hash_node) { |
4169 | hlist_del_rcu(&l->hash_node); |
4170 | - htab_elem_free(htab, l); |
4171 | + if (l->state != HTAB_EXTRA_ELEM_USED) |
4172 | + htab_elem_free(htab, l); |
4173 | } |
4174 | } |
4175 | } |
4176 | diff --git a/net/core/dev.c b/net/core/dev.c |
4177 | index 44b3ba462ba1..9ce9d7284ea7 100644 |
4178 | --- a/net/core/dev.c |
4179 | +++ b/net/core/dev.c |
4180 | @@ -2484,7 +2484,7 @@ int skb_checksum_help(struct sk_buff *skb) |
4181 | goto out; |
4182 | } |
4183 | |
4184 | - *(__sum16 *)(skb->data + offset) = csum_fold(csum); |
4185 | + *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; |
4186 | out_set_summed: |
4187 | skb->ip_summed = CHECKSUM_NONE; |
4188 | out: |
4189 | diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c |
4190 | index 52742a02814f..5550a86f7264 100644 |
4191 | --- a/net/core/flow_dissector.c |
4192 | +++ b/net/core/flow_dissector.c |
4193 | @@ -118,7 +118,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb, |
4194 | struct flow_dissector_key_tags *key_tags; |
4195 | struct flow_dissector_key_keyid *key_keyid; |
4196 | u8 ip_proto = 0; |
4197 | - bool ret = false; |
4198 | + bool ret; |
4199 | |
4200 | if (!data) { |
4201 | data = skb->data; |
4202 | @@ -481,12 +481,17 @@ ip_proto_again: |
4203 | out_good: |
4204 | ret = true; |
4205 | |
4206 | -out_bad: |
4207 | + key_control->thoff = (u16)nhoff; |
4208 | +out: |
4209 | key_basic->n_proto = proto; |
4210 | key_basic->ip_proto = ip_proto; |
4211 | - key_control->thoff = (u16)nhoff; |
4212 | |
4213 | return ret; |
4214 | + |
4215 | +out_bad: |
4216 | + ret = false; |
4217 | + key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen); |
4218 | + goto out; |
4219 | } |
4220 | EXPORT_SYMBOL(__skb_flow_dissect); |
4221 | |
4222 | diff --git a/net/core/sock.c b/net/core/sock.c |
4223 | index fd7b41edf1ce..10acaccca5c8 100644 |
4224 | --- a/net/core/sock.c |
4225 | +++ b/net/core/sock.c |
4226 | @@ -453,7 +453,7 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) |
4227 | EXPORT_SYMBOL(sock_queue_rcv_skb); |
4228 | |
4229 | int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, |
4230 | - const int nested, unsigned int trim_cap) |
4231 | + const int nested, unsigned int trim_cap, bool refcounted) |
4232 | { |
4233 | int rc = NET_RX_SUCCESS; |
4234 | |
4235 | @@ -487,7 +487,8 @@ int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, |
4236 | |
4237 | bh_unlock_sock(sk); |
4238 | out: |
4239 | - sock_put(sk); |
4240 | + if (refcounted) |
4241 | + sock_put(sk); |
4242 | return rc; |
4243 | discard_and_relse: |
4244 | kfree_skb(skb); |
4245 | @@ -1563,6 +1564,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) |
4246 | RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); |
4247 | |
4248 | newsk->sk_err = 0; |
4249 | + newsk->sk_err_soft = 0; |
4250 | newsk->sk_priority = 0; |
4251 | newsk->sk_incoming_cpu = raw_smp_processor_id(); |
4252 | atomic64_set(&newsk->sk_cookie, 0); |
4253 | diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c |
4254 | index 345a3aeb8c7e..b567c8725aea 100644 |
4255 | --- a/net/dccp/ipv4.c |
4256 | +++ b/net/dccp/ipv4.c |
4257 | @@ -235,7 +235,7 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info) |
4258 | { |
4259 | const struct iphdr *iph = (struct iphdr *)skb->data; |
4260 | const u8 offset = iph->ihl << 2; |
4261 | - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset); |
4262 | + const struct dccp_hdr *dh; |
4263 | struct dccp_sock *dp; |
4264 | struct inet_sock *inet; |
4265 | const int type = icmp_hdr(skb)->type; |
4266 | @@ -245,11 +245,13 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info) |
4267 | int err; |
4268 | struct net *net = dev_net(skb->dev); |
4269 | |
4270 | - if (skb->len < offset + sizeof(*dh) || |
4271 | - skb->len < offset + __dccp_basic_hdr_len(dh)) { |
4272 | - __ICMP_INC_STATS(net, ICMP_MIB_INERRORS); |
4273 | - return; |
4274 | - } |
4275 | + /* Only need dccph_dport & dccph_sport which are the first |
4276 | + * 4 bytes in dccp header. |
4277 | + * Our caller (icmp_socket_deliver()) already pulled 8 bytes for us. |
4278 | + */ |
4279 | + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8); |
4280 | + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8); |
4281 | + dh = (struct dccp_hdr *)(skb->data + offset); |
4282 | |
4283 | sk = __inet_lookup_established(net, &dccp_hashinfo, |
4284 | iph->daddr, dh->dccph_dport, |
4285 | @@ -868,7 +870,7 @@ lookup: |
4286 | goto discard_and_relse; |
4287 | nf_reset(skb); |
4288 | |
4289 | - return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4); |
4290 | + return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4, refcounted); |
4291 | |
4292 | no_dccp_socket: |
4293 | if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) |
4294 | diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c |
4295 | index 3828f94b234c..715e5d1dc107 100644 |
4296 | --- a/net/dccp/ipv6.c |
4297 | +++ b/net/dccp/ipv6.c |
4298 | @@ -70,7 +70,7 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, |
4299 | u8 type, u8 code, int offset, __be32 info) |
4300 | { |
4301 | const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data; |
4302 | - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset); |
4303 | + const struct dccp_hdr *dh; |
4304 | struct dccp_sock *dp; |
4305 | struct ipv6_pinfo *np; |
4306 | struct sock *sk; |
4307 | @@ -78,12 +78,13 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, |
4308 | __u64 seq; |
4309 | struct net *net = dev_net(skb->dev); |
4310 | |
4311 | - if (skb->len < offset + sizeof(*dh) || |
4312 | - skb->len < offset + __dccp_basic_hdr_len(dh)) { |
4313 | - __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), |
4314 | - ICMP6_MIB_INERRORS); |
4315 | - return; |
4316 | - } |
4317 | + /* Only need dccph_dport & dccph_sport which are the first |
4318 | + * 4 bytes in dccp header. |
4319 | + * Our caller (icmpv6_notify()) already pulled 8 bytes for us. |
4320 | + */ |
4321 | + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8); |
4322 | + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8); |
4323 | + dh = (struct dccp_hdr *)(skb->data + offset); |
4324 | |
4325 | sk = __inet6_lookup_established(net, &dccp_hashinfo, |
4326 | &hdr->daddr, dh->dccph_dport, |
4327 | @@ -738,7 +739,8 @@ lookup: |
4328 | if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) |
4329 | goto discard_and_relse; |
4330 | |
4331 | - return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4) ? -1 : 0; |
4332 | + return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4, |
4333 | + refcounted) ? -1 : 0; |
4334 | |
4335 | no_dccp_socket: |
4336 | if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) |
4337 | @@ -956,6 +958,7 @@ static const struct inet_connection_sock_af_ops dccp_ipv6_mapped = { |
4338 | .getsockopt = ipv6_getsockopt, |
4339 | .addr2sockaddr = inet6_csk_addr2sockaddr, |
4340 | .sockaddr_len = sizeof(struct sockaddr_in6), |
4341 | + .bind_conflict = inet6_csk_bind_conflict, |
4342 | #ifdef CONFIG_COMPAT |
4343 | .compat_setsockopt = compat_ipv6_setsockopt, |
4344 | .compat_getsockopt = compat_ipv6_getsockopt, |
4345 | diff --git a/net/dccp/proto.c b/net/dccp/proto.c |
4346 | index 41e65804ddf5..9fe25bf63296 100644 |
4347 | --- a/net/dccp/proto.c |
4348 | +++ b/net/dccp/proto.c |
4349 | @@ -1009,6 +1009,10 @@ void dccp_close(struct sock *sk, long timeout) |
4350 | __kfree_skb(skb); |
4351 | } |
4352 | |
4353 | + /* If socket has been already reset kill it. */ |
4354 | + if (sk->sk_state == DCCP_CLOSED) |
4355 | + goto adjudge_to_death; |
4356 | + |
4357 | if (data_was_unread) { |
4358 | /* Unread data was tossed, send an appropriate Reset Code */ |
4359 | DCCP_WARN("ABORT with %u bytes unread\n", data_was_unread); |
4360 | diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c |
4361 | index e2ffc2a5c7db..7ef703102dca 100644 |
4362 | --- a/net/ipv4/fib_trie.c |
4363 | +++ b/net/ipv4/fib_trie.c |
4364 | @@ -2455,22 +2455,19 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter, |
4365 | struct key_vector *l, **tp = &iter->tnode; |
4366 | t_key key; |
4367 | |
4368 | - /* use cache location of next-to-find key */ |
4369 | + /* use cached location of previously found key */ |
4370 | if (iter->pos > 0 && pos >= iter->pos) { |
4371 | - pos -= iter->pos; |
4372 | key = iter->key; |
4373 | } else { |
4374 | - iter->pos = 0; |
4375 | + iter->pos = 1; |
4376 | key = 0; |
4377 | } |
4378 | |
4379 | - while ((l = leaf_walk_rcu(tp, key)) != NULL) { |
4380 | + pos -= iter->pos; |
4381 | + |
4382 | + while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) { |
4383 | key = l->key + 1; |
4384 | iter->pos++; |
4385 | - |
4386 | - if (--pos <= 0) |
4387 | - break; |
4388 | - |
4389 | l = NULL; |
4390 | |
4391 | /* handle unlikely case of a key wrap */ |
4392 | @@ -2479,7 +2476,7 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter, |
4393 | } |
4394 | |
4395 | if (l) |
4396 | - iter->key = key; /* remember it */ |
4397 | + iter->key = l->key; /* remember it */ |
4398 | else |
4399 | iter->pos = 0; /* forget it */ |
4400 | |
4401 | @@ -2507,7 +2504,7 @@ static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos) |
4402 | return fib_route_get_idx(iter, *pos); |
4403 | |
4404 | iter->pos = 0; |
4405 | - iter->key = 0; |
4406 | + iter->key = KEY_MAX; |
4407 | |
4408 | return SEQ_START_TOKEN; |
4409 | } |
4410 | @@ -2516,7 +2513,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos) |
4411 | { |
4412 | struct fib_route_iter *iter = seq->private; |
4413 | struct key_vector *l = NULL; |
4414 | - t_key key = iter->key; |
4415 | + t_key key = iter->key + 1; |
4416 | |
4417 | ++*pos; |
4418 | |
4419 | @@ -2525,7 +2522,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos) |
4420 | l = leaf_walk_rcu(&iter->tnode, key); |
4421 | |
4422 | if (l) { |
4423 | - iter->key = l->key + 1; |
4424 | + iter->key = l->key; |
4425 | iter->pos++; |
4426 | } else { |
4427 | iter->pos = 0; |
4428 | diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c |
4429 | index 38abe70e595f..48734ee6293f 100644 |
4430 | --- a/net/ipv4/icmp.c |
4431 | +++ b/net/ipv4/icmp.c |
4432 | @@ -477,7 +477,7 @@ static struct rtable *icmp_route_lookup(struct net *net, |
4433 | fl4->flowi4_proto = IPPROTO_ICMP; |
4434 | fl4->fl4_icmp_type = type; |
4435 | fl4->fl4_icmp_code = code; |
4436 | - fl4->flowi4_oif = l3mdev_master_ifindex(skb_in->dev); |
4437 | + fl4->flowi4_oif = l3mdev_master_ifindex(skb_dst(skb_in)->dev); |
4438 | |
4439 | security_skb_classify_flow(skb_in, flowi4_to_flowi(fl4)); |
4440 | rt = __ip_route_output_key_hash(net, fl4, |
4441 | @@ -502,7 +502,7 @@ static struct rtable *icmp_route_lookup(struct net *net, |
4442 | if (err) |
4443 | goto relookup_failed; |
4444 | |
4445 | - if (inet_addr_type_dev_table(net, skb_in->dev, |
4446 | + if (inet_addr_type_dev_table(net, skb_dst(skb_in)->dev, |
4447 | fl4_dec.saddr) == RTN_LOCAL) { |
4448 | rt2 = __ip_route_output_key(net, &fl4_dec); |
4449 | if (IS_ERR(rt2)) |
4450 | diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c |
4451 | index 8b4ffd216839..9f0a7b96646f 100644 |
4452 | --- a/net/ipv4/ip_forward.c |
4453 | +++ b/net/ipv4/ip_forward.c |
4454 | @@ -117,7 +117,7 @@ int ip_forward(struct sk_buff *skb) |
4455 | if (opt->is_strictroute && rt->rt_uses_gateway) |
4456 | goto sr_failed; |
4457 | |
4458 | - IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS; |
4459 | + IPCB(skb)->flags |= IPSKB_FORWARDED; |
4460 | mtu = ip_dst_mtu_maybe_forward(&rt->dst, true); |
4461 | if (ip_exceeds_mtu(skb, mtu)) { |
4462 | IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS); |
4463 | diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c |
4464 | index dde37fb340bf..307daed9a4b9 100644 |
4465 | --- a/net/ipv4/ip_output.c |
4466 | +++ b/net/ipv4/ip_output.c |
4467 | @@ -223,11 +223,9 @@ static int ip_finish_output_gso(struct net *net, struct sock *sk, |
4468 | struct sk_buff *segs; |
4469 | int ret = 0; |
4470 | |
4471 | - /* common case: fragmentation of segments is not allowed, |
4472 | - * or seglen is <= mtu |
4473 | + /* common case: seglen is <= mtu |
4474 | */ |
4475 | - if (((IPCB(skb)->flags & IPSKB_FRAG_SEGS) == 0) || |
4476 | - skb_gso_validate_mtu(skb, mtu)) |
4477 | + if (skb_gso_validate_mtu(skb, mtu)) |
4478 | return ip_finish_output2(net, sk, skb); |
4479 | |
4480 | /* Slowpath - GSO segment length is exceeding the dst MTU. |
4481 | diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c |
4482 | index 0f227db0e9ac..afd6b5968caf 100644 |
4483 | --- a/net/ipv4/ip_tunnel_core.c |
4484 | +++ b/net/ipv4/ip_tunnel_core.c |
4485 | @@ -63,7 +63,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, |
4486 | int pkt_len = skb->len - skb_inner_network_offset(skb); |
4487 | struct net *net = dev_net(rt->dst.dev); |
4488 | struct net_device *dev = skb->dev; |
4489 | - int skb_iif = skb->skb_iif; |
4490 | struct iphdr *iph; |
4491 | int err; |
4492 | |
4493 | @@ -73,16 +72,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, |
4494 | skb_dst_set(skb, &rt->dst); |
4495 | memset(IPCB(skb), 0, sizeof(*IPCB(skb))); |
4496 | |
4497 | - if (skb_iif && !(df & htons(IP_DF))) { |
4498 | - /* Arrived from an ingress interface, got encapsulated, with |
4499 | - * fragmentation of encapulating frames allowed. |
4500 | - * If skb is gso, the resulting encapsulated network segments |
4501 | - * may exceed dst mtu. |
4502 | - * Allow IP Fragmentation of segments. |
4503 | - */ |
4504 | - IPCB(skb)->flags |= IPSKB_FRAG_SEGS; |
4505 | - } |
4506 | - |
4507 | /* Push down and install the IP header. */ |
4508 | skb_push(skb, sizeof(struct iphdr)); |
4509 | skb_reset_network_header(skb); |
4510 | diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c |
4511 | index 5f006e13de56..27089f5ebbb1 100644 |
4512 | --- a/net/ipv4/ipmr.c |
4513 | +++ b/net/ipv4/ipmr.c |
4514 | @@ -1749,7 +1749,7 @@ static void ipmr_queue_xmit(struct net *net, struct mr_table *mrt, |
4515 | vif->dev->stats.tx_bytes += skb->len; |
4516 | } |
4517 | |
4518 | - IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS; |
4519 | + IPCB(skb)->flags |= IPSKB_FORWARDED; |
4520 | |
4521 | /* RFC1584 teaches, that DVMRP/PIM router must deliver packets locally |
4522 | * not only before forwarding, but after forwarding on all output |
4523 | diff --git a/net/ipv4/route.c b/net/ipv4/route.c |
4524 | index 62c3ed0b7556..2f23ef1a8486 100644 |
4525 | --- a/net/ipv4/route.c |
4526 | +++ b/net/ipv4/route.c |
4527 | @@ -753,7 +753,9 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow |
4528 | goto reject_redirect; |
4529 | } |
4530 | |
4531 | - n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw); |
4532 | + n = __ipv4_neigh_lookup(rt->dst.dev, new_gw); |
4533 | + if (!n) |
4534 | + n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev); |
4535 | if (!IS_ERR(n)) { |
4536 | if (!(n->nud_state & NUD_VALID)) { |
4537 | neigh_event_send(n, NULL); |
4538 | diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c |
4539 | index ffbb218de520..c876f5ddc86c 100644 |
4540 | --- a/net/ipv4/tcp.c |
4541 | +++ b/net/ipv4/tcp.c |
4542 | @@ -1145,7 +1145,7 @@ restart: |
4543 | |
4544 | err = -EPIPE; |
4545 | if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) |
4546 | - goto out_err; |
4547 | + goto do_error; |
4548 | |
4549 | sg = !!(sk->sk_route_caps & NETIF_F_SG); |
4550 | |
4551 | @@ -1219,7 +1219,7 @@ new_segment: |
4552 | |
4553 | if (!skb_can_coalesce(skb, i, pfrag->page, |
4554 | pfrag->offset)) { |
4555 | - if (i == sysctl_max_skb_frags || !sg) { |
4556 | + if (i >= sysctl_max_skb_frags || !sg) { |
4557 | tcp_mark_push(tp, skb); |
4558 | goto new_segment; |
4559 | } |
4560 | diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c |
4561 | index 10d728b6804c..ab37c6775630 100644 |
4562 | --- a/net/ipv4/tcp_dctcp.c |
4563 | +++ b/net/ipv4/tcp_dctcp.c |
4564 | @@ -56,6 +56,7 @@ struct dctcp { |
4565 | u32 next_seq; |
4566 | u32 ce_state; |
4567 | u32 delayed_ack_reserved; |
4568 | + u32 loss_cwnd; |
4569 | }; |
4570 | |
4571 | static unsigned int dctcp_shift_g __read_mostly = 4; /* g = 1/2^4 */ |
4572 | @@ -96,6 +97,7 @@ static void dctcp_init(struct sock *sk) |
4573 | ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA); |
4574 | |
4575 | ca->delayed_ack_reserved = 0; |
4576 | + ca->loss_cwnd = 0; |
4577 | ca->ce_state = 0; |
4578 | |
4579 | dctcp_reset(tp, ca); |
4580 | @@ -111,9 +113,10 @@ static void dctcp_init(struct sock *sk) |
4581 | |
4582 | static u32 dctcp_ssthresh(struct sock *sk) |
4583 | { |
4584 | - const struct dctcp *ca = inet_csk_ca(sk); |
4585 | + struct dctcp *ca = inet_csk_ca(sk); |
4586 | struct tcp_sock *tp = tcp_sk(sk); |
4587 | |
4588 | + ca->loss_cwnd = tp->snd_cwnd; |
4589 | return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U); |
4590 | } |
4591 | |
4592 | @@ -308,12 +311,20 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr, |
4593 | return 0; |
4594 | } |
4595 | |
4596 | +static u32 dctcp_cwnd_undo(struct sock *sk) |
4597 | +{ |
4598 | + const struct dctcp *ca = inet_csk_ca(sk); |
4599 | + |
4600 | + return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd); |
4601 | +} |
4602 | + |
4603 | static struct tcp_congestion_ops dctcp __read_mostly = { |
4604 | .init = dctcp_init, |
4605 | .in_ack_event = dctcp_update_alpha, |
4606 | .cwnd_event = dctcp_cwnd_event, |
4607 | .ssthresh = dctcp_ssthresh, |
4608 | .cong_avoid = tcp_reno_cong_avoid, |
4609 | + .undo_cwnd = dctcp_cwnd_undo, |
4610 | .set_state = dctcp_state, |
4611 | .get_info = dctcp_get_info, |
4612 | .flags = TCP_CONG_NEEDS_ECN, |
4613 | diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c |
4614 | index 7158d4f8dae4..7b235fa12903 100644 |
4615 | --- a/net/ipv4/tcp_ipv4.c |
4616 | +++ b/net/ipv4/tcp_ipv4.c |
4617 | @@ -1537,6 +1537,21 @@ bool tcp_prequeue(struct sock *sk, struct sk_buff *skb) |
4618 | } |
4619 | EXPORT_SYMBOL(tcp_prequeue); |
4620 | |
4621 | +int tcp_filter(struct sock *sk, struct sk_buff *skb) |
4622 | +{ |
4623 | + struct tcphdr *th = (struct tcphdr *)skb->data; |
4624 | + unsigned int eaten = skb->len; |
4625 | + int err; |
4626 | + |
4627 | + err = sk_filter_trim_cap(sk, skb, th->doff * 4); |
4628 | + if (!err) { |
4629 | + eaten -= skb->len; |
4630 | + TCP_SKB_CB(skb)->end_seq -= eaten; |
4631 | + } |
4632 | + return err; |
4633 | +} |
4634 | +EXPORT_SYMBOL(tcp_filter); |
4635 | + |
4636 | /* |
4637 | * From tcp_input.c |
4638 | */ |
4639 | @@ -1648,8 +1663,10 @@ process: |
4640 | |
4641 | nf_reset(skb); |
4642 | |
4643 | - if (sk_filter(sk, skb)) |
4644 | + if (tcp_filter(sk, skb)) |
4645 | goto discard_and_relse; |
4646 | + th = (const struct tcphdr *)skb->data; |
4647 | + iph = ip_hdr(skb); |
4648 | |
4649 | skb->dev = NULL; |
4650 | |
4651 | diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c |
4652 | index bd59c343d35f..7370ad2e693a 100644 |
4653 | --- a/net/ipv6/icmp.c |
4654 | +++ b/net/ipv6/icmp.c |
4655 | @@ -448,7 +448,7 @@ static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info, |
4656 | if (__ipv6_addr_needs_scope_id(addr_type)) |
4657 | iif = skb->dev->ifindex; |
4658 | else |
4659 | - iif = l3mdev_master_ifindex(skb->dev); |
4660 | + iif = l3mdev_master_ifindex(skb_dst(skb)->dev); |
4661 | |
4662 | /* |
4663 | * Must not send error if the source does not uniquely |
4664 | diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c |
4665 | index fc67822c42e0..af6a09efad5b 100644 |
4666 | --- a/net/ipv6/tcp_ipv6.c |
4667 | +++ b/net/ipv6/tcp_ipv6.c |
4668 | @@ -1228,7 +1228,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) |
4669 | if (skb->protocol == htons(ETH_P_IP)) |
4670 | return tcp_v4_do_rcv(sk, skb); |
4671 | |
4672 | - if (sk_filter(sk, skb)) |
4673 | + if (tcp_filter(sk, skb)) |
4674 | goto discard; |
4675 | |
4676 | /* |
4677 | @@ -1455,8 +1455,10 @@ process: |
4678 | if (tcp_v6_inbound_md5_hash(sk, skb)) |
4679 | goto discard_and_relse; |
4680 | |
4681 | - if (sk_filter(sk, skb)) |
4682 | + if (tcp_filter(sk, skb)) |
4683 | goto discard_and_relse; |
4684 | + th = (const struct tcphdr *)skb->data; |
4685 | + hdr = ipv6_hdr(skb); |
4686 | |
4687 | skb->dev = NULL; |
4688 | |
4689 | diff --git a/net/sctp/socket.c b/net/sctp/socket.c |
4690 | index baccbf3c1c60..7b0e059bf13b 100644 |
4691 | --- a/net/sctp/socket.c |
4692 | +++ b/net/sctp/socket.c |
4693 | @@ -1214,9 +1214,12 @@ static int __sctp_connect(struct sock *sk, |
4694 | |
4695 | timeo = sock_sndtimeo(sk, f_flags & O_NONBLOCK); |
4696 | |
4697 | - err = sctp_wait_for_connect(asoc, &timeo); |
4698 | - if ((err == 0 || err == -EINPROGRESS) && assoc_id) |
4699 | + if (assoc_id) |
4700 | *assoc_id = asoc->assoc_id; |
4701 | + err = sctp_wait_for_connect(asoc, &timeo); |
4702 | + /* Note: the asoc may be freed after the return of |
4703 | + * sctp_wait_for_connect. |
4704 | + */ |
4705 | |
4706 | /* Don't free association on exit. */ |
4707 | asoc = NULL; |
4708 | @@ -4278,19 +4281,18 @@ static void sctp_shutdown(struct sock *sk, int how) |
4709 | { |
4710 | struct net *net = sock_net(sk); |
4711 | struct sctp_endpoint *ep; |
4712 | - struct sctp_association *asoc; |
4713 | |
4714 | if (!sctp_style(sk, TCP)) |
4715 | return; |
4716 | |
4717 | - if (how & SEND_SHUTDOWN) { |
4718 | + ep = sctp_sk(sk)->ep; |
4719 | + if (how & SEND_SHUTDOWN && !list_empty(&ep->asocs)) { |
4720 | + struct sctp_association *asoc; |
4721 | + |
4722 | sk->sk_state = SCTP_SS_CLOSING; |
4723 | - ep = sctp_sk(sk)->ep; |
4724 | - if (!list_empty(&ep->asocs)) { |
4725 | - asoc = list_entry(ep->asocs.next, |
4726 | - struct sctp_association, asocs); |
4727 | - sctp_primitive_SHUTDOWN(net, asoc, NULL); |
4728 | - } |
4729 | + asoc = list_entry(ep->asocs.next, |
4730 | + struct sctp_association, asocs); |
4731 | + sctp_primitive_SHUTDOWN(net, asoc, NULL); |
4732 | } |
4733 | } |
4734 | |
4735 | diff --git a/net/socket.c b/net/socket.c |
4736 | index a1bd16106625..03bc2c289c94 100644 |
4737 | --- a/net/socket.c |
4738 | +++ b/net/socket.c |
4739 | @@ -2041,6 +2041,8 @@ int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen, |
4740 | if (err) |
4741 | break; |
4742 | ++datagrams; |
4743 | + if (msg_data_left(&msg_sys)) |
4744 | + break; |
4745 | cond_resched(); |
4746 | } |
4747 | |
4748 | diff --git a/tools/spi/spidev_test.c b/tools/spi/spidev_test.c |
4749 | index f3825b676e38..f046b77cfefe 100644 |
4750 | --- a/tools/spi/spidev_test.c |
4751 | +++ b/tools/spi/spidev_test.c |
4752 | @@ -19,6 +19,7 @@ |
4753 | #include <getopt.h> |
4754 | #include <fcntl.h> |
4755 | #include <sys/ioctl.h> |
4756 | +#include <linux/ioctl.h> |
4757 | #include <sys/stat.h> |
4758 | #include <linux/types.h> |
4759 | #include <linux/spi/spidev.h> |