Magellan Linux

  • Back to homepage
    • /[pkg-src]/trunk/kernel-magellan/patches-4.8/0109-4.8.10-all-fixes.patch

    Contents of /trunk/kernel-magellan/patches-4.8/0109-4.8.10-all-fixes.patch

    Parent Directory Parent Directory | Revision Log Revision Log

    Revision 2849 - (show annotations) (download)
    Tue Nov 22 13:19:55 2016 UTC (7 years, 5 months ago) by niro
    File size: 141674 byte(s) 
    -linux-4.8.10
    1 diff --git a/Makefile b/Makefile
    2 index c1519ab85258..7cf2b4985703 100644
    3 --- a/Makefile
    4 +++ b/Makefile
    5 @@ -1,6 +1,6 @@
    6 VERSION = 4
    7 PATCHLEVEL = 8
    8 -SUBLEVEL = 9
    9 +SUBLEVEL = 10
    10 EXTRAVERSION =
    11 NAME = Psychotic Stoned Sheep
    12
    13 diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
    14 index 37a315d0ddd4..a6847fc05a6d 100644
    15 --- a/arch/sparc/include/asm/uaccess_64.h
    16 +++ b/arch/sparc/include/asm/uaccess_64.h
    17 @@ -98,7 +98,6 @@ struct exception_table_entry {
    18 unsigned int insn, fixup;
    19 };
    20
    21 -void __ret_efault(void);
    22 void __retl_efault(void);
    23
    24 /* Uh, these should become the main single-value transfer routines..
    25 @@ -205,55 +204,34 @@ int __get_user_bad(void);
    26 unsigned long __must_check ___copy_from_user(void *to,
    27 const void __user *from,
    28 unsigned long size);
    29 -unsigned long copy_from_user_fixup(void *to, const void __user *from,
    30 - unsigned long size);
    31 static inline unsigned long __must_check
    32 copy_from_user(void *to, const void __user *from, unsigned long size)
    33 {
    34 - unsigned long ret;
    35 -
    36 check_object_size(to, size, false);
    37
    38 - ret = ___copy_from_user(to, from, size);
    39 - if (unlikely(ret))
    40 - ret = copy_from_user_fixup(to, from, size);
    41 -
    42 - return ret;
    43 + return ___copy_from_user(to, from, size);
    44 }
    45 #define __copy_from_user copy_from_user
    46
    47 unsigned long __must_check ___copy_to_user(void __user *to,
    48 const void *from,
    49 unsigned long size);
    50 -unsigned long copy_to_user_fixup(void __user *to, const void *from,
    51 - unsigned long size);
    52 static inline unsigned long __must_check
    53 copy_to_user(void __user *to, const void *from, unsigned long size)
    54 {
    55 - unsigned long ret;
    56 -
    57 check_object_size(from, size, true);
    58
    59 - ret = ___copy_to_user(to, from, size);
    60 - if (unlikely(ret))
    61 - ret = copy_to_user_fixup(to, from, size);
    62 - return ret;
    63 + return ___copy_to_user(to, from, size);
    64 }
    65 #define __copy_to_user copy_to_user
    66
    67 unsigned long __must_check ___copy_in_user(void __user *to,
    68 const void __user *from,
    69 unsigned long size);
    70 -unsigned long copy_in_user_fixup(void __user *to, void __user *from,
    71 - unsigned long size);
    72 static inline unsigned long __must_check
    73 copy_in_user(void __user *to, void __user *from, unsigned long size)
    74 {
    75 - unsigned long ret = ___copy_in_user(to, from, size);
    76 -
    77 - if (unlikely(ret))
    78 - ret = copy_in_user_fixup(to, from, size);
    79 - return ret;
    80 + return ___copy_in_user(to, from, size);
    81 }
    82 #define __copy_in_user copy_in_user
    83
    84 diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
    85 index a076b4249e62..5f1f3ae21657 100644
    86 --- a/arch/sparc/kernel/head_64.S
    87 +++ b/arch/sparc/kernel/head_64.S
    88 @@ -922,47 +922,11 @@ prom_tba: .xword 0
    89 tlb_type: .word 0 /* Must NOT end up in BSS */
    90 .section ".fixup",#alloc,#execinstr
    91
    92 - .globl __ret_efault, __retl_efault, __ret_one, __retl_one
    93 -ENTRY(__ret_efault)
    94 - ret
    95 - restore %g0, -EFAULT, %o0
    96 -ENDPROC(__ret_efault)
    97 -
    98 ENTRY(__retl_efault)
    99 retl
    100 mov -EFAULT, %o0
    101 ENDPROC(__retl_efault)
    102
    103 -ENTRY(__retl_one)
    104 - retl
    105 - mov 1, %o0
    106 -ENDPROC(__retl_one)
    107 -
    108 -ENTRY(__retl_one_fp)
    109 - VISExitHalf
    110 - retl
    111 - mov 1, %o0
    112 -ENDPROC(__retl_one_fp)
    113 -
    114 -ENTRY(__ret_one_asi)
    115 - wr %g0, ASI_AIUS, %asi
    116 - ret
    117 - restore %g0, 1, %o0
    118 -ENDPROC(__ret_one_asi)
    119 -
    120 -ENTRY(__retl_one_asi)
    121 - wr %g0, ASI_AIUS, %asi
    122 - retl
    123 - mov 1, %o0
    124 -ENDPROC(__retl_one_asi)
    125 -
    126 -ENTRY(__retl_one_asi_fp)
    127 - wr %g0, ASI_AIUS, %asi
    128 - VISExitHalf
    129 - retl
    130 - mov 1, %o0
    131 -ENDPROC(__retl_one_asi_fp)
    132 -
    133 ENTRY(__retl_o1)
    134 retl
    135 mov %o1, %o0
    136 diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
    137 index 59bbeff55024..07933b9e9ce0 100644
    138 --- a/arch/sparc/kernel/jump_label.c
    139 +++ b/arch/sparc/kernel/jump_label.c
    140 @@ -13,19 +13,30 @@
    141 void arch_jump_label_transform(struct jump_entry *entry,
    142 enum jump_label_type type)
    143 {
    144 - u32 val;
    145 u32 *insn = (u32 *) (unsigned long) entry->code;
    146 + u32 val;
    147
    148 if (type == JUMP_LABEL_JMP) {
    149 s32 off = (s32)entry->target - (s32)entry->code;
    150 + bool use_v9_branch = false;
    151 +
    152 + BUG_ON(off & 3);
    153
    154 #ifdef CONFIG_SPARC64
    155 - /* ba,pt %xcc, . + (off << 2) */
    156 - val = 0x10680000 | ((u32) off >> 2);
    157 -#else
    158 - /* ba . + (off << 2) */
    159 - val = 0x10800000 | ((u32) off >> 2);
    160 + if (off <= 0xfffff && off >= -0x100000)
    161 + use_v9_branch = true;
    162 #endif
    163 + if (use_v9_branch) {
    164 + /* WDISP19 - target is . + immed << 2 */
    165 + /* ba,pt %xcc, . + off */
    166 + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff);
    167 + } else {
    168 + /* WDISP22 - target is . + immed << 2 */
    169 + BUG_ON(off > 0x7fffff);
    170 + BUG_ON(off < -0x800000);
    171 + /* ba . + off */
    172 + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff);
    173 + }
    174 } else {
    175 val = 0x01000000;
    176 }
    177 diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c
    178 index 9e034f29dcc5..20ffb052fe38 100644
    179 --- a/arch/sparc/kernel/sparc_ksyms_64.c
    180 +++ b/arch/sparc/kernel/sparc_ksyms_64.c
    181 @@ -27,7 +27,6 @@ EXPORT_SYMBOL(__flushw_user);
    182 EXPORT_SYMBOL_GPL(real_hard_smp_processor_id);
    183
    184 /* from head_64.S */
    185 -EXPORT_SYMBOL(__ret_efault);
    186 EXPORT_SYMBOL(tlb_type);
    187 EXPORT_SYMBOL(sun4v_chip_type);
    188 EXPORT_SYMBOL(prom_root_node);
    189 diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
    190 index b7d0bd6b1406..69a439fa2fc1 100644
    191 --- a/arch/sparc/lib/GENcopy_from_user.S
    192 +++ b/arch/sparc/lib/GENcopy_from_user.S
    193 @@ -3,11 +3,11 @@
    194 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    195 */
    196
    197 -#define EX_LD(x) \
    198 +#define EX_LD(x,y) \
    199 98: x; \
    200 .section __ex_table,"a";\
    201 .align 4; \
    202 - .word 98b, __retl_one; \
    203 + .word 98b, y; \
    204 .text; \
    205 .align 4;
    206
    207 diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
    208 index 780550e1afc7..9947427ce354 100644
    209 --- a/arch/sparc/lib/GENcopy_to_user.S
    210 +++ b/arch/sparc/lib/GENcopy_to_user.S
    211 @@ -3,11 +3,11 @@
    212 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    213 */
    214
    215 -#define EX_ST(x) \
    216 +#define EX_ST(x,y) \
    217 98: x; \
    218 .section __ex_table,"a";\
    219 .align 4; \
    220 - .word 98b, __retl_one; \
    221 + .word 98b, y; \
    222 .text; \
    223 .align 4;
    224
    225 diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S
    226 index 89358ee94851..059ea24ad73d 100644
    227 --- a/arch/sparc/lib/GENmemcpy.S
    228 +++ b/arch/sparc/lib/GENmemcpy.S
    229 @@ -4,21 +4,18 @@
    230 */
    231
    232 #ifdef __KERNEL__
    233 +#include <linux/linkage.h>
    234 #define GLOBAL_SPARE %g7
    235 #else
    236 #define GLOBAL_SPARE %g5
    237 #endif
    238
    239 #ifndef EX_LD
    240 -#define EX_LD(x) x
    241 +#define EX_LD(x,y) x
    242 #endif
    243
    244 #ifndef EX_ST
    245 -#define EX_ST(x) x
    246 -#endif
    247 -
    248 -#ifndef EX_RETVAL
    249 -#define EX_RETVAL(x) x
    250 +#define EX_ST(x,y) x
    251 #endif
    252
    253 #ifndef LOAD
    254 @@ -45,6 +42,29 @@
    255 .register %g3,#scratch
    256
    257 .text
    258 +
    259 +#ifndef EX_RETVAL
    260 +#define EX_RETVAL(x) x
    261 +ENTRY(GEN_retl_o4_1)
    262 + add %o4, %o2, %o4
    263 + retl
    264 + add %o4, 1, %o0
    265 +ENDPROC(GEN_retl_o4_1)
    266 +ENTRY(GEN_retl_g1_8)
    267 + add %g1, %o2, %g1
    268 + retl
    269 + add %g1, 8, %o0
    270 +ENDPROC(GEN_retl_g1_8)
    271 +ENTRY(GEN_retl_o2_4)
    272 + retl
    273 + add %o2, 4, %o0
    274 +ENDPROC(GEN_retl_o2_4)
    275 +ENTRY(GEN_retl_o2_1)
    276 + retl
    277 + add %o2, 1, %o0
    278 +ENDPROC(GEN_retl_o2_1)
    279 +#endif
    280 +
    281 .align 64
    282
    283 .globl FUNC_NAME
    284 @@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    285 sub %g0, %o4, %o4
    286 sub %o2, %o4, %o2
    287 1: subcc %o4, 1, %o4
    288 - EX_LD(LOAD(ldub, %o1, %g1))
    289 - EX_ST(STORE(stb, %g1, %o0))
    290 + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1)
    291 + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1)
    292 add %o1, 1, %o1
    293 bne,pt %XCC, 1b
    294 add %o0, 1, %o0
    295 @@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    296 andn %o2, 0x7, %g1
    297 sub %o2, %g1, %o2
    298 1: subcc %g1, 0x8, %g1
    299 - EX_LD(LOAD(ldx, %o1, %g2))
    300 - EX_ST(STORE(stx, %g2, %o0))
    301 + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8)
    302 + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8)
    303 add %o1, 0x8, %o1
    304 bne,pt %XCC, 1b
    305 add %o0, 0x8, %o0
    306 @@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    307
    308 1:
    309 subcc %o2, 4, %o2
    310 - EX_LD(LOAD(lduw, %o1, %g1))
    311 - EX_ST(STORE(stw, %g1, %o1 + %o3))
    312 + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4)
    313 + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4)
    314 bgu,pt %XCC, 1b
    315 add %o1, 4, %o1
    316
    317 @@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    318 .align 32
    319 90:
    320 subcc %o2, 1, %o2
    321 - EX_LD(LOAD(ldub, %o1, %g1))
    322 - EX_ST(STORE(stb, %g1, %o1 + %o3))
    323 + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1)
    324 + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1)
    325 bgu,pt %XCC, 90b
    326 add %o1, 1, %o1
    327 retl
    328 diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
    329 index 3269b0234093..4f2384a4286a 100644
    330 --- a/arch/sparc/lib/Makefile
    331 +++ b/arch/sparc/lib/Makefile
    332 @@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o
    333 lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
    334 lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
    335
    336 -lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o
    337 +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
    338 lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
    339
    340 obj-$(CONFIG_SPARC64) += iomap.o
    341 diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
    342 index d5242b8c4f94..b79a6998d87c 100644
    343 --- a/arch/sparc/lib/NG2copy_from_user.S
    344 +++ b/arch/sparc/lib/NG2copy_from_user.S
    345 @@ -3,19 +3,19 @@
    346 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    347 */
    348
    349 -#define EX_LD(x) \
    350 +#define EX_LD(x,y) \
    351 98: x; \
    352 .section __ex_table,"a";\
    353 .align 4; \
    354 - .word 98b, __retl_one_asi;\
    355 + .word 98b, y; \
    356 .text; \
    357 .align 4;
    358
    359 -#define EX_LD_FP(x) \
    360 +#define EX_LD_FP(x,y) \
    361 98: x; \
    362 .section __ex_table,"a";\
    363 .align 4; \
    364 - .word 98b, __retl_one_asi_fp;\
    365 + .word 98b, y##_fp; \
    366 .text; \
    367 .align 4;
    368
    369 diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
    370 index 4e962d993b10..dcec55f254ab 100644
    371 --- a/arch/sparc/lib/NG2copy_to_user.S
    372 +++ b/arch/sparc/lib/NG2copy_to_user.S
    373 @@ -3,19 +3,19 @@
    374 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    375 */
    376
    377 -#define EX_ST(x) \
    378 +#define EX_ST(x,y) \
    379 98: x; \
    380 .section __ex_table,"a";\
    381 .align 4; \
    382 - .word 98b, __retl_one_asi;\
    383 + .word 98b, y; \
    384 .text; \
    385 .align 4;
    386
    387 -#define EX_ST_FP(x) \
    388 +#define EX_ST_FP(x,y) \
    389 98: x; \
    390 .section __ex_table,"a";\
    391 .align 4; \
    392 - .word 98b, __retl_one_asi_fp;\
    393 + .word 98b, y##_fp; \
    394 .text; \
    395 .align 4;
    396
    397 diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
    398 index d5f585df2f3f..c629dbd121b6 100644
    399 --- a/arch/sparc/lib/NG2memcpy.S
    400 +++ b/arch/sparc/lib/NG2memcpy.S
    401 @@ -4,6 +4,7 @@
    402 */
    403
    404 #ifdef __KERNEL__
    405 +#include <linux/linkage.h>
    406 #include <asm/visasm.h>
    407 #include <asm/asi.h>
    408 #define GLOBAL_SPARE %g7
    409 @@ -32,21 +33,17 @@
    410 #endif
    411
    412 #ifndef EX_LD
    413 -#define EX_LD(x) x
    414 +#define EX_LD(x,y) x
    415 #endif
    416 #ifndef EX_LD_FP
    417 -#define EX_LD_FP(x) x
    418 +#define EX_LD_FP(x,y) x
    419 #endif
    420
    421 #ifndef EX_ST
    422 -#define EX_ST(x) x
    423 +#define EX_ST(x,y) x
    424 #endif
    425 #ifndef EX_ST_FP
    426 -#define EX_ST_FP(x) x
    427 -#endif
    428 -
    429 -#ifndef EX_RETVAL
    430 -#define EX_RETVAL(x) x
    431 +#define EX_ST_FP(x,y) x
    432 #endif
    433
    434 #ifndef LOAD
    435 @@ -140,45 +137,110 @@
    436 fsrc2 %x6, %f12; \
    437 fsrc2 %x7, %f14;
    438 #define FREG_LOAD_1(base, x0) \
    439 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0))
    440 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1)
    441 #define FREG_LOAD_2(base, x0, x1) \
    442 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    443 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1));
    444 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    445 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1);
    446 #define FREG_LOAD_3(base, x0, x1, x2) \
    447 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    448 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    449 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2));
    450 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    451 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    452 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1);
    453 #define FREG_LOAD_4(base, x0, x1, x2, x3) \
    454 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    455 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    456 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    457 - EX_LD_FP(LOAD(ldd, base + 0x18, %x3));
    458 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    459 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    460 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    461 + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1);
    462 #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
    463 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    464 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    465 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    466 - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    467 - EX_LD_FP(LOAD(ldd, base + 0x20, %x4));
    468 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    469 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    470 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    471 + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    472 + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1);
    473 #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
    474 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    475 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    476 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    477 - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    478 - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
    479 - EX_LD_FP(LOAD(ldd, base + 0x28, %x5));
    480 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    481 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    482 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    483 + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    484 + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
    485 + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1);
    486 #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
    487 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    488 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    489 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    490 - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    491 - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
    492 - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \
    493 - EX_LD_FP(LOAD(ldd, base + 0x30, %x6));
    494 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    495 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    496 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    497 + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    498 + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
    499 + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \
    500 + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1);
    501
    502 .register %g2,#scratch
    503 .register %g3,#scratch
    504
    505 .text
    506 +#ifndef EX_RETVAL
    507 +#define EX_RETVAL(x) x
    508 +__restore_fp:
    509 + VISExitHalf
    510 +__restore_asi:
    511 + retl
    512 + wr %g0, ASI_AIUS, %asi
    513 +ENTRY(NG2_retl_o2)
    514 + ba,pt %xcc, __restore_asi
    515 + mov %o2, %o0
    516 +ENDPROC(NG2_retl_o2)
    517 +ENTRY(NG2_retl_o2_plus_1)
    518 + ba,pt %xcc, __restore_asi
    519 + add %o2, 1, %o0
    520 +ENDPROC(NG2_retl_o2_plus_1)
    521 +ENTRY(NG2_retl_o2_plus_4)
    522 + ba,pt %xcc, __restore_asi
    523 + add %o2, 4, %o0
    524 +ENDPROC(NG2_retl_o2_plus_4)
    525 +ENTRY(NG2_retl_o2_plus_8)
    526 + ba,pt %xcc, __restore_asi
    527 + add %o2, 8, %o0
    528 +ENDPROC(NG2_retl_o2_plus_8)
    529 +ENTRY(NG2_retl_o2_plus_o4_plus_1)
    530 + add %o4, 1, %o4
    531 + ba,pt %xcc, __restore_asi
    532 + add %o2, %o4, %o0
    533 +ENDPROC(NG2_retl_o2_plus_o4_plus_1)
    534 +ENTRY(NG2_retl_o2_plus_o4_plus_8)
    535 + add %o4, 8, %o4
    536 + ba,pt %xcc, __restore_asi
    537 + add %o2, %o4, %o0
    538 +ENDPROC(NG2_retl_o2_plus_o4_plus_8)
    539 +ENTRY(NG2_retl_o2_plus_o4_plus_16)
    540 + add %o4, 16, %o4
    541 + ba,pt %xcc, __restore_asi
    542 + add %o2, %o4, %o0
    543 +ENDPROC(NG2_retl_o2_plus_o4_plus_16)
    544 +ENTRY(NG2_retl_o2_plus_g1_fp)
    545 + ba,pt %xcc, __restore_fp
    546 + add %o2, %g1, %o0
    547 +ENDPROC(NG2_retl_o2_plus_g1_fp)
    548 +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp)
    549 + add %g1, 64, %g1
    550 + ba,pt %xcc, __restore_fp
    551 + add %o2, %g1, %o0
    552 +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp)
    553 +ENTRY(NG2_retl_o2_plus_g1_plus_1)
    554 + add %g1, 1, %g1
    555 + ba,pt %xcc, __restore_asi
    556 + add %o2, %g1, %o0
    557 +ENDPROC(NG2_retl_o2_plus_g1_plus_1)
    558 +ENTRY(NG2_retl_o2_and_7_plus_o4)
    559 + and %o2, 7, %o2
    560 + ba,pt %xcc, __restore_asi
    561 + add %o2, %o4, %o0
    562 +ENDPROC(NG2_retl_o2_and_7_plus_o4)
    563 +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8)
    564 + and %o2, 7, %o2
    565 + add %o4, 8, %o4
    566 + ba,pt %xcc, __restore_asi
    567 + add %o2, %o4, %o0
    568 +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8)
    569 +#endif
    570 +
    571 .align 64
    572
    573 .globl FUNC_NAME
    574 @@ -230,8 +292,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    575 sub %g0, %o4, %o4 ! bytes to align dst
    576 sub %o2, %o4, %o2
    577 1: subcc %o4, 1, %o4
    578 - EX_LD(LOAD(ldub, %o1, %g1))
    579 - EX_ST(STORE(stb, %g1, %o0))
    580 + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1)
    581 + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1)
    582 add %o1, 1, %o1
    583 bne,pt %XCC, 1b
    584 add %o0, 1, %o0
    585 @@ -281,11 +343,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    586 nop
    587 /* fall through for 0 < low bits < 8 */
    588 110: sub %o4, 64, %g2
    589 - EX_LD_FP(LOAD_BLK(%g2, %f0))
    590 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    591 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    592 + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1)
    593 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    594 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    595 FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
    596 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    597 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    598 FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
    599 subcc %g1, 64, %g1
    600 add %o4, 64, %o4
    601 @@ -296,10 +358,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    602
    603 120: sub %o4, 56, %g2
    604 FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
    605 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    606 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    607 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    608 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    609 FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
    610 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    611 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    612 FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
    613 subcc %g1, 64, %g1
    614 add %o4, 64, %o4
    615 @@ -310,10 +372,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    616
    617 130: sub %o4, 48, %g2
    618 FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
    619 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    620 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    621 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    622 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    623 FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
    624 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    625 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    626 FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
    627 subcc %g1, 64, %g1
    628 add %o4, 64, %o4
    629 @@ -324,10 +386,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    630
    631 140: sub %o4, 40, %g2
    632 FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
    633 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    634 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    635 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    636 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    637 FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
    638 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    639 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    640 FREG_MOVE_5(f22, f24, f26, f28, f30)
    641 subcc %g1, 64, %g1
    642 add %o4, 64, %o4
    643 @@ -338,10 +400,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    644
    645 150: sub %o4, 32, %g2
    646 FREG_LOAD_4(%g2, f0, f2, f4, f6)
    647 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    648 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    649 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    650 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    651 FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
    652 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    653 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    654 FREG_MOVE_4(f24, f26, f28, f30)
    655 subcc %g1, 64, %g1
    656 add %o4, 64, %o4
    657 @@ -352,10 +414,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    658
    659 160: sub %o4, 24, %g2
    660 FREG_LOAD_3(%g2, f0, f2, f4)
    661 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    662 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    663 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    664 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    665 FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
    666 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    667 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    668 FREG_MOVE_3(f26, f28, f30)
    669 subcc %g1, 64, %g1
    670 add %o4, 64, %o4
    671 @@ -366,10 +428,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    672
    673 170: sub %o4, 16, %g2
    674 FREG_LOAD_2(%g2, f0, f2)
    675 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    676 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    677 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    678 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    679 FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
    680 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    681 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    682 FREG_MOVE_2(f28, f30)
    683 subcc %g1, 64, %g1
    684 add %o4, 64, %o4
    685 @@ -380,10 +442,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    686
    687 180: sub %o4, 8, %g2
    688 FREG_LOAD_1(%g2, f0)
    689 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    690 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    691 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    692 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    693 FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
    694 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    695 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    696 FREG_MOVE_1(f30)
    697 subcc %g1, 64, %g1
    698 add %o4, 64, %o4
    699 @@ -393,10 +455,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    700 nop
    701
    702 190:
    703 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    704 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    705 subcc %g1, 64, %g1
    706 - EX_LD_FP(LOAD_BLK(%o4, %f0))
    707 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    708 + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64)
    709 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64)
    710 add %o4, 64, %o4
    711 bne,pt %xcc, 1b
    712 LOAD(prefetch, %o4 + 64, #one_read)
    713 @@ -423,28 +485,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    714 andn %o2, 0xf, %o4
    715 and %o2, 0xf, %o2
    716 1: subcc %o4, 0x10, %o4
    717 - EX_LD(LOAD(ldx, %o1, %o5))
    718 + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16)
    719 add %o1, 0x08, %o1
    720 - EX_LD(LOAD(ldx, %o1, %g1))
    721 + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16)
    722 sub %o1, 0x08, %o1
    723 - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
    724 + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16)
    725 add %o1, 0x8, %o1
    726 - EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE))
    727 + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8)
    728 bgu,pt %XCC, 1b
    729 add %o1, 0x8, %o1
    730 73: andcc %o2, 0x8, %g0
    731 be,pt %XCC, 1f
    732 nop
    733 sub %o2, 0x8, %o2
    734 - EX_LD(LOAD(ldx, %o1, %o5))
    735 - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
    736 + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8)
    737 + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8)
    738 add %o1, 0x8, %o1
    739 1: andcc %o2, 0x4, %g0
    740 be,pt %XCC, 1f
    741 nop
    742 sub %o2, 0x4, %o2
    743 - EX_LD(LOAD(lduw, %o1, %o5))
    744 - EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE))
    745 + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4)
    746 + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
    747 add %o1, 0x4, %o1
    748 1: cmp %o2, 0
    749 be,pt %XCC, 85f
    750 @@ -460,8 +522,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    751 sub %o2, %g1, %o2
    752
    753 1: subcc %g1, 1, %g1
    754 - EX_LD(LOAD(ldub, %o1, %o5))
    755 - EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE))
    756 + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1)
    757 + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1)
    758 bgu,pt %icc, 1b
    759 add %o1, 1, %o1
    760
    761 @@ -477,16 +539,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    762
    763 8: mov 64, GLOBAL_SPARE
    764 andn %o1, 0x7, %o1
    765 - EX_LD(LOAD(ldx, %o1, %g2))
    766 + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2)
    767 sub GLOBAL_SPARE, %g1, GLOBAL_SPARE
    768 andn %o2, 0x7, %o4
    769 sllx %g2, %g1, %g2
    770 1: add %o1, 0x8, %o1
    771 - EX_LD(LOAD(ldx, %o1, %g3))
    772 + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4)
    773 subcc %o4, 0x8, %o4
    774 srlx %g3, GLOBAL_SPARE, %o5
    775 or %o5, %g2, %o5
    776 - EX_ST(STORE(stx, %o5, %o0))
    777 + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8)
    778 add %o0, 0x8, %o0
    779 bgu,pt %icc, 1b
    780 sllx %g3, %g1, %g2
    781 @@ -506,8 +568,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    782
    783 1:
    784 subcc %o2, 4, %o2
    785 - EX_LD(LOAD(lduw, %o1, %g1))
    786 - EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE))
    787 + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4)
    788 + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
    789 bgu,pt %XCC, 1b
    790 add %o1, 4, %o1
    791
    792 @@ -517,8 +579,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    793 .align 32
    794 90:
    795 subcc %o2, 1, %o2
    796 - EX_LD(LOAD(ldub, %o1, %g1))
    797 - EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE))
    798 + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1)
    799 + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1)
    800 bgu,pt %XCC, 90b
    801 add %o1, 1, %o1
    802 retl
    803 diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S
    804 index 2e8ee7ad07a9..16a286c1a528 100644
    805 --- a/arch/sparc/lib/NG4copy_from_user.S
    806 +++ b/arch/sparc/lib/NG4copy_from_user.S
    807 @@ -3,19 +3,19 @@
    808 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
    809 */
    810
    811 -#define EX_LD(x) \
    812 +#define EX_LD(x, y) \
    813 98: x; \
    814 .section __ex_table,"a";\
    815 .align 4; \
    816 - .word 98b, __retl_one_asi;\
    817 + .word 98b, y; \
    818 .text; \
    819 .align 4;
    820
    821 -#define EX_LD_FP(x) \
    822 +#define EX_LD_FP(x,y) \
    823 98: x; \
    824 .section __ex_table,"a";\
    825 .align 4; \
    826 - .word 98b, __retl_one_asi_fp;\
    827 + .word 98b, y##_fp; \
    828 .text; \
    829 .align 4;
    830
    831 diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S
    832 index be0bf4590df8..6b0276ffc858 100644
    833 --- a/arch/sparc/lib/NG4copy_to_user.S
    834 +++ b/arch/sparc/lib/NG4copy_to_user.S
    835 @@ -3,19 +3,19 @@
    836 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
    837 */
    838
    839 -#define EX_ST(x) \
    840 +#define EX_ST(x,y) \
    841 98: x; \
    842 .section __ex_table,"a";\
    843 .align 4; \
    844 - .word 98b, __retl_one_asi;\
    845 + .word 98b, y; \
    846 .text; \
    847 .align 4;
    848
    849 -#define EX_ST_FP(x) \
    850 +#define EX_ST_FP(x,y) \
    851 98: x; \
    852 .section __ex_table,"a";\
    853 .align 4; \
    854 - .word 98b, __retl_one_asi_fp;\
    855 + .word 98b, y##_fp; \
    856 .text; \
    857 .align 4;
    858
    859 diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S
    860 index 8e13ee1f4454..75bb93b1437f 100644
    861 --- a/arch/sparc/lib/NG4memcpy.S
    862 +++ b/arch/sparc/lib/NG4memcpy.S
    863 @@ -4,6 +4,7 @@
    864 */
    865
    866 #ifdef __KERNEL__
    867 +#include <linux/linkage.h>
    868 #include <asm/visasm.h>
    869 #include <asm/asi.h>
    870 #define GLOBAL_SPARE %g7
    871 @@ -46,22 +47,19 @@
    872 #endif
    873
    874 #ifndef EX_LD
    875 -#define EX_LD(x) x
    876 +#define EX_LD(x,y) x
    877 #endif
    878 #ifndef EX_LD_FP
    879 -#define EX_LD_FP(x) x
    880 +#define EX_LD_FP(x,y) x
    881 #endif
    882
    883 #ifndef EX_ST
    884 -#define EX_ST(x) x
    885 +#define EX_ST(x,y) x
    886 #endif
    887 #ifndef EX_ST_FP
    888 -#define EX_ST_FP(x) x
    889 +#define EX_ST_FP(x,y) x
    890 #endif
    891
    892 -#ifndef EX_RETVAL
    893 -#define EX_RETVAL(x) x
    894 -#endif
    895
    896 #ifndef LOAD
    897 #define LOAD(type,addr,dest) type [addr], dest
    898 @@ -94,6 +92,158 @@
    899 .register %g3,#scratch
    900
    901 .text
    902 +#ifndef EX_RETVAL
    903 +#define EX_RETVAL(x) x
    904 +__restore_asi_fp:
    905 + VISExitHalf
    906 +__restore_asi:
    907 + retl
    908 + wr %g0, ASI_AIUS, %asi
    909 +
    910 +ENTRY(NG4_retl_o2)
    911 + ba,pt %xcc, __restore_asi
    912 + mov %o2, %o0
    913 +ENDPROC(NG4_retl_o2)
    914 +ENTRY(NG4_retl_o2_plus_1)
    915 + ba,pt %xcc, __restore_asi
    916 + add %o2, 1, %o0
    917 +ENDPROC(NG4_retl_o2_plus_1)
    918 +ENTRY(NG4_retl_o2_plus_4)
    919 + ba,pt %xcc, __restore_asi
    920 + add %o2, 4, %o0
    921 +ENDPROC(NG4_retl_o2_plus_4)
    922 +ENTRY(NG4_retl_o2_plus_o5)
    923 + ba,pt %xcc, __restore_asi
    924 + add %o2, %o5, %o0
    925 +ENDPROC(NG4_retl_o2_plus_o5)
    926 +ENTRY(NG4_retl_o2_plus_o5_plus_4)
    927 + add %o5, 4, %o5
    928 + ba,pt %xcc, __restore_asi
    929 + add %o2, %o5, %o0
    930 +ENDPROC(NG4_retl_o2_plus_o5_plus_4)
    931 +ENTRY(NG4_retl_o2_plus_o5_plus_8)
    932 + add %o5, 8, %o5
    933 + ba,pt %xcc, __restore_asi
    934 + add %o2, %o5, %o0
    935 +ENDPROC(NG4_retl_o2_plus_o5_plus_8)
    936 +ENTRY(NG4_retl_o2_plus_o5_plus_16)
    937 + add %o5, 16, %o5
    938 + ba,pt %xcc, __restore_asi
    939 + add %o2, %o5, %o0
    940 +ENDPROC(NG4_retl_o2_plus_o5_plus_16)
    941 +ENTRY(NG4_retl_o2_plus_o5_plus_24)
    942 + add %o5, 24, %o5
    943 + ba,pt %xcc, __restore_asi
    944 + add %o2, %o5, %o0
    945 +ENDPROC(NG4_retl_o2_plus_o5_plus_24)
    946 +ENTRY(NG4_retl_o2_plus_o5_plus_32)
    947 + add %o5, 32, %o5
    948 + ba,pt %xcc, __restore_asi
    949 + add %o2, %o5, %o0
    950 +ENDPROC(NG4_retl_o2_plus_o5_plus_32)
    951 +ENTRY(NG4_retl_o2_plus_g1)
    952 + ba,pt %xcc, __restore_asi
    953 + add %o2, %g1, %o0
    954 +ENDPROC(NG4_retl_o2_plus_g1)
    955 +ENTRY(NG4_retl_o2_plus_g1_plus_1)
    956 + add %g1, 1, %g1
    957 + ba,pt %xcc, __restore_asi
    958 + add %o2, %g1, %o0
    959 +ENDPROC(NG4_retl_o2_plus_g1_plus_1)
    960 +ENTRY(NG4_retl_o2_plus_g1_plus_8)
    961 + add %g1, 8, %g1
    962 + ba,pt %xcc, __restore_asi
    963 + add %o2, %g1, %o0
    964 +ENDPROC(NG4_retl_o2_plus_g1_plus_8)
    965 +ENTRY(NG4_retl_o2_plus_o4)
    966 + ba,pt %xcc, __restore_asi
    967 + add %o2, %o4, %o0
    968 +ENDPROC(NG4_retl_o2_plus_o4)
    969 +ENTRY(NG4_retl_o2_plus_o4_plus_8)
    970 + add %o4, 8, %o4
    971 + ba,pt %xcc, __restore_asi
    972 + add %o2, %o4, %o0
    973 +ENDPROC(NG4_retl_o2_plus_o4_plus_8)
    974 +ENTRY(NG4_retl_o2_plus_o4_plus_16)
    975 + add %o4, 16, %o4
    976 + ba,pt %xcc, __restore_asi
    977 + add %o2, %o4, %o0
    978 +ENDPROC(NG4_retl_o2_plus_o4_plus_16)
    979 +ENTRY(NG4_retl_o2_plus_o4_plus_24)
    980 + add %o4, 24, %o4
    981 + ba,pt %xcc, __restore_asi
    982 + add %o2, %o4, %o0
    983 +ENDPROC(NG4_retl_o2_plus_o4_plus_24)
    984 +ENTRY(NG4_retl_o2_plus_o4_plus_32)
    985 + add %o4, 32, %o4
    986 + ba,pt %xcc, __restore_asi
    987 + add %o2, %o4, %o0
    988 +ENDPROC(NG4_retl_o2_plus_o4_plus_32)
    989 +ENTRY(NG4_retl_o2_plus_o4_plus_40)
    990 + add %o4, 40, %o4
    991 + ba,pt %xcc, __restore_asi
    992 + add %o2, %o4, %o0
    993 +ENDPROC(NG4_retl_o2_plus_o4_plus_40)
    994 +ENTRY(NG4_retl_o2_plus_o4_plus_48)
    995 + add %o4, 48, %o4
    996 + ba,pt %xcc, __restore_asi
    997 + add %o2, %o4, %o0
    998 +ENDPROC(NG4_retl_o2_plus_o4_plus_48)
    999 +ENTRY(NG4_retl_o2_plus_o4_plus_56)
    1000 + add %o4, 56, %o4
    1001 + ba,pt %xcc, __restore_asi
    1002 + add %o2, %o4, %o0
    1003 +ENDPROC(NG4_retl_o2_plus_o4_plus_56)
    1004 +ENTRY(NG4_retl_o2_plus_o4_plus_64)
    1005 + add %o4, 64, %o4
    1006 + ba,pt %xcc, __restore_asi
    1007 + add %o2, %o4, %o0
    1008 +ENDPROC(NG4_retl_o2_plus_o4_plus_64)
    1009 +ENTRY(NG4_retl_o2_plus_o4_fp)
    1010 + ba,pt %xcc, __restore_asi_fp
    1011 + add %o2, %o4, %o0
    1012 +ENDPROC(NG4_retl_o2_plus_o4_fp)
    1013 +ENTRY(NG4_retl_o2_plus_o4_plus_8_fp)
    1014 + add %o4, 8, %o4
    1015 + ba,pt %xcc, __restore_asi_fp
    1016 + add %o2, %o4, %o0
    1017 +ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp)
    1018 +ENTRY(NG4_retl_o2_plus_o4_plus_16_fp)
    1019 + add %o4, 16, %o4
    1020 + ba,pt %xcc, __restore_asi_fp
    1021 + add %o2, %o4, %o0
    1022 +ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp)
    1023 +ENTRY(NG4_retl_o2_plus_o4_plus_24_fp)
    1024 + add %o4, 24, %o4
    1025 + ba,pt %xcc, __restore_asi_fp
    1026 + add %o2, %o4, %o0
    1027 +ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp)
    1028 +ENTRY(NG4_retl_o2_plus_o4_plus_32_fp)
    1029 + add %o4, 32, %o4
    1030 + ba,pt %xcc, __restore_asi_fp
    1031 + add %o2, %o4, %o0
    1032 +ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp)
    1033 +ENTRY(NG4_retl_o2_plus_o4_plus_40_fp)
    1034 + add %o4, 40, %o4
    1035 + ba,pt %xcc, __restore_asi_fp
    1036 + add %o2, %o4, %o0
    1037 +ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp)
    1038 +ENTRY(NG4_retl_o2_plus_o4_plus_48_fp)
    1039 + add %o4, 48, %o4
    1040 + ba,pt %xcc, __restore_asi_fp
    1041 + add %o2, %o4, %o0
    1042 +ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp)
    1043 +ENTRY(NG4_retl_o2_plus_o4_plus_56_fp)
    1044 + add %o4, 56, %o4
    1045 + ba,pt %xcc, __restore_asi_fp
    1046 + add %o2, %o4, %o0
    1047 +ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp)
    1048 +ENTRY(NG4_retl_o2_plus_o4_plus_64_fp)
    1049 + add %o4, 64, %o4
    1050 + ba,pt %xcc, __restore_asi_fp
    1051 + add %o2, %o4, %o0
    1052 +ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp)
    1053 +#endif
    1054 .align 64
    1055
    1056 .globl FUNC_NAME
    1057 @@ -124,12 +274,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1058 brz,pt %g1, 51f
    1059 sub %o2, %g1, %o2
    1060
    1061 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
    1062 +
    1063 +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1064 add %o1, 1, %o1
    1065 subcc %g1, 1, %g1
    1066 add %o0, 1, %o0
    1067 bne,pt %icc, 1b
    1068 - EX_ST(STORE(stb, %g2, %o0 - 0x01))
    1069 + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
    1070
    1071 51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong)
    1072 LOAD(prefetch, %o1 + 0x080, #n_reads_strong)
    1073 @@ -154,43 +305,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1074 brz,pt %g1, .Llarge_aligned
    1075 sub %o2, %g1, %o2
    1076
    1077 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2))
    1078 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1079 add %o1, 8, %o1
    1080 subcc %g1, 8, %g1
    1081 add %o0, 8, %o0
    1082 bne,pt %icc, 1b
    1083 - EX_ST(STORE(stx, %g2, %o0 - 0x08))
    1084 + EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8)
    1085
    1086 .Llarge_aligned:
    1087 /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */
    1088 andn %o2, 0x3f, %o4
    1089 sub %o2, %o4, %o2
    1090
    1091 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1092 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4)
    1093 add %o1, 0x40, %o1
    1094 - EX_LD(LOAD(ldx, %o1 - 0x38, %g2))
    1095 + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4)
    1096 subcc %o4, 0x40, %o4
    1097 - EX_LD(LOAD(ldx, %o1 - 0x30, %g3))
    1098 - EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE))
    1099 - EX_LD(LOAD(ldx, %o1 - 0x20, %o5))
    1100 - EX_ST(STORE_INIT(%g1, %o0))
    1101 + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64)
    1102 + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64)
    1103 + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64)
    1104 + EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64)
    1105 add %o0, 0x08, %o0
    1106 - EX_ST(STORE_INIT(%g2, %o0))
    1107 + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56)
    1108 add %o0, 0x08, %o0
    1109 - EX_LD(LOAD(ldx, %o1 - 0x18, %g2))
    1110 - EX_ST(STORE_INIT(%g3, %o0))
    1111 + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48)
    1112 + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48)
    1113 add %o0, 0x08, %o0
    1114 - EX_LD(LOAD(ldx, %o1 - 0x10, %g3))
    1115 - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
    1116 + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40)
    1117 + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40)
    1118 add %o0, 0x08, %o0
    1119 - EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE))
    1120 - EX_ST(STORE_INIT(%o5, %o0))
    1121 + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32)
    1122 + EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32)
    1123 add %o0, 0x08, %o0
    1124 - EX_ST(STORE_INIT(%g2, %o0))
    1125 + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24)
    1126 add %o0, 0x08, %o0
    1127 - EX_ST(STORE_INIT(%g3, %o0))
    1128 + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16)
    1129 add %o0, 0x08, %o0
    1130 - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
    1131 + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8)
    1132 add %o0, 0x08, %o0
    1133 bne,pt %icc, 1b
    1134 LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
    1135 @@ -216,17 +367,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1136 sub %o2, %o4, %o2
    1137 alignaddr %o1, %g0, %g1
    1138 add %o1, %o4, %o1
    1139 - EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0))
    1140 -1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2))
    1141 + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4)
    1142 +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4)
    1143 subcc %o4, 0x40, %o4
    1144 - EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4))
    1145 - EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6))
    1146 - EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8))
    1147 - EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10))
    1148 - EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12))
    1149 - EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14))
    1150 + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64)
    1151 + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64)
    1152 + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64)
    1153 + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64)
    1154 + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64)
    1155 + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64)
    1156 faligndata %f0, %f2, %f16
    1157 - EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0))
    1158 + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64)
    1159 faligndata %f2, %f4, %f18
    1160 add %g1, 0x40, %g1
    1161 faligndata %f4, %f6, %f20
    1162 @@ -235,14 +386,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1163 faligndata %f10, %f12, %f26
    1164 faligndata %f12, %f14, %f28
    1165 faligndata %f14, %f0, %f30
    1166 - EX_ST_FP(STORE(std, %f16, %o0 + 0x00))
    1167 - EX_ST_FP(STORE(std, %f18, %o0 + 0x08))
    1168 - EX_ST_FP(STORE(std, %f20, %o0 + 0x10))
    1169 - EX_ST_FP(STORE(std, %f22, %o0 + 0x18))
    1170 - EX_ST_FP(STORE(std, %f24, %o0 + 0x20))
    1171 - EX_ST_FP(STORE(std, %f26, %o0 + 0x28))
    1172 - EX_ST_FP(STORE(std, %f28, %o0 + 0x30))
    1173 - EX_ST_FP(STORE(std, %f30, %o0 + 0x38))
    1174 + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64)
    1175 + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56)
    1176 + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48)
    1177 + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40)
    1178 + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32)
    1179 + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24)
    1180 + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16)
    1181 + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8)
    1182 add %o0, 0x40, %o0
    1183 bne,pt %icc, 1b
    1184 LOAD(prefetch, %g1 + 0x200, #n_reads_strong)
    1185 @@ -270,37 +421,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1186 andncc %o2, 0x20 - 1, %o5
    1187 be,pn %icc, 2f
    1188 sub %o2, %o5, %o2
    1189 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1190 - EX_LD(LOAD(ldx, %o1 + 0x08, %g2))
    1191 - EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE))
    1192 - EX_LD(LOAD(ldx, %o1 + 0x18, %o4))
    1193 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1194 + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5)
    1195 + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5)
    1196 + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5)
    1197 add %o1, 0x20, %o1
    1198 subcc %o5, 0x20, %o5
    1199 - EX_ST(STORE(stx, %g1, %o0 + 0x00))
    1200 - EX_ST(STORE(stx, %g2, %o0 + 0x08))
    1201 - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10))
    1202 - EX_ST(STORE(stx, %o4, %o0 + 0x18))
    1203 + EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32)
    1204 + EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24)
    1205 + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24)
    1206 + EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8)
    1207 bne,pt %icc, 1b
    1208 add %o0, 0x20, %o0
    1209 2: andcc %o2, 0x18, %o5
    1210 be,pt %icc, 3f
    1211 sub %o2, %o5, %o2
    1212 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1213 +
    1214 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1215 add %o1, 0x08, %o1
    1216 add %o0, 0x08, %o0
    1217 subcc %o5, 0x08, %o5
    1218 bne,pt %icc, 1b
    1219 - EX_ST(STORE(stx, %g1, %o0 - 0x08))
    1220 + EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8)
    1221 3: brz,pt %o2, .Lexit
    1222 cmp %o2, 0x04
    1223 bl,pn %icc, .Ltiny
    1224 nop
    1225 - EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
    1226 + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2)
    1227 add %o1, 0x04, %o1
    1228 add %o0, 0x04, %o0
    1229 subcc %o2, 0x04, %o2
    1230 bne,pn %icc, .Ltiny
    1231 - EX_ST(STORE(stw, %g1, %o0 - 0x04))
    1232 + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4)
    1233 ba,a,pt %icc, .Lexit
    1234 .Lmedium_unaligned:
    1235 /* First get dest 8 byte aligned. */
    1236 @@ -309,12 +461,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1237 brz,pt %g1, 2f
    1238 sub %o2, %g1, %o2
    1239
    1240 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
    1241 +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1242 add %o1, 1, %o1
    1243 subcc %g1, 1, %g1
    1244 add %o0, 1, %o0
    1245 bne,pt %icc, 1b
    1246 - EX_ST(STORE(stb, %g2, %o0 - 0x01))
    1247 + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
    1248 2:
    1249 and %o1, 0x7, %g1
    1250 brz,pn %g1, .Lmedium_noprefetch
    1251 @@ -322,16 +474,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1252 mov 64, %g2
    1253 sub %g2, %g1, %g2
    1254 andn %o1, 0x7, %o1
    1255 - EX_LD(LOAD(ldx, %o1 + 0x00, %o4))
    1256 + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2)
    1257 sllx %o4, %g1, %o4
    1258 andn %o2, 0x08 - 1, %o5
    1259 sub %o2, %o5, %o2
    1260 -1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3))
    1261 +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5)
    1262 add %o1, 0x08, %o1
    1263 subcc %o5, 0x08, %o5
    1264 srlx %g3, %g2, GLOBAL_SPARE
    1265 or GLOBAL_SPARE, %o4, GLOBAL_SPARE
    1266 - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00))
    1267 + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8)
    1268 add %o0, 0x08, %o0
    1269 bne,pt %icc, 1b
    1270 sllx %g3, %g1, %o4
    1271 @@ -342,17 +494,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1272 ba,pt %icc, .Lsmall_unaligned
    1273
    1274 .Ltiny:
    1275 - EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
    1276 + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
    1277 subcc %o2, 1, %o2
    1278 be,pn %icc, .Lexit
    1279 - EX_ST(STORE(stb, %g1, %o0 + 0x00))
    1280 - EX_LD(LOAD(ldub, %o1 + 0x01, %g1))
    1281 + EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1)
    1282 + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2)
    1283 subcc %o2, 1, %o2
    1284 be,pn %icc, .Lexit
    1285 - EX_ST(STORE(stb, %g1, %o0 + 0x01))
    1286 - EX_LD(LOAD(ldub, %o1 + 0x02, %g1))
    1287 + EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1)
    1288 + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2)
    1289 ba,pt %icc, .Lexit
    1290 - EX_ST(STORE(stb, %g1, %o0 + 0x02))
    1291 + EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2)
    1292
    1293 .Lsmall:
    1294 andcc %g2, 0x3, %g0
    1295 @@ -360,22 +512,22 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1296 andn %o2, 0x4 - 1, %o5
    1297 sub %o2, %o5, %o2
    1298 1:
    1299 - EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
    1300 + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1301 add %o1, 0x04, %o1
    1302 subcc %o5, 0x04, %o5
    1303 add %o0, 0x04, %o0
    1304 bne,pt %icc, 1b
    1305 - EX_ST(STORE(stw, %g1, %o0 - 0x04))
    1306 + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4)
    1307 brz,pt %o2, .Lexit
    1308 nop
    1309 ba,a,pt %icc, .Ltiny
    1310
    1311 .Lsmall_unaligned:
    1312 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
    1313 +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
    1314 add %o1, 1, %o1
    1315 add %o0, 1, %o0
    1316 subcc %o2, 1, %o2
    1317 bne,pt %icc, 1b
    1318 - EX_ST(STORE(stb, %g1, %o0 - 0x01))
    1319 + EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1)
    1320 ba,a,pt %icc, .Lexit
    1321 .size FUNC_NAME, .-FUNC_NAME
    1322 diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
    1323 index 5d1e4d1ac21e..9cd42fcbc781 100644
    1324 --- a/arch/sparc/lib/NGcopy_from_user.S
    1325 +++ b/arch/sparc/lib/NGcopy_from_user.S
    1326 @@ -3,11 +3,11 @@
    1327 * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
    1328 */
    1329
    1330 -#define EX_LD(x) \
    1331 +#define EX_LD(x,y) \
    1332 98: x; \
    1333 .section __ex_table,"a";\
    1334 .align 4; \
    1335 - .word 98b, __ret_one_asi;\
    1336 + .word 98b, y; \
    1337 .text; \
    1338 .align 4;
    1339
    1340 diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
    1341 index ff630dcb273c..5c358afd464e 100644
    1342 --- a/arch/sparc/lib/NGcopy_to_user.S
    1343 +++ b/arch/sparc/lib/NGcopy_to_user.S
    1344 @@ -3,11 +3,11 @@
    1345 * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
    1346 */
    1347
    1348 -#define EX_ST(x) \
    1349 +#define EX_ST(x,y) \
    1350 98: x; \
    1351 .section __ex_table,"a";\
    1352 .align 4; \
    1353 - .word 98b, __ret_one_asi;\
    1354 + .word 98b, y; \
    1355 .text; \
    1356 .align 4;
    1357
    1358 diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S
    1359 index 96a14caf6966..d88c4ed50a00 100644
    1360 --- a/arch/sparc/lib/NGmemcpy.S
    1361 +++ b/arch/sparc/lib/NGmemcpy.S
    1362 @@ -4,6 +4,7 @@
    1363 */
    1364
    1365 #ifdef __KERNEL__
    1366 +#include <linux/linkage.h>
    1367 #include <asm/asi.h>
    1368 #include <asm/thread_info.h>
    1369 #define GLOBAL_SPARE %g7
    1370 @@ -27,15 +28,11 @@
    1371 #endif
    1372
    1373 #ifndef EX_LD
    1374 -#define EX_LD(x) x
    1375 +#define EX_LD(x,y) x
    1376 #endif
    1377
    1378 #ifndef EX_ST
    1379 -#define EX_ST(x) x
    1380 -#endif
    1381 -
    1382 -#ifndef EX_RETVAL
    1383 -#define EX_RETVAL(x) x
    1384 +#define EX_ST(x,y) x
    1385 #endif
    1386
    1387 #ifndef LOAD
    1388 @@ -79,6 +76,92 @@
    1389 .register %g3,#scratch
    1390
    1391 .text
    1392 +#ifndef EX_RETVAL
    1393 +#define EX_RETVAL(x) x
    1394 +__restore_asi:
    1395 + ret
    1396 + wr %g0, ASI_AIUS, %asi
    1397 + restore
    1398 +ENTRY(NG_ret_i2_plus_i4_plus_1)
    1399 + ba,pt %xcc, __restore_asi
    1400 + add %i2, %i5, %i0
    1401 +ENDPROC(NG_ret_i2_plus_i4_plus_1)
    1402 +ENTRY(NG_ret_i2_plus_g1)
    1403 + ba,pt %xcc, __restore_asi
    1404 + add %i2, %g1, %i0
    1405 +ENDPROC(NG_ret_i2_plus_g1)
    1406 +ENTRY(NG_ret_i2_plus_g1_minus_8)
    1407 + sub %g1, 8, %g1
    1408 + ba,pt %xcc, __restore_asi
    1409 + add %i2, %g1, %i0
    1410 +ENDPROC(NG_ret_i2_plus_g1_minus_8)
    1411 +ENTRY(NG_ret_i2_plus_g1_minus_16)
    1412 + sub %g1, 16, %g1
    1413 + ba,pt %xcc, __restore_asi
    1414 + add %i2, %g1, %i0
    1415 +ENDPROC(NG_ret_i2_plus_g1_minus_16)
    1416 +ENTRY(NG_ret_i2_plus_g1_minus_24)
    1417 + sub %g1, 24, %g1
    1418 + ba,pt %xcc, __restore_asi
    1419 + add %i2, %g1, %i0
    1420 +ENDPROC(NG_ret_i2_plus_g1_minus_24)
    1421 +ENTRY(NG_ret_i2_plus_g1_minus_32)
    1422 + sub %g1, 32, %g1
    1423 + ba,pt %xcc, __restore_asi
    1424 + add %i2, %g1, %i0
    1425 +ENDPROC(NG_ret_i2_plus_g1_minus_32)
    1426 +ENTRY(NG_ret_i2_plus_g1_minus_40)
    1427 + sub %g1, 40, %g1
    1428 + ba,pt %xcc, __restore_asi
    1429 + add %i2, %g1, %i0
    1430 +ENDPROC(NG_ret_i2_plus_g1_minus_40)
    1431 +ENTRY(NG_ret_i2_plus_g1_minus_48)
    1432 + sub %g1, 48, %g1
    1433 + ba,pt %xcc, __restore_asi
    1434 + add %i2, %g1, %i0
    1435 +ENDPROC(NG_ret_i2_plus_g1_minus_48)
    1436 +ENTRY(NG_ret_i2_plus_g1_minus_56)
    1437 + sub %g1, 56, %g1
    1438 + ba,pt %xcc, __restore_asi
    1439 + add %i2, %g1, %i0
    1440 +ENDPROC(NG_ret_i2_plus_g1_minus_56)
    1441 +ENTRY(NG_ret_i2_plus_i4)
    1442 + ba,pt %xcc, __restore_asi
    1443 + add %i2, %i4, %i0
    1444 +ENDPROC(NG_ret_i2_plus_i4)
    1445 +ENTRY(NG_ret_i2_plus_i4_minus_8)
    1446 + sub %i4, 8, %i4
    1447 + ba,pt %xcc, __restore_asi
    1448 + add %i2, %i4, %i0
    1449 +ENDPROC(NG_ret_i2_plus_i4_minus_8)
    1450 +ENTRY(NG_ret_i2_plus_8)
    1451 + ba,pt %xcc, __restore_asi
    1452 + add %i2, 8, %i0
    1453 +ENDPROC(NG_ret_i2_plus_8)
    1454 +ENTRY(NG_ret_i2_plus_4)
    1455 + ba,pt %xcc, __restore_asi
    1456 + add %i2, 4, %i0
    1457 +ENDPROC(NG_ret_i2_plus_4)
    1458 +ENTRY(NG_ret_i2_plus_1)
    1459 + ba,pt %xcc, __restore_asi
    1460 + add %i2, 1, %i0
    1461 +ENDPROC(NG_ret_i2_plus_1)
    1462 +ENTRY(NG_ret_i2_plus_g1_plus_1)
    1463 + add %g1, 1, %g1
    1464 + ba,pt %xcc, __restore_asi
    1465 + add %i2, %g1, %i0
    1466 +ENDPROC(NG_ret_i2_plus_g1_plus_1)
    1467 +ENTRY(NG_ret_i2)
    1468 + ba,pt %xcc, __restore_asi
    1469 + mov %i2, %i0
    1470 +ENDPROC(NG_ret_i2)
    1471 +ENTRY(NG_ret_i2_and_7_plus_i4)
    1472 + and %i2, 7, %i2
    1473 + ba,pt %xcc, __restore_asi
    1474 + add %i2, %i4, %i0
    1475 +ENDPROC(NG_ret_i2_and_7_plus_i4)
    1476 +#endif
    1477 +
    1478 .align 64
    1479
    1480 .globl FUNC_NAME
    1481 @@ -126,8 +209,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1482 sub %g0, %i4, %i4 ! bytes to align dst
    1483 sub %i2, %i4, %i2
    1484 1: subcc %i4, 1, %i4
    1485 - EX_LD(LOAD(ldub, %i1, %g1))
    1486 - EX_ST(STORE(stb, %g1, %o0))
    1487 + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1)
    1488 + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1)
    1489 add %i1, 1, %i1
    1490 bne,pt %XCC, 1b
    1491 add %o0, 1, %o0
    1492 @@ -160,7 +243,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1493 and %i4, 0x7, GLOBAL_SPARE
    1494 sll GLOBAL_SPARE, 3, GLOBAL_SPARE
    1495 mov 64, %i5
    1496 - EX_LD(LOAD_TWIN(%i1, %g2, %g3))
    1497 + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1)
    1498 sub %i5, GLOBAL_SPARE, %i5
    1499 mov 16, %o4
    1500 mov 32, %o5
    1501 @@ -178,31 +261,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1502 srlx WORD3, PRE_SHIFT, TMP; \
    1503 or WORD2, TMP, WORD2;
    1504
    1505 -8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
    1506 +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
    1507 MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
    1508 LOAD(prefetch, %i1 + %i3, #one_read)
    1509
    1510 - EX_ST(STORE_INIT(%g2, %o0 + 0x00))
    1511 - EX_ST(STORE_INIT(%g3, %o0 + 0x08))
    1512 + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1)
    1513 + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1514
    1515 - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
    1516 + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
    1517 MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
    1518
    1519 - EX_ST(STORE_INIT(%o2, %o0 + 0x10))
    1520 - EX_ST(STORE_INIT(%o3, %o0 + 0x18))
    1521 + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1522 + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1523
    1524 - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1525 + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1526 MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
    1527
    1528 - EX_ST(STORE_INIT(%g2, %o0 + 0x20))
    1529 - EX_ST(STORE_INIT(%g3, %o0 + 0x28))
    1530 + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1531 + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1532
    1533 - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
    1534 + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
    1535 add %i1, 64, %i1
    1536 MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
    1537
    1538 - EX_ST(STORE_INIT(%o2, %o0 + 0x30))
    1539 - EX_ST(STORE_INIT(%o3, %o0 + 0x38))
    1540 + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1541 + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1542
    1543 subcc %g1, 64, %g1
    1544 bne,pt %XCC, 8b
    1545 @@ -211,31 +294,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1546 ba,pt %XCC, 60f
    1547 add %i1, %i4, %i1
    1548
    1549 -9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
    1550 +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
    1551 MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
    1552 LOAD(prefetch, %i1 + %i3, #one_read)
    1553
    1554 - EX_ST(STORE_INIT(%g3, %o0 + 0x00))
    1555 - EX_ST(STORE_INIT(%o2, %o0 + 0x08))
    1556 + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1)
    1557 + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1558
    1559 - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
    1560 + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
    1561 MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
    1562
    1563 - EX_ST(STORE_INIT(%o3, %o0 + 0x10))
    1564 - EX_ST(STORE_INIT(%g2, %o0 + 0x18))
    1565 + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1566 + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1567
    1568 - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1569 + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1570 MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
    1571
    1572 - EX_ST(STORE_INIT(%g3, %o0 + 0x20))
    1573 - EX_ST(STORE_INIT(%o2, %o0 + 0x28))
    1574 + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1575 + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1576
    1577 - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
    1578 + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
    1579 add %i1, 64, %i1
    1580 MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
    1581
    1582 - EX_ST(STORE_INIT(%o3, %o0 + 0x30))
    1583 - EX_ST(STORE_INIT(%g2, %o0 + 0x38))
    1584 + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1585 + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1586
    1587 subcc %g1, 64, %g1
    1588 bne,pt %XCC, 9b
    1589 @@ -249,25 +332,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1590 * one twin load ahead, then add 8 back into source when
    1591 * we finish the loop.
    1592 */
    1593 - EX_LD(LOAD_TWIN(%i1, %o4, %o5))
    1594 + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1)
    1595 mov 16, %o7
    1596 mov 32, %g2
    1597 mov 48, %g3
    1598 mov 64, %o1
    1599 -1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1600 +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
    1601 LOAD(prefetch, %i1 + %o1, #one_read)
    1602 - EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line
    1603 - EX_ST(STORE_INIT(%o2, %o0 + 0x08))
    1604 - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
    1605 - EX_ST(STORE_INIT(%o3, %o0 + 0x10))
    1606 - EX_ST(STORE_INIT(%o4, %o0 + 0x18))
    1607 - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
    1608 - EX_ST(STORE_INIT(%o5, %o0 + 0x20))
    1609 - EX_ST(STORE_INIT(%o2, %o0 + 0x28))
    1610 - EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5))
    1611 + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line
    1612 + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1613 + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
    1614 + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1615 + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1616 + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1617 + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1618 + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1619 + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48)
    1620 add %i1, 64, %i1
    1621 - EX_ST(STORE_INIT(%o3, %o0 + 0x30))
    1622 - EX_ST(STORE_INIT(%o4, %o0 + 0x38))
    1623 + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1624 + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1625 subcc %g1, 64, %g1
    1626 bne,pt %XCC, 1b
    1627 add %o0, 64, %o0
    1628 @@ -282,20 +365,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1629 mov 32, %g2
    1630 mov 48, %g3
    1631 mov 64, %o1
    1632 -1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5))
    1633 - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1634 +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1)
    1635 + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
    1636 LOAD(prefetch, %i1 + %o1, #one_read)
    1637 - EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line
    1638 - EX_ST(STORE_INIT(%o5, %o0 + 0x08))
    1639 - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
    1640 - EX_ST(STORE_INIT(%o2, %o0 + 0x10))
    1641 - EX_ST(STORE_INIT(%o3, %o0 + 0x18))
    1642 - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
    1643 + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line
    1644 + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1645 + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
    1646 + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1647 + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1648 + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1649 add %i1, 64, %i1
    1650 - EX_ST(STORE_INIT(%o4, %o0 + 0x20))
    1651 - EX_ST(STORE_INIT(%o5, %o0 + 0x28))
    1652 - EX_ST(STORE_INIT(%o2, %o0 + 0x30))
    1653 - EX_ST(STORE_INIT(%o3, %o0 + 0x38))
    1654 + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1655 + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1656 + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1657 + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1658 subcc %g1, 64, %g1
    1659 bne,pt %XCC, 1b
    1660 add %o0, 64, %o0
    1661 @@ -321,28 +404,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1662 andn %i2, 0xf, %i4
    1663 and %i2, 0xf, %i2
    1664 1: subcc %i4, 0x10, %i4
    1665 - EX_LD(LOAD(ldx, %i1, %o4))
    1666 + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4)
    1667 add %i1, 0x08, %i1
    1668 - EX_LD(LOAD(ldx, %i1, %g1))
    1669 + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4)
    1670 sub %i1, 0x08, %i1
    1671 - EX_ST(STORE(stx, %o4, %i1 + %i3))
    1672 + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4)
    1673 add %i1, 0x8, %i1
    1674 - EX_ST(STORE(stx, %g1, %i1 + %i3))
    1675 + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8)
    1676 bgu,pt %XCC, 1b
    1677 add %i1, 0x8, %i1
    1678 73: andcc %i2, 0x8, %g0
    1679 be,pt %XCC, 1f
    1680 nop
    1681 sub %i2, 0x8, %i2
    1682 - EX_LD(LOAD(ldx, %i1, %o4))
    1683 - EX_ST(STORE(stx, %o4, %i1 + %i3))
    1684 + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8)
    1685 + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8)
    1686 add %i1, 0x8, %i1
    1687 1: andcc %i2, 0x4, %g0
    1688 be,pt %XCC, 1f
    1689 nop
    1690 sub %i2, 0x4, %i2
    1691 - EX_LD(LOAD(lduw, %i1, %i5))
    1692 - EX_ST(STORE(stw, %i5, %i1 + %i3))
    1693 + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4)
    1694 + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4)
    1695 add %i1, 0x4, %i1
    1696 1: cmp %i2, 0
    1697 be,pt %XCC, 85f
    1698 @@ -358,8 +441,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1699 sub %i2, %g1, %i2
    1700
    1701 1: subcc %g1, 1, %g1
    1702 - EX_LD(LOAD(ldub, %i1, %i5))
    1703 - EX_ST(STORE(stb, %i5, %i1 + %i3))
    1704 + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1)
    1705 + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1)
    1706 bgu,pt %icc, 1b
    1707 add %i1, 1, %i1
    1708
    1709 @@ -375,16 +458,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1710
    1711 8: mov 64, %i3
    1712 andn %i1, 0x7, %i1
    1713 - EX_LD(LOAD(ldx, %i1, %g2))
    1714 + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2)
    1715 sub %i3, %g1, %i3
    1716 andn %i2, 0x7, %i4
    1717 sllx %g2, %g1, %g2
    1718 1: add %i1, 0x8, %i1
    1719 - EX_LD(LOAD(ldx, %i1, %g3))
    1720 + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4)
    1721 subcc %i4, 0x8, %i4
    1722 srlx %g3, %i3, %i5
    1723 or %i5, %g2, %i5
    1724 - EX_ST(STORE(stx, %i5, %o0))
    1725 + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4)
    1726 add %o0, 0x8, %o0
    1727 bgu,pt %icc, 1b
    1728 sllx %g3, %g1, %g2
    1729 @@ -404,8 +487,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1730
    1731 1:
    1732 subcc %i2, 4, %i2
    1733 - EX_LD(LOAD(lduw, %i1, %g1))
    1734 - EX_ST(STORE(stw, %g1, %i1 + %i3))
    1735 + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4)
    1736 + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4)
    1737 bgu,pt %XCC, 1b
    1738 add %i1, 4, %i1
    1739
    1740 @@ -415,8 +498,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1741 .align 32
    1742 90:
    1743 subcc %i2, 1, %i2
    1744 - EX_LD(LOAD(ldub, %i1, %g1))
    1745 - EX_ST(STORE(stb, %g1, %i1 + %i3))
    1746 + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1)
    1747 + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1)
    1748 bgu,pt %XCC, 90b
    1749 add %i1, 1, %i1
    1750 ret
    1751 diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
    1752 index ecc5692fa2b4..bb6ff73229e3 100644
    1753 --- a/arch/sparc/lib/U1copy_from_user.S
    1754 +++ b/arch/sparc/lib/U1copy_from_user.S
    1755 @@ -3,19 +3,19 @@
    1756 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    1757 */
    1758
    1759 -#define EX_LD(x) \
    1760 +#define EX_LD(x,y) \
    1761 98: x; \
    1762 .section __ex_table,"a";\
    1763 .align 4; \
    1764 - .word 98b, __retl_one; \
    1765 + .word 98b, y; \
    1766 .text; \
    1767 .align 4;
    1768
    1769 -#define EX_LD_FP(x) \
    1770 +#define EX_LD_FP(x,y) \
    1771 98: x; \
    1772 .section __ex_table,"a";\
    1773 .align 4; \
    1774 - .word 98b, __retl_one_fp;\
    1775 + .word 98b, y; \
    1776 .text; \
    1777 .align 4;
    1778
    1779 diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
    1780 index 9eea392e44d4..ed92ce739558 100644
    1781 --- a/arch/sparc/lib/U1copy_to_user.S
    1782 +++ b/arch/sparc/lib/U1copy_to_user.S
    1783 @@ -3,19 +3,19 @@
    1784 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    1785 */
    1786
    1787 -#define EX_ST(x) \
    1788 +#define EX_ST(x,y) \
    1789 98: x; \
    1790 .section __ex_table,"a";\
    1791 .align 4; \
    1792 - .word 98b, __retl_one; \
    1793 + .word 98b, y; \
    1794 .text; \
    1795 .align 4;
    1796
    1797 -#define EX_ST_FP(x) \
    1798 +#define EX_ST_FP(x,y) \
    1799 98: x; \
    1800 .section __ex_table,"a";\
    1801 .align 4; \
    1802 - .word 98b, __retl_one_fp;\
    1803 + .word 98b, y; \
    1804 .text; \
    1805 .align 4;
    1806
    1807 diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S
    1808 index 3e6209ebb7d7..f30d2ab2c371 100644
    1809 --- a/arch/sparc/lib/U1memcpy.S
    1810 +++ b/arch/sparc/lib/U1memcpy.S
    1811 @@ -5,6 +5,7 @@
    1812 */
    1813
    1814 #ifdef __KERNEL__
    1815 +#include <linux/linkage.h>
    1816 #include <asm/visasm.h>
    1817 #include <asm/asi.h>
    1818 #define GLOBAL_SPARE g7
    1819 @@ -23,21 +24,17 @@
    1820 #endif
    1821
    1822 #ifndef EX_LD
    1823 -#define EX_LD(x) x
    1824 +#define EX_LD(x,y) x
    1825 #endif
    1826 #ifndef EX_LD_FP
    1827 -#define EX_LD_FP(x) x
    1828 +#define EX_LD_FP(x,y) x
    1829 #endif
    1830
    1831 #ifndef EX_ST
    1832 -#define EX_ST(x) x
    1833 +#define EX_ST(x,y) x
    1834 #endif
    1835 #ifndef EX_ST_FP
    1836 -#define EX_ST_FP(x) x
    1837 -#endif
    1838 -
    1839 -#ifndef EX_RETVAL
    1840 -#define EX_RETVAL(x) x
    1841 +#define EX_ST_FP(x,y) x
    1842 #endif
    1843
    1844 #ifndef LOAD
    1845 @@ -78,53 +75,169 @@
    1846 faligndata %f7, %f8, %f60; \
    1847 faligndata %f8, %f9, %f62;
    1848
    1849 -#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
    1850 - EX_LD_FP(LOAD_BLK(%src, %fdest)); \
    1851 - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    1852 - add %src, 0x40, %src; \
    1853 - subcc %len, 0x40, %len; \
    1854 - be,pn %xcc, jmptgt; \
    1855 - add %dest, 0x40, %dest; \
    1856 -
    1857 -#define LOOP_CHUNK1(src, dest, len, branch_dest) \
    1858 - MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
    1859 -#define LOOP_CHUNK2(src, dest, len, branch_dest) \
    1860 - MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
    1861 -#define LOOP_CHUNK3(src, dest, len, branch_dest) \
    1862 - MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
    1863 +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \
    1864 + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \
    1865 + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
    1866 + add %src, 0x40, %src; \
    1867 + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \
    1868 + be,pn %xcc, jmptgt; \
    1869 + add %dest, 0x40, %dest; \
    1870 +
    1871 +#define LOOP_CHUNK1(src, dest, branch_dest) \
    1872 + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest)
    1873 +#define LOOP_CHUNK2(src, dest, branch_dest) \
    1874 + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest)
    1875 +#define LOOP_CHUNK3(src, dest, branch_dest) \
    1876 + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest)
    1877
    1878 #define DO_SYNC membar #Sync;
    1879 #define STORE_SYNC(dest, fsrc) \
    1880 - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    1881 + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
    1882 add %dest, 0x40, %dest; \
    1883 DO_SYNC
    1884
    1885 #define STORE_JUMP(dest, fsrc, target) \
    1886 - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    1887 + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \
    1888 add %dest, 0x40, %dest; \
    1889 ba,pt %xcc, target; \
    1890 nop;
    1891
    1892 -#define FINISH_VISCHUNK(dest, f0, f1, left) \
    1893 - subcc %left, 8, %left;\
    1894 - bl,pn %xcc, 95f; \
    1895 - faligndata %f0, %f1, %f48; \
    1896 - EX_ST_FP(STORE(std, %f48, %dest)); \
    1897 +#define FINISH_VISCHUNK(dest, f0, f1) \
    1898 + subcc %g3, 8, %g3; \
    1899 + bl,pn %xcc, 95f; \
    1900 + faligndata %f0, %f1, %f48; \
    1901 + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \
    1902 add %dest, 8, %dest;
    1903
    1904 -#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
    1905 - subcc %left, 8, %left; \
    1906 - bl,pn %xcc, 95f; \
    1907 +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
    1908 + subcc %g3, 8, %g3; \
    1909 + bl,pn %xcc, 95f; \
    1910 fsrc2 %f0, %f1;
    1911
    1912 -#define UNEVEN_VISCHUNK(dest, f0, f1, left) \
    1913 - UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
    1914 +#define UNEVEN_VISCHUNK(dest, f0, f1) \
    1915 + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
    1916 ba,a,pt %xcc, 93f;
    1917
    1918 .register %g2,#scratch
    1919 .register %g3,#scratch
    1920
    1921 .text
    1922 +#ifndef EX_RETVAL
    1923 +#define EX_RETVAL(x) x
    1924 +ENTRY(U1_g1_1_fp)
    1925 + VISExitHalf
    1926 + add %g1, 1, %g1
    1927 + add %g1, %g2, %g1
    1928 + retl
    1929 + add %g1, %o2, %o0
    1930 +ENDPROC(U1_g1_1_fp)
    1931 +ENTRY(U1_g2_0_fp)
    1932 + VISExitHalf
    1933 + retl
    1934 + add %g2, %o2, %o0
    1935 +ENDPROC(U1_g2_0_fp)
    1936 +ENTRY(U1_g2_8_fp)
    1937 + VISExitHalf
    1938 + add %g2, 8, %g2
    1939 + retl
    1940 + add %g2, %o2, %o0
    1941 +ENDPROC(U1_g2_8_fp)
    1942 +ENTRY(U1_gs_0_fp)
    1943 + VISExitHalf
    1944 + add %GLOBAL_SPARE, %g3, %o0
    1945 + retl
    1946 + add %o0, %o2, %o0
    1947 +ENDPROC(U1_gs_0_fp)
    1948 +ENTRY(U1_gs_80_fp)
    1949 + VISExitHalf
    1950 + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
    1951 + add %GLOBAL_SPARE, %g3, %o0
    1952 + retl
    1953 + add %o0, %o2, %o0
    1954 +ENDPROC(U1_gs_80_fp)
    1955 +ENTRY(U1_gs_40_fp)
    1956 + VISExitHalf
    1957 + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE
    1958 + add %GLOBAL_SPARE, %g3, %o0
    1959 + retl
    1960 + add %o0, %o2, %o0
    1961 +ENDPROC(U1_gs_40_fp)
    1962 +ENTRY(U1_g3_0_fp)
    1963 + VISExitHalf
    1964 + retl
    1965 + add %g3, %o2, %o0
    1966 +ENDPROC(U1_g3_0_fp)
    1967 +ENTRY(U1_g3_8_fp)
    1968 + VISExitHalf
    1969 + add %g3, 8, %g3
    1970 + retl
    1971 + add %g3, %o2, %o0
    1972 +ENDPROC(U1_g3_8_fp)
    1973 +ENTRY(U1_o2_0_fp)
    1974 + VISExitHalf
    1975 + retl
    1976 + mov %o2, %o0
    1977 +ENDPROC(U1_o2_0_fp)
    1978 +ENTRY(U1_o2_1_fp)
    1979 + VISExitHalf
    1980 + retl
    1981 + add %o2, 1, %o0
    1982 +ENDPROC(U1_o2_1_fp)
    1983 +ENTRY(U1_gs_0)
    1984 + VISExitHalf
    1985 + retl
    1986 + add %GLOBAL_SPARE, %o2, %o0
    1987 +ENDPROC(U1_gs_0)
    1988 +ENTRY(U1_gs_8)
    1989 + VISExitHalf
    1990 + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
    1991 + retl
    1992 + add %GLOBAL_SPARE, 0x8, %o0
    1993 +ENDPROC(U1_gs_8)
    1994 +ENTRY(U1_gs_10)
    1995 + VISExitHalf
    1996 + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
    1997 + retl
    1998 + add %GLOBAL_SPARE, 0x10, %o0
    1999 +ENDPROC(U1_gs_10)
    2000 +ENTRY(U1_o2_0)
    2001 + retl
    2002 + mov %o2, %o0
    2003 +ENDPROC(U1_o2_0)
    2004 +ENTRY(U1_o2_8)
    2005 + retl
    2006 + add %o2, 8, %o0
    2007 +ENDPROC(U1_o2_8)
    2008 +ENTRY(U1_o2_4)
    2009 + retl
    2010 + add %o2, 4, %o0
    2011 +ENDPROC(U1_o2_4)
    2012 +ENTRY(U1_o2_1)
    2013 + retl
    2014 + add %o2, 1, %o0
    2015 +ENDPROC(U1_o2_1)
    2016 +ENTRY(U1_g1_0)
    2017 + retl
    2018 + add %g1, %o2, %o0
    2019 +ENDPROC(U1_g1_0)
    2020 +ENTRY(U1_g1_1)
    2021 + add %g1, 1, %g1
    2022 + retl
    2023 + add %g1, %o2, %o0
    2024 +ENDPROC(U1_g1_1)
    2025 +ENTRY(U1_gs_0_o2_adj)
    2026 + and %o2, 7, %o2
    2027 + retl
    2028 + add %GLOBAL_SPARE, %o2, %o0
    2029 +ENDPROC(U1_gs_0_o2_adj)
    2030 +ENTRY(U1_gs_8_o2_adj)
    2031 + and %o2, 7, %o2
    2032 + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE
    2033 + retl
    2034 + add %GLOBAL_SPARE, %o2, %o0
    2035 +ENDPROC(U1_gs_8_o2_adj)
    2036 +#endif
    2037 +
    2038 .align 64
    2039
    2040 .globl FUNC_NAME
    2041 @@ -166,8 +279,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2042 and %g2, 0x38, %g2
    2043
    2044 1: subcc %g1, 0x1, %g1
    2045 - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
    2046 - EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE))
    2047 + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp)
    2048 + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp)
    2049 bgu,pt %XCC, 1b
    2050 add %o1, 0x1, %o1
    2051
    2052 @@ -178,20 +291,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2053 be,pt %icc, 3f
    2054 alignaddr %o1, %g0, %o1
    2055
    2056 - EX_LD_FP(LOAD(ldd, %o1, %f4))
    2057 -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
    2058 + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp)
    2059 +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp)
    2060 add %o1, 0x8, %o1
    2061 subcc %g2, 0x8, %g2
    2062 faligndata %f4, %f6, %f0
    2063 - EX_ST_FP(STORE(std, %f0, %o0))
    2064 + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
    2065 be,pn %icc, 3f
    2066 add %o0, 0x8, %o0
    2067
    2068 - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
    2069 + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp)
    2070 add %o1, 0x8, %o1
    2071 subcc %g2, 0x8, %g2
    2072 faligndata %f6, %f4, %f0
    2073 - EX_ST_FP(STORE(std, %f0, %o0))
    2074 + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
    2075 bne,pt %icc, 1b
    2076 add %o0, 0x8, %o0
    2077
    2078 @@ -214,13 +327,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2079 add %g1, %GLOBAL_SPARE, %g1
    2080 subcc %o2, %g3, %o2
    2081
    2082 - EX_LD_FP(LOAD_BLK(%o1, %f0))
    2083 + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp)
    2084 add %o1, 0x40, %o1
    2085 add %g1, %g3, %g1
    2086 - EX_LD_FP(LOAD_BLK(%o1, %f16))
    2087 + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp)
    2088 add %o1, 0x40, %o1
    2089 sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
    2090 - EX_LD_FP(LOAD_BLK(%o1, %f32))
    2091 + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp)
    2092 add %o1, 0x40, %o1
    2093
    2094 /* There are 8 instances of the unrolled loop,
    2095 @@ -240,11 +353,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2096
    2097 .align 64
    2098 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
    2099 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2100 + LOOP_CHUNK1(o1, o0, 1f)
    2101 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
    2102 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2103 + LOOP_CHUNK2(o1, o0, 2f)
    2104 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
    2105 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2106 + LOOP_CHUNK3(o1, o0, 3f)
    2107 ba,pt %xcc, 1b+4
    2108 faligndata %f0, %f2, %f48
    2109 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
    2110 @@ -261,11 +374,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2111 STORE_JUMP(o0, f48, 56f)
    2112
    2113 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
    2114 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2115 + LOOP_CHUNK1(o1, o0, 1f)
    2116 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
    2117 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2118 + LOOP_CHUNK2(o1, o0, 2f)
    2119 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
    2120 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2121 + LOOP_CHUNK3(o1, o0, 3f)
    2122 ba,pt %xcc, 1b+4
    2123 faligndata %f2, %f4, %f48
    2124 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
    2125 @@ -282,11 +395,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2126 STORE_JUMP(o0, f48, 57f)
    2127
    2128 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
    2129 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2130 + LOOP_CHUNK1(o1, o0, 1f)
    2131 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
    2132 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2133 + LOOP_CHUNK2(o1, o0, 2f)
    2134 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
    2135 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2136 + LOOP_CHUNK3(o1, o0, 3f)
    2137 ba,pt %xcc, 1b+4
    2138 faligndata %f4, %f6, %f48
    2139 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
    2140 @@ -303,11 +416,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2141 STORE_JUMP(o0, f48, 58f)
    2142
    2143 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
    2144 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2145 + LOOP_CHUNK1(o1, o0, 1f)
    2146 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
    2147 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2148 + LOOP_CHUNK2(o1, o0, 2f)
    2149 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
    2150 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2151 + LOOP_CHUNK3(o1, o0, 3f)
    2152 ba,pt %xcc, 1b+4
    2153 faligndata %f6, %f8, %f48
    2154 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
    2155 @@ -324,11 +437,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2156 STORE_JUMP(o0, f48, 59f)
    2157
    2158 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
    2159 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2160 + LOOP_CHUNK1(o1, o0, 1f)
    2161 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
    2162 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2163 + LOOP_CHUNK2(o1, o0, 2f)
    2164 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
    2165 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2166 + LOOP_CHUNK3(o1, o0, 3f)
    2167 ba,pt %xcc, 1b+4
    2168 faligndata %f8, %f10, %f48
    2169 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
    2170 @@ -345,11 +458,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2171 STORE_JUMP(o0, f48, 60f)
    2172
    2173 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
    2174 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2175 + LOOP_CHUNK1(o1, o0, 1f)
    2176 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
    2177 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2178 + LOOP_CHUNK2(o1, o0, 2f)
    2179 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
    2180 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2181 + LOOP_CHUNK3(o1, o0, 3f)
    2182 ba,pt %xcc, 1b+4
    2183 faligndata %f10, %f12, %f48
    2184 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
    2185 @@ -366,11 +479,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2186 STORE_JUMP(o0, f48, 61f)
    2187
    2188 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
    2189 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2190 + LOOP_CHUNK1(o1, o0, 1f)
    2191 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
    2192 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2193 + LOOP_CHUNK2(o1, o0, 2f)
    2194 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
    2195 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2196 + LOOP_CHUNK3(o1, o0, 3f)
    2197 ba,pt %xcc, 1b+4
    2198 faligndata %f12, %f14, %f48
    2199 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
    2200 @@ -387,11 +500,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2201 STORE_JUMP(o0, f48, 62f)
    2202
    2203 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
    2204 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2205 + LOOP_CHUNK1(o1, o0, 1f)
    2206 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2207 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2208 + LOOP_CHUNK2(o1, o0, 2f)
    2209 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
    2210 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2211 + LOOP_CHUNK3(o1, o0, 3f)
    2212 ba,pt %xcc, 1b+4
    2213 faligndata %f14, %f16, %f48
    2214 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2215 @@ -407,53 +520,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2216 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2217 STORE_JUMP(o0, f48, 63f)
    2218
    2219 -40: FINISH_VISCHUNK(o0, f0, f2, g3)
    2220 -41: FINISH_VISCHUNK(o0, f2, f4, g3)
    2221 -42: FINISH_VISCHUNK(o0, f4, f6, g3)
    2222 -43: FINISH_VISCHUNK(o0, f6, f8, g3)
    2223 -44: FINISH_VISCHUNK(o0, f8, f10, g3)
    2224 -45: FINISH_VISCHUNK(o0, f10, f12, g3)
    2225 -46: FINISH_VISCHUNK(o0, f12, f14, g3)
    2226 -47: UNEVEN_VISCHUNK(o0, f14, f0, g3)
    2227 -48: FINISH_VISCHUNK(o0, f16, f18, g3)
    2228 -49: FINISH_VISCHUNK(o0, f18, f20, g3)
    2229 -50: FINISH_VISCHUNK(o0, f20, f22, g3)
    2230 -51: FINISH_VISCHUNK(o0, f22, f24, g3)
    2231 -52: FINISH_VISCHUNK(o0, f24, f26, g3)
    2232 -53: FINISH_VISCHUNK(o0, f26, f28, g3)
    2233 -54: FINISH_VISCHUNK(o0, f28, f30, g3)
    2234 -55: UNEVEN_VISCHUNK(o0, f30, f0, g3)
    2235 -56: FINISH_VISCHUNK(o0, f32, f34, g3)
    2236 -57: FINISH_VISCHUNK(o0, f34, f36, g3)
    2237 -58: FINISH_VISCHUNK(o0, f36, f38, g3)
    2238 -59: FINISH_VISCHUNK(o0, f38, f40, g3)
    2239 -60: FINISH_VISCHUNK(o0, f40, f42, g3)
    2240 -61: FINISH_VISCHUNK(o0, f42, f44, g3)
    2241 -62: FINISH_VISCHUNK(o0, f44, f46, g3)
    2242 -63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
    2243 -
    2244 -93: EX_LD_FP(LOAD(ldd, %o1, %f2))
    2245 +40: FINISH_VISCHUNK(o0, f0, f2)
    2246 +41: FINISH_VISCHUNK(o0, f2, f4)
    2247 +42: FINISH_VISCHUNK(o0, f4, f6)
    2248 +43: FINISH_VISCHUNK(o0, f6, f8)
    2249 +44: FINISH_VISCHUNK(o0, f8, f10)
    2250 +45: FINISH_VISCHUNK(o0, f10, f12)
    2251 +46: FINISH_VISCHUNK(o0, f12, f14)
    2252 +47: UNEVEN_VISCHUNK(o0, f14, f0)
    2253 +48: FINISH_VISCHUNK(o0, f16, f18)
    2254 +49: FINISH_VISCHUNK(o0, f18, f20)
    2255 +50: FINISH_VISCHUNK(o0, f20, f22)
    2256 +51: FINISH_VISCHUNK(o0, f22, f24)
    2257 +52: FINISH_VISCHUNK(o0, f24, f26)
    2258 +53: FINISH_VISCHUNK(o0, f26, f28)
    2259 +54: FINISH_VISCHUNK(o0, f28, f30)
    2260 +55: UNEVEN_VISCHUNK(o0, f30, f0)
    2261 +56: FINISH_VISCHUNK(o0, f32, f34)
    2262 +57: FINISH_VISCHUNK(o0, f34, f36)
    2263 +58: FINISH_VISCHUNK(o0, f36, f38)
    2264 +59: FINISH_VISCHUNK(o0, f38, f40)
    2265 +60: FINISH_VISCHUNK(o0, f40, f42)
    2266 +61: FINISH_VISCHUNK(o0, f42, f44)
    2267 +62: FINISH_VISCHUNK(o0, f44, f46)
    2268 +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0)
    2269 +
    2270 +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp)
    2271 add %o1, 8, %o1
    2272 subcc %g3, 8, %g3
    2273 faligndata %f0, %f2, %f8
    2274 - EX_ST_FP(STORE(std, %f8, %o0))
    2275 + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
    2276 bl,pn %xcc, 95f
    2277 add %o0, 8, %o0
    2278 - EX_LD_FP(LOAD(ldd, %o1, %f0))
    2279 + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp)
    2280 add %o1, 8, %o1
    2281 subcc %g3, 8, %g3
    2282 faligndata %f2, %f0, %f8
    2283 - EX_ST_FP(STORE(std, %f8, %o0))
    2284 + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
    2285 bge,pt %xcc, 93b
    2286 add %o0, 8, %o0
    2287
    2288 95: brz,pt %o2, 2f
    2289 mov %g1, %o1
    2290
    2291 -1: EX_LD_FP(LOAD(ldub, %o1, %o3))
    2292 +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp)
    2293 add %o1, 1, %o1
    2294 subcc %o2, 1, %o2
    2295 - EX_ST_FP(STORE(stb, %o3, %o0))
    2296 + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp)
    2297 bne,pt %xcc, 1b
    2298 add %o0, 1, %o0
    2299
    2300 @@ -469,27 +582,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2301
    2302 72: andn %o2, 0xf, %GLOBAL_SPARE
    2303 and %o2, 0xf, %o2
    2304 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
    2305 - EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
    2306 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0)
    2307 + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0)
    2308 subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
    2309 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2310 + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10)
    2311 add %o1, 0x8, %o1
    2312 - EX_ST(STORE(stx, %g1, %o1 + %o3))
    2313 + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8)
    2314 bgu,pt %XCC, 1b
    2315 add %o1, 0x8, %o1
    2316 73: andcc %o2, 0x8, %g0
    2317 be,pt %XCC, 1f
    2318 nop
    2319 - EX_LD(LOAD(ldx, %o1, %o5))
    2320 + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0)
    2321 sub %o2, 0x8, %o2
    2322 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2323 + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8)
    2324 add %o1, 0x8, %o1
    2325 1: andcc %o2, 0x4, %g0
    2326 be,pt %XCC, 1f
    2327 nop
    2328 - EX_LD(LOAD(lduw, %o1, %o5))
    2329 + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0)
    2330 sub %o2, 0x4, %o2
    2331 - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2332 + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4)
    2333 add %o1, 0x4, %o1
    2334 1: cmp %o2, 0
    2335 be,pt %XCC, 85f
    2336 @@ -503,9 +616,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2337 sub %g0, %g1, %g1
    2338 sub %o2, %g1, %o2
    2339
    2340 -1: EX_LD(LOAD(ldub, %o1, %o5))
    2341 +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0)
    2342 subcc %g1, 1, %g1
    2343 - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2344 + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1)
    2345 bgu,pt %icc, 1b
    2346 add %o1, 1, %o1
    2347
    2348 @@ -521,16 +634,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2349
    2350 8: mov 64, %o3
    2351 andn %o1, 0x7, %o1
    2352 - EX_LD(LOAD(ldx, %o1, %g2))
    2353 + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0)
    2354 sub %o3, %g1, %o3
    2355 andn %o2, 0x7, %GLOBAL_SPARE
    2356 sllx %g2, %g1, %g2
    2357 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
    2358 +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj)
    2359 subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
    2360 add %o1, 0x8, %o1
    2361 srlx %g3, %o3, %o5
    2362 or %o5, %g2, %o5
    2363 - EX_ST(STORE(stx, %o5, %o0))
    2364 + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj)
    2365 add %o0, 0x8, %o0
    2366 bgu,pt %icc, 1b
    2367 sllx %g3, %g1, %g2
    2368 @@ -548,9 +661,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2369 bne,pn %XCC, 90f
    2370 sub %o0, %o1, %o3
    2371
    2372 -1: EX_LD(LOAD(lduw, %o1, %g1))
    2373 +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0)
    2374 subcc %o2, 4, %o2
    2375 - EX_ST(STORE(stw, %g1, %o1 + %o3))
    2376 + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4)
    2377 bgu,pt %XCC, 1b
    2378 add %o1, 4, %o1
    2379
    2380 @@ -558,9 +671,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2381 mov EX_RETVAL(%o4), %o0
    2382
    2383 .align 32
    2384 -90: EX_LD(LOAD(ldub, %o1, %g1))
    2385 +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0)
    2386 subcc %o2, 1, %o2
    2387 - EX_ST(STORE(stb, %g1, %o1 + %o3))
    2388 + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1)
    2389 bgu,pt %XCC, 90b
    2390 add %o1, 1, %o1
    2391 retl
    2392 diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S
    2393 index 88ad73d86fe4..db73010a1af8 100644
    2394 --- a/arch/sparc/lib/U3copy_from_user.S
    2395 +++ b/arch/sparc/lib/U3copy_from_user.S
    2396 @@ -3,19 +3,19 @@
    2397 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    2398 */
    2399
    2400 -#define EX_LD(x) \
    2401 +#define EX_LD(x,y) \
    2402 98: x; \
    2403 .section __ex_table,"a";\
    2404 .align 4; \
    2405 - .word 98b, __retl_one; \
    2406 + .word 98b, y; \
    2407 .text; \
    2408 .align 4;
    2409
    2410 -#define EX_LD_FP(x) \
    2411 +#define EX_LD_FP(x,y) \
    2412 98: x; \
    2413 .section __ex_table,"a";\
    2414 .align 4; \
    2415 - .word 98b, __retl_one_fp;\
    2416 + .word 98b, y##_fp; \
    2417 .text; \
    2418 .align 4;
    2419
    2420 diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
    2421 index 845139d75537..c4ee858e352a 100644
    2422 --- a/arch/sparc/lib/U3copy_to_user.S
    2423 +++ b/arch/sparc/lib/U3copy_to_user.S
    2424 @@ -3,19 +3,19 @@
    2425 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    2426 */
    2427
    2428 -#define EX_ST(x) \
    2429 +#define EX_ST(x,y) \
    2430 98: x; \
    2431 .section __ex_table,"a";\
    2432 .align 4; \
    2433 - .word 98b, __retl_one; \
    2434 + .word 98b, y; \
    2435 .text; \
    2436 .align 4;
    2437
    2438 -#define EX_ST_FP(x) \
    2439 +#define EX_ST_FP(x,y) \
    2440 98: x; \
    2441 .section __ex_table,"a";\
    2442 .align 4; \
    2443 - .word 98b, __retl_one_fp;\
    2444 + .word 98b, y##_fp; \
    2445 .text; \
    2446 .align 4;
    2447
    2448 diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S
    2449 index 491ee69e4995..54f98706b03b 100644
    2450 --- a/arch/sparc/lib/U3memcpy.S
    2451 +++ b/arch/sparc/lib/U3memcpy.S
    2452 @@ -4,6 +4,7 @@
    2453 */
    2454
    2455 #ifdef __KERNEL__
    2456 +#include <linux/linkage.h>
    2457 #include <asm/visasm.h>
    2458 #include <asm/asi.h>
    2459 #define GLOBAL_SPARE %g7
    2460 @@ -22,21 +23,17 @@
    2461 #endif
    2462
    2463 #ifndef EX_LD
    2464 -#define EX_LD(x) x
    2465 +#define EX_LD(x,y) x
    2466 #endif
    2467 #ifndef EX_LD_FP
    2468 -#define EX_LD_FP(x) x
    2469 +#define EX_LD_FP(x,y) x
    2470 #endif
    2471
    2472 #ifndef EX_ST
    2473 -#define EX_ST(x) x
    2474 +#define EX_ST(x,y) x
    2475 #endif
    2476 #ifndef EX_ST_FP
    2477 -#define EX_ST_FP(x) x
    2478 -#endif
    2479 -
    2480 -#ifndef EX_RETVAL
    2481 -#define EX_RETVAL(x) x
    2482 +#define EX_ST_FP(x,y) x
    2483 #endif
    2484
    2485 #ifndef LOAD
    2486 @@ -77,6 +74,87 @@
    2487 */
    2488
    2489 .text
    2490 +#ifndef EX_RETVAL
    2491 +#define EX_RETVAL(x) x
    2492 +__restore_fp:
    2493 + VISExitHalf
    2494 + retl
    2495 + nop
    2496 +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
    2497 + add %g1, 1, %g1
    2498 + add %g2, %g1, %g2
    2499 + ba,pt %xcc, __restore_fp
    2500 + add %o2, %g2, %o0
    2501 +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
    2502 +ENTRY(U3_retl_o2_plus_g2_fp)
    2503 + ba,pt %xcc, __restore_fp
    2504 + add %o2, %g2, %o0
    2505 +ENDPROC(U3_retl_o2_plus_g2_fp)
    2506 +ENTRY(U3_retl_o2_plus_g2_plus_8_fp)
    2507 + add %g2, 8, %g2
    2508 + ba,pt %xcc, __restore_fp
    2509 + add %o2, %g2, %o0
    2510 +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp)
    2511 +ENTRY(U3_retl_o2)
    2512 + retl
    2513 + mov %o2, %o0
    2514 +ENDPROC(U3_retl_o2)
    2515 +ENTRY(U3_retl_o2_plus_1)
    2516 + retl
    2517 + add %o2, 1, %o0
    2518 +ENDPROC(U3_retl_o2_plus_1)
    2519 +ENTRY(U3_retl_o2_plus_4)
    2520 + retl
    2521 + add %o2, 4, %o0
    2522 +ENDPROC(U3_retl_o2_plus_4)
    2523 +ENTRY(U3_retl_o2_plus_8)
    2524 + retl
    2525 + add %o2, 8, %o0
    2526 +ENDPROC(U3_retl_o2_plus_8)
    2527 +ENTRY(U3_retl_o2_plus_g1_plus_1)
    2528 + add %g1, 1, %g1
    2529 + retl
    2530 + add %o2, %g1, %o0
    2531 +ENDPROC(U3_retl_o2_plus_g1_plus_1)
    2532 +ENTRY(U3_retl_o2_fp)
    2533 + ba,pt %xcc, __restore_fp
    2534 + mov %o2, %o0
    2535 +ENDPROC(U3_retl_o2_fp)
    2536 +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
    2537 + sll %o3, 6, %o3
    2538 + add %o3, 0x80, %o3
    2539 + ba,pt %xcc, __restore_fp
    2540 + add %o2, %o3, %o0
    2541 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
    2542 +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
    2543 + sll %o3, 6, %o3
    2544 + add %o3, 0x40, %o3
    2545 + ba,pt %xcc, __restore_fp
    2546 + add %o2, %o3, %o0
    2547 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
    2548 +ENTRY(U3_retl_o2_plus_GS_plus_0x10)
    2549 + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE
    2550 + retl
    2551 + add %o2, GLOBAL_SPARE, %o0
    2552 +ENDPROC(U3_retl_o2_plus_GS_plus_0x10)
    2553 +ENTRY(U3_retl_o2_plus_GS_plus_0x08)
    2554 + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE
    2555 + retl
    2556 + add %o2, GLOBAL_SPARE, %o0
    2557 +ENDPROC(U3_retl_o2_plus_GS_plus_0x08)
    2558 +ENTRY(U3_retl_o2_and_7_plus_GS)
    2559 + and %o2, 7, %o2
    2560 + retl
    2561 + add %o2, GLOBAL_SPARE, %o2
    2562 +ENDPROC(U3_retl_o2_and_7_plus_GS)
    2563 +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8)
    2564 + add GLOBAL_SPARE, 8, GLOBAL_SPARE
    2565 + and %o2, 7, %o2
    2566 + retl
    2567 + add %o2, GLOBAL_SPARE, %o2
    2568 +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8)
    2569 +#endif
    2570 +
    2571 .align 64
    2572
    2573 /* The cheetah's flexible spine, oversized liver, enlarged heart,
    2574 @@ -126,8 +204,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2575 and %g2, 0x38, %g2
    2576
    2577 1: subcc %g1, 0x1, %g1
    2578 - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
    2579 - EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
    2580 + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1)
    2581 + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1)
    2582 bgu,pt %XCC, 1b
    2583 add %o1, 0x1, %o1
    2584
    2585 @@ -138,20 +216,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2586 be,pt %icc, 3f
    2587 alignaddr %o1, %g0, %o1
    2588
    2589 - EX_LD_FP(LOAD(ldd, %o1, %f4))
    2590 -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
    2591 + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2)
    2592 +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2)
    2593 add %o1, 0x8, %o1
    2594 subcc %g2, 0x8, %g2
    2595 faligndata %f4, %f6, %f0
    2596 - EX_ST_FP(STORE(std, %f0, %o0))
    2597 + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8)
    2598 be,pn %icc, 3f
    2599 add %o0, 0x8, %o0
    2600
    2601 - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
    2602 + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2)
    2603 add %o1, 0x8, %o1
    2604 subcc %g2, 0x8, %g2
    2605 faligndata %f6, %f4, %f2
    2606 - EX_ST_FP(STORE(std, %f2, %o0))
    2607 + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8)
    2608 bne,pt %icc, 1b
    2609 add %o0, 0x8, %o0
    2610
    2611 @@ -161,25 +239,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2612 LOAD(prefetch, %o1 + 0x080, #one_read)
    2613 LOAD(prefetch, %o1 + 0x0c0, #one_read)
    2614 LOAD(prefetch, %o1 + 0x100, #one_read)
    2615 - EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0))
    2616 + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2)
    2617 LOAD(prefetch, %o1 + 0x140, #one_read)
    2618 - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2619 + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2)
    2620 LOAD(prefetch, %o1 + 0x180, #one_read)
    2621 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2622 + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2)
    2623 LOAD(prefetch, %o1 + 0x1c0, #one_read)
    2624 faligndata %f0, %f2, %f16
    2625 - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2626 + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2)
    2627 faligndata %f2, %f4, %f18
    2628 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2629 + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2)
    2630 faligndata %f4, %f6, %f20
    2631 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2632 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2)
    2633 faligndata %f6, %f8, %f22
    2634
    2635 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2636 + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2)
    2637 faligndata %f8, %f10, %f24
    2638 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2639 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2)
    2640 faligndata %f10, %f12, %f26
    2641 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2642 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2)
    2643
    2644 subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE
    2645 add %o1, 0x40, %o1
    2646 @@ -190,26 +268,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2647
    2648 .align 64
    2649 1:
    2650 - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2651 + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2652 faligndata %f12, %f14, %f28
    2653 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2654 + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2655 faligndata %f14, %f0, %f30
    2656 - EX_ST_FP(STORE_BLK(%f16, %o0))
    2657 - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2658 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2659 + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2660 faligndata %f0, %f2, %f16
    2661 add %o0, 0x40, %o0
    2662
    2663 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2664 + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2665 faligndata %f2, %f4, %f18
    2666 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2667 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2668 faligndata %f4, %f6, %f20
    2669 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2670 + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2671 subcc %o3, 0x01, %o3
    2672 faligndata %f6, %f8, %f22
    2673 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2674 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2675
    2676 faligndata %f8, %f10, %f24
    2677 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2678 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2679 LOAD(prefetch, %o1 + 0x1c0, #one_read)
    2680 faligndata %f10, %f12, %f26
    2681 bg,pt %XCC, 1b
    2682 @@ -217,29 +295,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2683
    2684 /* Finally we copy the last full 64-byte block. */
    2685 2:
    2686 - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2687 + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2688 faligndata %f12, %f14, %f28
    2689 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2690 + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2691 faligndata %f14, %f0, %f30
    2692 - EX_ST_FP(STORE_BLK(%f16, %o0))
    2693 - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2694 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2695 + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2696 faligndata %f0, %f2, %f16
    2697 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2698 + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2699 faligndata %f2, %f4, %f18
    2700 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2701 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2702 faligndata %f4, %f6, %f20
    2703 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2704 + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2705 faligndata %f6, %f8, %f22
    2706 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2707 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2708 faligndata %f8, %f10, %f24
    2709 cmp %g1, 0
    2710 be,pt %XCC, 1f
    2711 add %o0, 0x40, %o0
    2712 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2713 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2714 1: faligndata %f10, %f12, %f26
    2715 faligndata %f12, %f14, %f28
    2716 faligndata %f14, %f0, %f30
    2717 - EX_ST_FP(STORE_BLK(%f16, %o0))
    2718 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2719 add %o0, 0x40, %o0
    2720 add %o1, 0x40, %o1
    2721 membar #Sync
    2722 @@ -259,20 +337,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2723
    2724 sub %o2, %g2, %o2
    2725 be,a,pt %XCC, 1f
    2726 - EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0))
    2727 + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2)
    2728
    2729 -1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2))
    2730 +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2)
    2731 add %o1, 0x8, %o1
    2732 subcc %g2, 0x8, %g2
    2733 faligndata %f0, %f2, %f8
    2734 - EX_ST_FP(STORE(std, %f8, %o0))
    2735 + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
    2736 be,pn %XCC, 2f
    2737 add %o0, 0x8, %o0
    2738 - EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0))
    2739 + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2)
    2740 add %o1, 0x8, %o1
    2741 subcc %g2, 0x8, %g2
    2742 faligndata %f2, %f0, %f8
    2743 - EX_ST_FP(STORE(std, %f8, %o0))
    2744 + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
    2745 bne,pn %XCC, 1b
    2746 add %o0, 0x8, %o0
    2747
    2748 @@ -292,30 +370,33 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2749 andcc %o2, 0x8, %g0
    2750 be,pt %icc, 1f
    2751 nop
    2752 - EX_LD(LOAD(ldx, %o1, %o5))
    2753 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2754 + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2)
    2755 + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2)
    2756 add %o1, 0x8, %o1
    2757 + sub %o2, 8, %o2
    2758
    2759 1: andcc %o2, 0x4, %g0
    2760 be,pt %icc, 1f
    2761 nop
    2762 - EX_LD(LOAD(lduw, %o1, %o5))
    2763 - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2764 + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2)
    2765 + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2)
    2766 add %o1, 0x4, %o1
    2767 + sub %o2, 4, %o2
    2768
    2769 1: andcc %o2, 0x2, %g0
    2770 be,pt %icc, 1f
    2771 nop
    2772 - EX_LD(LOAD(lduh, %o1, %o5))
    2773 - EX_ST(STORE(sth, %o5, %o1 + %o3))
    2774 + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2)
    2775 + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2)
    2776 add %o1, 0x2, %o1
    2777 + sub %o2, 2, %o2
    2778
    2779 1: andcc %o2, 0x1, %g0
    2780 be,pt %icc, 85f
    2781 nop
    2782 - EX_LD(LOAD(ldub, %o1, %o5))
    2783 + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2)
    2784 ba,pt %xcc, 85f
    2785 - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2786 + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2)
    2787
    2788 .align 64
    2789 70: /* 16 < len <= 64 */
    2790 @@ -326,26 +407,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2791 andn %o2, 0xf, GLOBAL_SPARE
    2792 and %o2, 0xf, %o2
    2793 1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE
    2794 - EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
    2795 - EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
    2796 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2797 + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10)
    2798 + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10)
    2799 + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10)
    2800 add %o1, 0x8, %o1
    2801 - EX_ST(STORE(stx, %g1, %o1 + %o3))
    2802 + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08)
    2803 bgu,pt %XCC, 1b
    2804 add %o1, 0x8, %o1
    2805 73: andcc %o2, 0x8, %g0
    2806 be,pt %XCC, 1f
    2807 nop
    2808 sub %o2, 0x8, %o2
    2809 - EX_LD(LOAD(ldx, %o1, %o5))
    2810 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2811 + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8)
    2812 + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8)
    2813 add %o1, 0x8, %o1
    2814 1: andcc %o2, 0x4, %g0
    2815 be,pt %XCC, 1f
    2816 nop
    2817 sub %o2, 0x4, %o2
    2818 - EX_LD(LOAD(lduw, %o1, %o5))
    2819 - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2820 + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4)
    2821 + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4)
    2822 add %o1, 0x4, %o1
    2823 1: cmp %o2, 0
    2824 be,pt %XCC, 85f
    2825 @@ -361,8 +442,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2826 sub %o2, %g1, %o2
    2827
    2828 1: subcc %g1, 1, %g1
    2829 - EX_LD(LOAD(ldub, %o1, %o5))
    2830 - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2831 + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1)
    2832 + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1)
    2833 bgu,pt %icc, 1b
    2834 add %o1, 1, %o1
    2835
    2836 @@ -378,16 +459,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2837
    2838 8: mov 64, %o3
    2839 andn %o1, 0x7, %o1
    2840 - EX_LD(LOAD(ldx, %o1, %g2))
    2841 + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2)
    2842 sub %o3, %g1, %o3
    2843 andn %o2, 0x7, GLOBAL_SPARE
    2844 sllx %g2, %g1, %g2
    2845 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
    2846 +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS)
    2847 subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE
    2848 add %o1, 0x8, %o1
    2849 srlx %g3, %o3, %o5
    2850 or %o5, %g2, %o5
    2851 - EX_ST(STORE(stx, %o5, %o0))
    2852 + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8)
    2853 add %o0, 0x8, %o0
    2854 bgu,pt %icc, 1b
    2855 sllx %g3, %g1, %g2
    2856 @@ -407,8 +488,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2857
    2858 1:
    2859 subcc %o2, 4, %o2
    2860 - EX_LD(LOAD(lduw, %o1, %g1))
    2861 - EX_ST(STORE(stw, %g1, %o1 + %o3))
    2862 + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4)
    2863 + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4)
    2864 bgu,pt %XCC, 1b
    2865 add %o1, 4, %o1
    2866
    2867 @@ -418,8 +499,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2868 .align 32
    2869 90:
    2870 subcc %o2, 1, %o2
    2871 - EX_LD(LOAD(ldub, %o1, %g1))
    2872 - EX_ST(STORE(stb, %g1, %o1 + %o3))
    2873 + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1)
    2874 + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1)
    2875 bgu,pt %XCC, 90b
    2876 add %o1, 1, %o1
    2877 retl
    2878 diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
    2879 index 302c0e60dc2c..4c89b486fa0d 100644
    2880 --- a/arch/sparc/lib/copy_in_user.S
    2881 +++ b/arch/sparc/lib/copy_in_user.S
    2882 @@ -8,18 +8,33 @@
    2883
    2884 #define XCC xcc
    2885
    2886 -#define EX(x,y) \
    2887 +#define EX(x,y,z) \
    2888 98: x,y; \
    2889 .section __ex_table,"a";\
    2890 .align 4; \
    2891 - .word 98b, __retl_one; \
    2892 + .word 98b, z; \
    2893 .text; \
    2894 .align 4;
    2895
    2896 +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8)
    2897 +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4)
    2898 +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1)
    2899 +
    2900 .register %g2,#scratch
    2901 .register %g3,#scratch
    2902
    2903 .text
    2904 +__retl_o4_plus_8:
    2905 + add %o4, %o2, %o4
    2906 + retl
    2907 + add %o4, 8, %o0
    2908 +__retl_o2_plus_4:
    2909 + retl
    2910 + add %o2, 4, %o0
    2911 +__retl_o2_plus_1:
    2912 + retl
    2913 + add %o2, 1, %o0
    2914 +
    2915 .align 32
    2916
    2917 /* Don't try to get too fancy here, just nice and
    2918 @@ -44,8 +59,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    2919 andn %o2, 0x7, %o4
    2920 and %o2, 0x7, %o2
    2921 1: subcc %o4, 0x8, %o4
    2922 - EX(ldxa [%o1] %asi, %o5)
    2923 - EX(stxa %o5, [%o0] %asi)
    2924 + EX_O4(ldxa [%o1] %asi, %o5)
    2925 + EX_O4(stxa %o5, [%o0] %asi)
    2926 add %o1, 0x8, %o1
    2927 bgu,pt %XCC, 1b
    2928 add %o0, 0x8, %o0
    2929 @@ -53,8 +68,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    2930 be,pt %XCC, 1f
    2931 nop
    2932 sub %o2, 0x4, %o2
    2933 - EX(lduwa [%o1] %asi, %o5)
    2934 - EX(stwa %o5, [%o0] %asi)
    2935 + EX_O2_4(lduwa [%o1] %asi, %o5)
    2936 + EX_O2_4(stwa %o5, [%o0] %asi)
    2937 add %o1, 0x4, %o1
    2938 add %o0, 0x4, %o0
    2939 1: cmp %o2, 0
    2940 @@ -70,8 +85,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    2941
    2942 82:
    2943 subcc %o2, 4, %o2
    2944 - EX(lduwa [%o1] %asi, %g1)
    2945 - EX(stwa %g1, [%o0] %asi)
    2946 + EX_O2_4(lduwa [%o1] %asi, %g1)
    2947 + EX_O2_4(stwa %g1, [%o0] %asi)
    2948 add %o1, 4, %o1
    2949 bgu,pt %XCC, 82b
    2950 add %o0, 4, %o0
    2951 @@ -82,8 +97,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    2952 .align 32
    2953 90:
    2954 subcc %o2, 1, %o2
    2955 - EX(lduba [%o1] %asi, %g1)
    2956 - EX(stba %g1, [%o0] %asi)
    2957 + EX_O2_1(lduba [%o1] %asi, %g1)
    2958 + EX_O2_1(stba %g1, [%o0] %asi)
    2959 add %o1, 1, %o1
    2960 bgu,pt %XCC, 90b
    2961 add %o0, 1, %o0
    2962 diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c
    2963 deleted file mode 100644
    2964 index ac96ae236709..000000000000
    2965 --- a/arch/sparc/lib/user_fixup.c
    2966 +++ /dev/null
    2967 @@ -1,71 +0,0 @@
    2968 -/* user_fixup.c: Fix up user copy faults.
    2969 - *
    2970 - * Copyright (C) 2004 David S. Miller <davem@redhat.com>
    2971 - */
    2972 -
    2973 -#include <linux/compiler.h>
    2974 -#include <linux/kernel.h>
    2975 -#include <linux/string.h>
    2976 -#include <linux/errno.h>
    2977 -#include <linux/module.h>
    2978 -
    2979 -#include <asm/uaccess.h>
    2980 -
    2981 -/* Calculating the exact fault address when using
    2982 - * block loads and stores can be very complicated.
    2983 - *
    2984 - * Instead of trying to be clever and handling all
    2985 - * of the cases, just fix things up simply here.
    2986 - */
    2987 -
    2988 -static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset)
    2989 -{
    2990 - unsigned long fault_addr = current_thread_info()->fault_address;
    2991 - unsigned long end = start + size;
    2992 -
    2993 - if (fault_addr < start || fault_addr >= end) {
    2994 - *offset = 0;
    2995 - } else {
    2996 - *offset = fault_addr - start;
    2997 - size = end - fault_addr;
    2998 - }
    2999 - return size;
    3000 -}
    3001 -
    3002 -unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size)
    3003 -{
    3004 - unsigned long offset;
    3005 -
    3006 - size = compute_size((unsigned long) from, size, &offset);
    3007 - if (likely(size))
    3008 - memset(to + offset, 0, size);
    3009 -
    3010 - return size;
    3011 -}
    3012 -EXPORT_SYMBOL(copy_from_user_fixup);
    3013 -
    3014 -unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size)
    3015 -{
    3016 - unsigned long offset;
    3017 -
    3018 - return compute_size((unsigned long) to, size, &offset);
    3019 -}
    3020 -EXPORT_SYMBOL(copy_to_user_fixup);
    3021 -
    3022 -unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size)
    3023 -{
    3024 - unsigned long fault_addr = current_thread_info()->fault_address;
    3025 - unsigned long start = (unsigned long) to;
    3026 - unsigned long end = start + size;
    3027 -
    3028 - if (fault_addr >= start && fault_addr < end)
    3029 - return end - fault_addr;
    3030 -
    3031 - start = (unsigned long) from;
    3032 - end = start + size;
    3033 - if (fault_addr >= start && fault_addr < end)
    3034 - return end - fault_addr;
    3035 -
    3036 - return size;
    3037 -}
    3038 -EXPORT_SYMBOL(copy_in_user_fixup);
    3039 diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
    3040 index f2b77112e9d8..e20fbbafb0b0 100644
    3041 --- a/arch/sparc/mm/tsb.c
    3042 +++ b/arch/sparc/mm/tsb.c
    3043 @@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr)
    3044 return (tag == (vaddr >> 22));
    3045 }
    3046
    3047 +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end)
    3048 +{
    3049 + unsigned long idx;
    3050 +
    3051 + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) {
    3052 + struct tsb *ent = &swapper_tsb[idx];
    3053 + unsigned long match = idx << 13;
    3054 +
    3055 + match |= (ent->tag << 22);
    3056 + if (match >= start && match < end)
    3057 + ent->tag = (1UL << TSB_TAG_INVALID_BIT);
    3058 + }
    3059 +}
    3060 +
    3061 /* TSB flushes need only occur on the processor initiating the address
    3062 * space modification, not on each cpu the address space has run on.
    3063 * Only the TLB flush needs that treatment.
    3064 @@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
    3065 {
    3066 unsigned long v;
    3067
    3068 + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES)
    3069 + return flush_tsb_kernel_range_scan(start, end);
    3070 +
    3071 for (v = start; v < end; v += PAGE_SIZE) {
    3072 unsigned long hash = tsb_hash(v, PAGE_SHIFT,
    3073 KERNEL_TSB_NENTRIES);
    3074 diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
    3075 index b4f4733abc6e..5d2fd6cd3189 100644
    3076 --- a/arch/sparc/mm/ultra.S
    3077 +++ b/arch/sparc/mm/ultra.S
    3078 @@ -30,7 +30,7 @@
    3079 .text
    3080 .align 32
    3081 .globl __flush_tlb_mm
    3082 -__flush_tlb_mm: /* 18 insns */
    3083 +__flush_tlb_mm: /* 19 insns */
    3084 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
    3085 ldxa [%o1] ASI_DMMU, %g2
    3086 cmp %g2, %o0
    3087 @@ -81,7 +81,7 @@ __flush_tlb_page: /* 22 insns */
    3088
    3089 .align 32
    3090 .globl __flush_tlb_pending
    3091 -__flush_tlb_pending: /* 26 insns */
    3092 +__flush_tlb_pending: /* 27 insns */
    3093 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
    3094 rdpr %pstate, %g7
    3095 sllx %o1, 3, %o1
    3096 @@ -113,12 +113,14 @@ __flush_tlb_pending: /* 26 insns */
    3097
    3098 .align 32
    3099 .globl __flush_tlb_kernel_range
    3100 -__flush_tlb_kernel_range: /* 16 insns */
    3101 +__flush_tlb_kernel_range: /* 31 insns */
    3102 /* %o0=start, %o1=end */
    3103 cmp %o0, %o1
    3104 be,pn %xcc, 2f
    3105 + sub %o1, %o0, %o3
    3106 + srlx %o3, 18, %o4
    3107 + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow
    3108 sethi %hi(PAGE_SIZE), %o4
    3109 - sub %o1, %o0, %o3
    3110 sub %o3, %o4, %o3
    3111 or %o0, 0x20, %o0 ! Nucleus
    3112 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
    3113 @@ -131,6 +133,41 @@ __flush_tlb_kernel_range: /* 16 insns */
    3114 retl
    3115 nop
    3116 nop
    3117 + nop
    3118 + nop
    3119 + nop
    3120 + nop
    3121 + nop
    3122 + nop
    3123 + nop
    3124 + nop
    3125 + nop
    3126 + nop
    3127 + nop
    3128 + nop
    3129 + nop
    3130 +
    3131 +__spitfire_flush_tlb_kernel_range_slow:
    3132 + mov 63 * 8, %o4
    3133 +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3
    3134 + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */
    3135 + bne,pn %xcc, 2f
    3136 + mov TLB_TAG_ACCESS, %o3
    3137 + stxa %g0, [%o3] ASI_IMMU
    3138 + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS
    3139 + membar #Sync
    3140 +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3
    3141 + andcc %o3, 0x40, %g0
    3142 + bne,pn %xcc, 2f
    3143 + mov TLB_TAG_ACCESS, %o3
    3144 + stxa %g0, [%o3] ASI_DMMU
    3145 + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS
    3146 + membar #Sync
    3147 +2: sub %o4, 8, %o4
    3148 + brgez,pt %o4, 1b
    3149 + nop
    3150 + retl
    3151 + nop
    3152
    3153 __spitfire_flush_tlb_mm_slow:
    3154 rdpr %pstate, %g1
    3155 @@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */
    3156 retl
    3157 wrpr %g7, 0x0, %pstate
    3158
    3159 +__cheetah_flush_tlb_kernel_range: /* 31 insns */
    3160 + /* %o0=start, %o1=end */
    3161 + cmp %o0, %o1
    3162 + be,pn %xcc, 2f
    3163 + sub %o1, %o0, %o3
    3164 + srlx %o3, 18, %o4
    3165 + brnz,pn %o4, 3f
    3166 + sethi %hi(PAGE_SIZE), %o4
    3167 + sub %o3, %o4, %o3
    3168 + or %o0, 0x20, %o0 ! Nucleus
    3169 +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
    3170 + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
    3171 + membar #Sync
    3172 + brnz,pt %o3, 1b
    3173 + sub %o3, %o4, %o3
    3174 +2: sethi %hi(KERNBASE), %o3
    3175 + flush %o3
    3176 + retl
    3177 + nop
    3178 +3: mov 0x80, %o4
    3179 + stxa %g0, [%o4] ASI_DMMU_DEMAP
    3180 + membar #Sync
    3181 + stxa %g0, [%o4] ASI_IMMU_DEMAP
    3182 + membar #Sync
    3183 + retl
    3184 + nop
    3185 + nop
    3186 + nop
    3187 + nop
    3188 + nop
    3189 + nop
    3190 + nop
    3191 + nop
    3192 +
    3193 #ifdef DCACHE_ALIASING_POSSIBLE
    3194 __cheetah_flush_dcache_page: /* 11 insns */
    3195 sethi %hi(PAGE_OFFSET), %g1
    3196 @@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error:
    3197 ret
    3198 restore
    3199
    3200 -__hypervisor_flush_tlb_mm: /* 10 insns */
    3201 +__hypervisor_flush_tlb_mm: /* 19 insns */
    3202 mov %o0, %o2 /* ARG2: mmu context */
    3203 mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3204 mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3205 mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3206 mov HV_FAST_MMU_DEMAP_CTX, %o5
    3207 ta HV_FAST_TRAP
    3208 - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3209 + brnz,pn %o0, 1f
    3210 mov HV_FAST_MMU_DEMAP_CTX, %o1
    3211 retl
    3212 nop
    3213 +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5
    3214 + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0
    3215 + nop
    3216 + nop
    3217 + nop
    3218 + nop
    3219 + nop
    3220 + nop
    3221 + nop
    3222
    3223 -__hypervisor_flush_tlb_page: /* 11 insns */
    3224 +__hypervisor_flush_tlb_page: /* 22 insns */
    3225 /* %o0 = context, %o1 = vaddr */
    3226 mov %o0, %g2
    3227 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */
    3228 @@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */
    3229 srlx %o0, PAGE_SHIFT, %o0
    3230 sllx %o0, PAGE_SHIFT, %o0
    3231 ta HV_MMU_UNMAP_ADDR_TRAP
    3232 - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3233 + brnz,pn %o0, 1f
    3234 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3235 retl
    3236 nop
    3237 +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3238 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3239 + nop
    3240 + nop
    3241 + nop
    3242 + nop
    3243 + nop
    3244 + nop
    3245 + nop
    3246 + nop
    3247 + nop
    3248
    3249 -__hypervisor_flush_tlb_pending: /* 16 insns */
    3250 +__hypervisor_flush_tlb_pending: /* 27 insns */
    3251 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
    3252 sllx %o1, 3, %g1
    3253 mov %o2, %g2
    3254 @@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */
    3255 srlx %o0, PAGE_SHIFT, %o0
    3256 sllx %o0, PAGE_SHIFT, %o0
    3257 ta HV_MMU_UNMAP_ADDR_TRAP
    3258 - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3259 + brnz,pn %o0, 1f
    3260 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3261 brnz,pt %g1, 1b
    3262 nop
    3263 retl
    3264 nop
    3265 +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3266 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3267 + nop
    3268 + nop
    3269 + nop
    3270 + nop
    3271 + nop
    3272 + nop
    3273 + nop
    3274 + nop
    3275 + nop
    3276
    3277 -__hypervisor_flush_tlb_kernel_range: /* 16 insns */
    3278 +__hypervisor_flush_tlb_kernel_range: /* 31 insns */
    3279 /* %o0=start, %o1=end */
    3280 cmp %o0, %o1
    3281 be,pn %xcc, 2f
    3282 - sethi %hi(PAGE_SIZE), %g3
    3283 - mov %o0, %g1
    3284 - sub %o1, %g1, %g2
    3285 + sub %o1, %o0, %g2
    3286 + srlx %g2, 18, %g3
    3287 + brnz,pn %g3, 4f
    3288 + mov %o0, %g1
    3289 + sethi %hi(PAGE_SIZE), %g3
    3290 sub %g2, %g3, %g2
    3291 1: add %g1, %g2, %o0 /* ARG0: virtual address */
    3292 mov 0, %o1 /* ARG1: mmu context */
    3293 mov HV_MMU_ALL, %o2 /* ARG2: flags */
    3294 ta HV_MMU_UNMAP_ADDR_TRAP
    3295 - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3296 + brnz,pn %o0, 3f
    3297 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3298 brnz,pt %g2, 1b
    3299 sub %g2, %g3, %g2
    3300 2: retl
    3301 nop
    3302 +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3303 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3304 + nop
    3305 +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3306 + mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3307 + mov 0, %o2 /* ARG2: mmu context == nucleus */
    3308 + mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3309 + mov HV_FAST_MMU_DEMAP_CTX, %o5
    3310 + ta HV_FAST_TRAP
    3311 + brnz,pn %o0, 3b
    3312 + mov HV_FAST_MMU_DEMAP_CTX, %o1
    3313 + retl
    3314 + nop
    3315
    3316 #ifdef DCACHE_ALIASING_POSSIBLE
    3317 /* XXX Niagara and friends have an 8K cache, so no aliasing is
    3318 @@ -394,43 +511,6 @@ tlb_patch_one:
    3319 retl
    3320 nop
    3321
    3322 - .globl cheetah_patch_cachetlbops
    3323 -cheetah_patch_cachetlbops:
    3324 - save %sp, -128, %sp
    3325 -
    3326 - sethi %hi(__flush_tlb_mm), %o0
    3327 - or %o0, %lo(__flush_tlb_mm), %o0
    3328 - sethi %hi(__cheetah_flush_tlb_mm), %o1
    3329 - or %o1, %lo(__cheetah_flush_tlb_mm), %o1
    3330 - call tlb_patch_one
    3331 - mov 19, %o2
    3332 -
    3333 - sethi %hi(__flush_tlb_page), %o0
    3334 - or %o0, %lo(__flush_tlb_page), %o0
    3335 - sethi %hi(__cheetah_flush_tlb_page), %o1
    3336 - or %o1, %lo(__cheetah_flush_tlb_page), %o1
    3337 - call tlb_patch_one
    3338 - mov 22, %o2
    3339 -
    3340 - sethi %hi(__flush_tlb_pending), %o0
    3341 - or %o0, %lo(__flush_tlb_pending), %o0
    3342 - sethi %hi(__cheetah_flush_tlb_pending), %o1
    3343 - or %o1, %lo(__cheetah_flush_tlb_pending), %o1
    3344 - call tlb_patch_one
    3345 - mov 27, %o2
    3346 -
    3347 -#ifdef DCACHE_ALIASING_POSSIBLE
    3348 - sethi %hi(__flush_dcache_page), %o0
    3349 - or %o0, %lo(__flush_dcache_page), %o0
    3350 - sethi %hi(__cheetah_flush_dcache_page), %o1
    3351 - or %o1, %lo(__cheetah_flush_dcache_page), %o1
    3352 - call tlb_patch_one
    3353 - mov 11, %o2
    3354 -#endif /* DCACHE_ALIASING_POSSIBLE */
    3355 -
    3356 - ret
    3357 - restore
    3358 -
    3359 #ifdef CONFIG_SMP
    3360 /* These are all called by the slaves of a cross call, at
    3361 * trap level 1, with interrupts fully disabled.
    3362 @@ -447,7 +527,7 @@ cheetah_patch_cachetlbops:
    3363 */
    3364 .align 32
    3365 .globl xcall_flush_tlb_mm
    3366 -xcall_flush_tlb_mm: /* 21 insns */
    3367 +xcall_flush_tlb_mm: /* 24 insns */
    3368 mov PRIMARY_CONTEXT, %g2
    3369 ldxa [%g2] ASI_DMMU, %g3
    3370 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
    3371 @@ -469,9 +549,12 @@ xcall_flush_tlb_mm: /* 21 insns */
    3372 nop
    3373 nop
    3374 nop
    3375 + nop
    3376 + nop
    3377 + nop
    3378
    3379 .globl xcall_flush_tlb_page
    3380 -xcall_flush_tlb_page: /* 17 insns */
    3381 +xcall_flush_tlb_page: /* 20 insns */
    3382 /* %g5=context, %g1=vaddr */
    3383 mov PRIMARY_CONTEXT, %g4
    3384 ldxa [%g4] ASI_DMMU, %g2
    3385 @@ -490,15 +573,20 @@ xcall_flush_tlb_page: /* 17 insns */
    3386 retry
    3387 nop
    3388 nop
    3389 + nop
    3390 + nop
    3391 + nop
    3392
    3393 .globl xcall_flush_tlb_kernel_range
    3394 -xcall_flush_tlb_kernel_range: /* 25 insns */
    3395 +xcall_flush_tlb_kernel_range: /* 44 insns */
    3396 sethi %hi(PAGE_SIZE - 1), %g2
    3397 or %g2, %lo(PAGE_SIZE - 1), %g2
    3398 andn %g1, %g2, %g1
    3399 andn %g7, %g2, %g7
    3400 sub %g7, %g1, %g3
    3401 - add %g2, 1, %g2
    3402 + srlx %g3, 18, %g2
    3403 + brnz,pn %g2, 2f
    3404 + add %g2, 1, %g2
    3405 sub %g3, %g2, %g3
    3406 or %g1, 0x20, %g1 ! Nucleus
    3407 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
    3408 @@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */
    3409 brnz,pt %g3, 1b
    3410 sub %g3, %g2, %g3
    3411 retry
    3412 - nop
    3413 - nop
    3414 +2: mov 63 * 8, %g1
    3415 +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2
    3416 + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */
    3417 + bne,pn %xcc, 2f
    3418 + mov TLB_TAG_ACCESS, %g2
    3419 + stxa %g0, [%g2] ASI_IMMU
    3420 + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS
    3421 + membar #Sync
    3422 +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2
    3423 + andcc %g2, 0x40, %g0
    3424 + bne,pn %xcc, 2f
    3425 + mov TLB_TAG_ACCESS, %g2
    3426 + stxa %g0, [%g2] ASI_DMMU
    3427 + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS
    3428 + membar #Sync
    3429 +2: sub %g1, 8, %g1
    3430 + brgez,pt %g1, 1b
    3431 + nop
    3432 + retry
    3433 nop
    3434 nop
    3435 nop
    3436 @@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4:
    3437
    3438 retry
    3439
    3440 +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */
    3441 + sethi %hi(PAGE_SIZE - 1), %g2
    3442 + or %g2, %lo(PAGE_SIZE - 1), %g2
    3443 + andn %g1, %g2, %g1
    3444 + andn %g7, %g2, %g7
    3445 + sub %g7, %g1, %g3
    3446 + srlx %g3, 18, %g2
    3447 + brnz,pn %g2, 2f
    3448 + add %g2, 1, %g2
    3449 + sub %g3, %g2, %g3
    3450 + or %g1, 0x20, %g1 ! Nucleus
    3451 +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
    3452 + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
    3453 + membar #Sync
    3454 + brnz,pt %g3, 1b
    3455 + sub %g3, %g2, %g3
    3456 + retry
    3457 +2: mov 0x80, %g2
    3458 + stxa %g0, [%g2] ASI_DMMU_DEMAP
    3459 + membar #Sync
    3460 + stxa %g0, [%g2] ASI_IMMU_DEMAP
    3461 + membar #Sync
    3462 + retry
    3463 + nop
    3464 + nop
    3465 + nop
    3466 + nop
    3467 + nop
    3468 + nop
    3469 + nop
    3470 + nop
    3471 + nop
    3472 + nop
    3473 + nop
    3474 + nop
    3475 + nop
    3476 + nop
    3477 + nop
    3478 + nop
    3479 + nop
    3480 + nop
    3481 + nop
    3482 + nop
    3483 + nop
    3484 + nop
    3485 +
    3486 #ifdef DCACHE_ALIASING_POSSIBLE
    3487 .align 32
    3488 .globl xcall_flush_dcache_page_cheetah
    3489 @@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error:
    3490 ba,a,pt %xcc, rtrap
    3491
    3492 .globl __hypervisor_xcall_flush_tlb_mm
    3493 -__hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3494 +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
    3495 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
    3496 mov %o0, %g2
    3497 mov %o1, %g3
    3498 @@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3499 mov HV_FAST_MMU_DEMAP_CTX, %o5
    3500 ta HV_FAST_TRAP
    3501 mov HV_FAST_MMU_DEMAP_CTX, %g6
    3502 - brnz,pn %o0, __hypervisor_tlb_xcall_error
    3503 + brnz,pn %o0, 1f
    3504 mov %o0, %g5
    3505 mov %g2, %o0
    3506 mov %g3, %o1
    3507 @@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3508 mov %g7, %o5
    3509 membar #Sync
    3510 retry
    3511 +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3512 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3513 + nop
    3514
    3515 .globl __hypervisor_xcall_flush_tlb_page
    3516 -__hypervisor_xcall_flush_tlb_page: /* 17 insns */
    3517 +__hypervisor_xcall_flush_tlb_page: /* 20 insns */
    3518 /* %g5=ctx, %g1=vaddr */
    3519 mov %o0, %g2
    3520 mov %o1, %g3
    3521 @@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */
    3522 sllx %o0, PAGE_SHIFT, %o0
    3523 ta HV_MMU_UNMAP_ADDR_TRAP
    3524 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
    3525 - brnz,a,pn %o0, __hypervisor_tlb_xcall_error
    3526 + brnz,a,pn %o0, 1f
    3527 mov %o0, %g5
    3528 mov %g2, %o0
    3529 mov %g3, %o1
    3530 mov %g4, %o2
    3531 membar #Sync
    3532 retry
    3533 +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3534 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3535 + nop
    3536
    3537 .globl __hypervisor_xcall_flush_tlb_kernel_range
    3538 -__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
    3539 +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
    3540 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
    3541 sethi %hi(PAGE_SIZE - 1), %g2
    3542 or %g2, %lo(PAGE_SIZE - 1), %g2
    3543 andn %g1, %g2, %g1
    3544 andn %g7, %g2, %g7
    3545 sub %g7, %g1, %g3
    3546 + srlx %g3, 18, %g7
    3547 add %g2, 1, %g2
    3548 sub %g3, %g2, %g3
    3549 mov %o0, %g2
    3550 mov %o1, %g4
    3551 - mov %o2, %g7
    3552 + brnz,pn %g7, 2f
    3553 + mov %o2, %g7
    3554 1: add %g1, %g3, %o0 /* ARG0: virtual address */
    3555 mov 0, %o1 /* ARG1: mmu context */
    3556 mov HV_MMU_ALL, %o2 /* ARG2: flags */
    3557 ta HV_MMU_UNMAP_ADDR_TRAP
    3558 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
    3559 - brnz,pn %o0, __hypervisor_tlb_xcall_error
    3560 + brnz,pn %o0, 1f
    3561 mov %o0, %g5
    3562 sethi %hi(PAGE_SIZE), %o2
    3563 brnz,pt %g3, 1b
    3564 sub %g3, %o2, %g3
    3565 - mov %g2, %o0
    3566 +5: mov %g2, %o0
    3567 mov %g4, %o1
    3568 mov %g7, %o2
    3569 membar #Sync
    3570 retry
    3571 +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3572 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3573 + nop
    3574 +2: mov %o3, %g1
    3575 + mov %o5, %g3
    3576 + mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3577 + mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3578 + mov 0, %o2 /* ARG2: mmu context == nucleus */
    3579 + mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3580 + mov HV_FAST_MMU_DEMAP_CTX, %o5
    3581 + ta HV_FAST_TRAP
    3582 + mov %g1, %o3
    3583 + brz,pt %o0, 5b
    3584 + mov %g3, %o5
    3585 + mov HV_FAST_MMU_DEMAP_CTX, %g6
    3586 + ba,pt %xcc, 1b
    3587 + clr %g5
    3588
    3589 /* These just get rescheduled to PIL vectors. */
    3590 .globl xcall_call_function
    3591 @@ -809,6 +985,58 @@ xcall_kgdb_capture:
    3592
    3593 #endif /* CONFIG_SMP */
    3594
    3595 + .globl cheetah_patch_cachetlbops
    3596 +cheetah_patch_cachetlbops:
    3597 + save %sp, -128, %sp
    3598 +
    3599 + sethi %hi(__flush_tlb_mm), %o0
    3600 + or %o0, %lo(__flush_tlb_mm), %o0
    3601 + sethi %hi(__cheetah_flush_tlb_mm), %o1
    3602 + or %o1, %lo(__cheetah_flush_tlb_mm), %o1
    3603 + call tlb_patch_one
    3604 + mov 19, %o2
    3605 +
    3606 + sethi %hi(__flush_tlb_page), %o0
    3607 + or %o0, %lo(__flush_tlb_page), %o0
    3608 + sethi %hi(__cheetah_flush_tlb_page), %o1
    3609 + or %o1, %lo(__cheetah_flush_tlb_page), %o1
    3610 + call tlb_patch_one
    3611 + mov 22, %o2
    3612 +
    3613 + sethi %hi(__flush_tlb_pending), %o0
    3614 + or %o0, %lo(__flush_tlb_pending), %o0
    3615 + sethi %hi(__cheetah_flush_tlb_pending), %o1
    3616 + or %o1, %lo(__cheetah_flush_tlb_pending), %o1
    3617 + call tlb_patch_one
    3618 + mov 27, %o2
    3619 +
    3620 + sethi %hi(__flush_tlb_kernel_range), %o0
    3621 + or %o0, %lo(__flush_tlb_kernel_range), %o0
    3622 + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1
    3623 + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
    3624 + call tlb_patch_one
    3625 + mov 31, %o2
    3626 +
    3627 +#ifdef DCACHE_ALIASING_POSSIBLE
    3628 + sethi %hi(__flush_dcache_page), %o0
    3629 + or %o0, %lo(__flush_dcache_page), %o0
    3630 + sethi %hi(__cheetah_flush_dcache_page), %o1
    3631 + or %o1, %lo(__cheetah_flush_dcache_page), %o1
    3632 + call tlb_patch_one
    3633 + mov 11, %o2
    3634 +#endif /* DCACHE_ALIASING_POSSIBLE */
    3635 +
    3636 +#ifdef CONFIG_SMP
    3637 + sethi %hi(xcall_flush_tlb_kernel_range), %o0
    3638 + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
    3639 + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
    3640 + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
    3641 + call tlb_patch_one
    3642 + mov 44, %o2
    3643 +#endif /* CONFIG_SMP */
    3644 +
    3645 + ret
    3646 + restore
    3647
    3648 .globl hypervisor_patch_cachetlbops
    3649 hypervisor_patch_cachetlbops:
    3650 @@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops:
    3651 sethi %hi(__hypervisor_flush_tlb_mm), %o1
    3652 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1
    3653 call tlb_patch_one
    3654 - mov 10, %o2
    3655 + mov 19, %o2
    3656
    3657 sethi %hi(__flush_tlb_page), %o0
    3658 or %o0, %lo(__flush_tlb_page), %o0
    3659 sethi %hi(__hypervisor_flush_tlb_page), %o1
    3660 or %o1, %lo(__hypervisor_flush_tlb_page), %o1
    3661 call tlb_patch_one
    3662 - mov 11, %o2
    3663 + mov 22, %o2
    3664
    3665 sethi %hi(__flush_tlb_pending), %o0
    3666 or %o0, %lo(__flush_tlb_pending), %o0
    3667 sethi %hi(__hypervisor_flush_tlb_pending), %o1
    3668 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1
    3669 call tlb_patch_one
    3670 - mov 16, %o2
    3671 + mov 27, %o2
    3672
    3673 sethi %hi(__flush_tlb_kernel_range), %o0
    3674 or %o0, %lo(__flush_tlb_kernel_range), %o0
    3675 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
    3676 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
    3677 call tlb_patch_one
    3678 - mov 16, %o2
    3679 + mov 31, %o2
    3680
    3681 #ifdef DCACHE_ALIASING_POSSIBLE
    3682 sethi %hi(__flush_dcache_page), %o0
    3683 @@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops:
    3684 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1
    3685 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
    3686 call tlb_patch_one
    3687 - mov 21, %o2
    3688 + mov 24, %o2
    3689
    3690 sethi %hi(xcall_flush_tlb_page), %o0
    3691 or %o0, %lo(xcall_flush_tlb_page), %o0
    3692 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1
    3693 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
    3694 call tlb_patch_one
    3695 - mov 17, %o2
    3696 + mov 20, %o2
    3697
    3698 sethi %hi(xcall_flush_tlb_kernel_range), %o0
    3699 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
    3700 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
    3701 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
    3702 call tlb_patch_one
    3703 - mov 25, %o2
    3704 + mov 44, %o2
    3705 #endif /* CONFIG_SMP */
    3706
    3707 ret
    3708 diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
    3709 index c4751ece76f6..45e87c9cc828 100644
    3710 --- a/drivers/net/ethernet/broadcom/bgmac.c
    3711 +++ b/drivers/net/ethernet/broadcom/bgmac.c
    3712 @@ -307,6 +307,10 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
    3713 u32 ctl;
    3714
    3715 ctl = bgmac_read(bgmac, ring->mmio_base + BGMAC_DMA_RX_CTL);
    3716 +
    3717 + /* preserve ONLY bits 16-17 from current hardware value */
    3718 + ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
    3719 +
    3720 if (bgmac->feature_flags & BGMAC_FEAT_RX_MASK_SETUP) {
    3721 ctl &= ~BGMAC_DMA_RX_BL_MASK;
    3722 ctl |= BGMAC_DMA_RX_BL_128 << BGMAC_DMA_RX_BL_SHIFT;
    3723 @@ -317,7 +321,6 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
    3724 ctl &= ~BGMAC_DMA_RX_PT_MASK;
    3725 ctl |= BGMAC_DMA_RX_PT_1 << BGMAC_DMA_RX_PT_SHIFT;
    3726 }
    3727 - ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
    3728 ctl |= BGMAC_DMA_RX_ENABLE;
    3729 ctl |= BGMAC_DMA_RX_PARITY_DISABLE;
    3730 ctl |= BGMAC_DMA_RX_OVERFLOW_CONT;
    3731 diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
    3732 index 505ceaf451e2..2c850a92ab15 100644
    3733 --- a/drivers/net/ethernet/broadcom/bnx2.c
    3734 +++ b/drivers/net/ethernet/broadcom/bnx2.c
    3735 @@ -49,6 +49,7 @@
    3736 #include <linux/firmware.h>
    3737 #include <linux/log2.h>
    3738 #include <linux/aer.h>
    3739 +#include <linux/crash_dump.h>
    3740
    3741 #if defined(CONFIG_CNIC) || defined(CONFIG_CNIC_MODULE)
    3742 #define BCM_CNIC 1
    3743 @@ -4759,15 +4760,16 @@ bnx2_setup_msix_tbl(struct bnx2 *bp)
    3744 BNX2_WR(bp, BNX2_PCI_GRC_WINDOW3_ADDR, BNX2_MSIX_PBA_ADDR);
    3745 }
    3746
    3747 -static int
    3748 -bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
    3749 +static void
    3750 +bnx2_wait_dma_complete(struct bnx2 *bp)
    3751 {
    3752 u32 val;
    3753 - int i, rc = 0;
    3754 - u8 old_port;
    3755 + int i;
    3756
    3757 - /* Wait for the current PCI transaction to complete before
    3758 - * issuing a reset. */
    3759 + /*
    3760 + * Wait for the current PCI transaction to complete before
    3761 + * issuing a reset.
    3762 + */
    3763 if ((BNX2_CHIP(bp) == BNX2_CHIP_5706) ||
    3764 (BNX2_CHIP(bp) == BNX2_CHIP_5708)) {
    3765 BNX2_WR(bp, BNX2_MISC_ENABLE_CLR_BITS,
    3766 @@ -4791,6 +4793,21 @@ bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
    3767 }
    3768 }
    3769
    3770 + return;
    3771 +}
    3772 +
    3773 +
    3774 +static int
    3775 +bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
    3776 +{
    3777 + u32 val;
    3778 + int i, rc = 0;
    3779 + u8 old_port;
    3780 +
    3781 + /* Wait for the current PCI transaction to complete before
    3782 + * issuing a reset. */
    3783 + bnx2_wait_dma_complete(bp);
    3784 +
    3785 /* Wait for the firmware to tell us it is ok to issue a reset. */
    3786 bnx2_fw_sync(bp, BNX2_DRV_MSG_DATA_WAIT0 | reset_code, 1, 1);
    3787
    3788 @@ -6356,6 +6373,10 @@ bnx2_open(struct net_device *dev)
    3789 struct bnx2 *bp = netdev_priv(dev);
    3790 int rc;
    3791
    3792 + rc = bnx2_request_firmware(bp);
    3793 + if (rc < 0)
    3794 + goto out;
    3795 +
    3796 netif_carrier_off(dev);
    3797
    3798 bnx2_disable_int(bp);
    3799 @@ -6424,6 +6445,7 @@ open_err:
    3800 bnx2_free_irq(bp);
    3801 bnx2_free_mem(bp);
    3802 bnx2_del_napi(bp);
    3803 + bnx2_release_firmware(bp);
    3804 goto out;
    3805 }
    3806
    3807 @@ -8570,12 +8592,15 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
    3808
    3809 pci_set_drvdata(pdev, dev);
    3810
    3811 - rc = bnx2_request_firmware(bp);
    3812 - if (rc < 0)
    3813 - goto error;
    3814 -
    3815 + /*
    3816 + * In-flight DMA from 1st kernel could continue going in kdump kernel.
    3817 + * New io-page table has been created before bnx2 does reset at open stage.
    3818 + * We have to wait for the in-flight DMA to complete to avoid it look up
    3819 + * into the newly created io-page table.
    3820 + */
    3821 + if (is_kdump_kernel())
    3822 + bnx2_wait_dma_complete(bp);
    3823
    3824 - bnx2_reset_chip(bp, BNX2_DRV_MSG_CODE_RESET);
    3825 memcpy(dev->dev_addr, bp->mac_addr, ETH_ALEN);
    3826
    3827 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG |
    3828 @@ -8608,7 +8633,6 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
    3829 return 0;
    3830
    3831 error:
    3832 - bnx2_release_firmware(bp);
    3833 pci_iounmap(pdev, bp->regview);
    3834 pci_release_regions(pdev);
    3835 pci_disable_device(pdev);
    3836 diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
    3837 index d48873bcbddf..5cdc96bdd444 100644
    3838 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
    3839 +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
    3840 @@ -231,7 +231,7 @@ mlxsw_sp_span_entry_create(struct mlxsw_sp_port *port)
    3841
    3842 span_entry->used = true;
    3843 span_entry->id = index;
    3844 - span_entry->ref_count = 0;
    3845 + span_entry->ref_count = 1;
    3846 span_entry->local_port = local_port;
    3847 return span_entry;
    3848 }
    3849 @@ -268,6 +268,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port)
    3850
    3851 span_entry = mlxsw_sp_span_entry_find(port);
    3852 if (span_entry) {
    3853 + /* Already exists, just take a reference */
    3854 span_entry->ref_count++;
    3855 return span_entry;
    3856 }
    3857 @@ -278,6 +279,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port)
    3858 static int mlxsw_sp_span_entry_put(struct mlxsw_sp *mlxsw_sp,
    3859 struct mlxsw_sp_span_entry *span_entry)
    3860 {
    3861 + WARN_ON(!span_entry->ref_count);
    3862 if (--span_entry->ref_count == 0)
    3863 mlxsw_sp_span_entry_destroy(mlxsw_sp, span_entry);
    3864 return 0;
    3865 diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
    3866 index 3f5c51da6d3e..62514b9bf988 100644
    3867 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
    3868 +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
    3869 @@ -777,6 +777,26 @@ static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
    3870 }
    3871 }
    3872
    3873 +static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
    3874 +{
    3875 + u8 num_rec, last_rec_index, num_entries;
    3876 +
    3877 + num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
    3878 + last_rec_index = num_rec - 1;
    3879 +
    3880 + if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
    3881 + return false;
    3882 + if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
    3883 + MLXSW_REG_RAUHTD_TYPE_IPV6)
    3884 + return true;
    3885 +
    3886 + num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
    3887 + last_rec_index);
    3888 + if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
    3889 + return true;
    3890 + return false;
    3891 +}
    3892 +
    3893 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
    3894 {
    3895 char *rauhtd_pl;
    3896 @@ -803,7 +823,7 @@ static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
    3897 for (i = 0; i < num_rec; i++)
    3898 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
    3899 i);
    3900 - } while (num_rec);
    3901 + } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
    3902 rtnl_unlock();
    3903
    3904 kfree(rauhtd_pl);
    3905 diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
    3906 index 4c8c60af7985..fe9e7b1979b8 100644
    3907 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
    3908 +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
    3909 @@ -871,6 +871,13 @@ static int stmmac_init_phy(struct net_device *dev)
    3910 return -ENODEV;
    3911 }
    3912
    3913 + /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
    3914 + * subsequent PHY polling, make sure we force a link transition if
    3915 + * we have a UP/DOWN/UP transition
    3916 + */
    3917 + if (phydev->is_pseudo_fixed_link)
    3918 + phydev->irq = PHY_POLL;
    3919 +
    3920 pr_debug("stmmac_init_phy: %s: attached to PHY (UID 0x%x)"
    3921 " Link = %d\n", dev->name, phydev->phy_id, phydev->link);
    3922
    3923 diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
    3924 index 5c8429f23a89..3a5530d0511b 100644
    3925 --- a/drivers/usb/gadget/function/f_fs.c
    3926 +++ b/drivers/usb/gadget/function/f_fs.c
    3927 @@ -133,8 +133,60 @@ struct ffs_epfile {
    3928 /*
    3929 * Buffer for holding data from partial reads which may happen since
    3930 * we’re rounding user read requests to a multiple of a max packet size.
    3931 + *
    3932 + * The pointer is initialised with NULL value and may be set by
    3933 + * __ffs_epfile_read_data function to point to a temporary buffer.
    3934 + *
    3935 + * In normal operation, calls to __ffs_epfile_read_buffered will consume
    3936 + * data from said buffer and eventually free it. Importantly, while the
    3937 + * function is using the buffer, it sets the pointer to NULL. This is
    3938 + * all right since __ffs_epfile_read_data and __ffs_epfile_read_buffered
    3939 + * can never run concurrently (they are synchronised by epfile->mutex)
    3940 + * so the latter will not assign a new value to the pointer.
    3941 + *
    3942 + * Meanwhile ffs_func_eps_disable frees the buffer (if the pointer is
    3943 + * valid) and sets the pointer to READ_BUFFER_DROP value. This special
    3944 + * value is crux of the synchronisation between ffs_func_eps_disable and
    3945 + * __ffs_epfile_read_data.
    3946 + *
    3947 + * Once __ffs_epfile_read_data is about to finish it will try to set the
    3948 + * pointer back to its old value (as described above), but seeing as the
    3949 + * pointer is not-NULL (namely READ_BUFFER_DROP) it will instead free
    3950 + * the buffer.
    3951 + *
    3952 + * == State transitions ==
    3953 + *
    3954 + * • ptr == NULL: (initial state)
    3955 + * â—¦ __ffs_epfile_read_buffer_free: go to ptr == DROP
    3956 + * â—¦ __ffs_epfile_read_buffered: nop
    3957 + * â—¦ __ffs_epfile_read_data allocates temp buffer: go to ptr == buf
    3958 + * ◦ reading finishes: n/a, not in ‘and reading’ state
    3959 + * • ptr == DROP:
    3960 + * â—¦ __ffs_epfile_read_buffer_free: nop
    3961 + * â—¦ __ffs_epfile_read_buffered: go to ptr == NULL
    3962 + * â—¦ __ffs_epfile_read_data allocates temp buffer: free buf, nop
    3963 + * ◦ reading finishes: n/a, not in ‘and reading’ state
    3964 + * • ptr == buf:
    3965 + * â—¦ __ffs_epfile_read_buffer_free: free buf, go to ptr == DROP
    3966 + * â—¦ __ffs_epfile_read_buffered: go to ptr == NULL and reading
    3967 + * â—¦ __ffs_epfile_read_data: n/a, __ffs_epfile_read_buffered
    3968 + * is always called first
    3969 + * ◦ reading finishes: n/a, not in ‘and reading’ state
    3970 + * • ptr == NULL and reading:
    3971 + * â—¦ __ffs_epfile_read_buffer_free: go to ptr == DROP and reading
    3972 + * â—¦ __ffs_epfile_read_buffered: n/a, mutex is held
    3973 + * â—¦ __ffs_epfile_read_data: n/a, mutex is held
    3974 + * ◦ reading finishes and …
    3975 + * … all data read: free buf, go to ptr == NULL
    3976 + * … otherwise: go to ptr == buf and reading
    3977 + * • ptr == DROP and reading:
    3978 + * â—¦ __ffs_epfile_read_buffer_free: nop
    3979 + * â—¦ __ffs_epfile_read_buffered: n/a, mutex is held
    3980 + * â—¦ __ffs_epfile_read_data: n/a, mutex is held
    3981 + * â—¦ reading finishes: free buf, go to ptr == DROP
    3982 */
    3983 - struct ffs_buffer *read_buffer; /* P: epfile->mutex */
    3984 + struct ffs_buffer *read_buffer;
    3985 +#define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN))
    3986
    3987 char name[5];
    3988
    3989 @@ -733,25 +785,47 @@ static void ffs_epfile_async_io_complete(struct usb_ep *_ep,
    3990 schedule_work(&io_data->work);
    3991 }
    3992
    3993 +static void __ffs_epfile_read_buffer_free(struct ffs_epfile *epfile)
    3994 +{
    3995 + /*
    3996 + * See comment in struct ffs_epfile for full read_buffer pointer
    3997 + * synchronisation story.
    3998 + */
    3999 + struct ffs_buffer *buf = xchg(&epfile->read_buffer, READ_BUFFER_DROP);
    4000 + if (buf && buf != READ_BUFFER_DROP)
    4001 + kfree(buf);
    4002 +}
    4003 +
    4004 /* Assumes epfile->mutex is held. */
    4005 static ssize_t __ffs_epfile_read_buffered(struct ffs_epfile *epfile,
    4006 struct iov_iter *iter)
    4007 {
    4008 - struct ffs_buffer *buf = epfile->read_buffer;
    4009 + /*
    4010 + * Null out epfile->read_buffer so ffs_func_eps_disable does not free
    4011 + * the buffer while we are using it. See comment in struct ffs_epfile
    4012 + * for full read_buffer pointer synchronisation story.
    4013 + */
    4014 + struct ffs_buffer *buf = xchg(&epfile->read_buffer, NULL);
    4015 ssize_t ret;
    4016 - if (!buf)
    4017 + if (!buf || buf == READ_BUFFER_DROP)
    4018 return 0;
    4019
    4020 ret = copy_to_iter(buf->data, buf->length, iter);
    4021 if (buf->length == ret) {
    4022 kfree(buf);
    4023 - epfile->read_buffer = NULL;
    4024 - } else if (unlikely(iov_iter_count(iter))) {
    4025 + return ret;
    4026 + }
    4027 +
    4028 + if (unlikely(iov_iter_count(iter))) {
    4029 ret = -EFAULT;
    4030 } else {
    4031 buf->length -= ret;
    4032 buf->data += ret;
    4033 }
    4034 +
    4035 + if (cmpxchg(&epfile->read_buffer, NULL, buf))
    4036 + kfree(buf);
    4037 +
    4038 return ret;
    4039 }
    4040
    4041 @@ -780,7 +854,15 @@ static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile,
    4042 buf->length = data_len;
    4043 buf->data = buf->storage;
    4044 memcpy(buf->storage, data + ret, data_len);
    4045 - epfile->read_buffer = buf;
    4046 +
    4047 + /*
    4048 + * At this point read_buffer is NULL or READ_BUFFER_DROP (if
    4049 + * ffs_func_eps_disable has been called in the meanwhile). See comment
    4050 + * in struct ffs_epfile for full read_buffer pointer synchronisation
    4051 + * story.
    4052 + */
    4053 + if (unlikely(cmpxchg(&epfile->read_buffer, NULL, buf)))
    4054 + kfree(buf);
    4055
    4056 return ret;
    4057 }
    4058 @@ -1094,8 +1176,7 @@ ffs_epfile_release(struct inode *inode, struct file *file)
    4059
    4060 ENTER();
    4061
    4062 - kfree(epfile->read_buffer);
    4063 - epfile->read_buffer = NULL;
    4064 + __ffs_epfile_read_buffer_free(epfile);
    4065 ffs_data_closed(epfile->ffs);
    4066
    4067 return 0;
    4068 @@ -1721,24 +1802,20 @@ static void ffs_func_eps_disable(struct ffs_function *func)
    4069 unsigned count = func->ffs->eps_count;
    4070 unsigned long flags;
    4071
    4072 + spin_lock_irqsave(&func->ffs->eps_lock, flags);
    4073 do {
    4074 - if (epfile)
    4075 - mutex_lock(&epfile->mutex);
    4076 - spin_lock_irqsave(&func->ffs->eps_lock, flags);
    4077 /* pending requests get nuked */
    4078 if (likely(ep->ep))
    4079 usb_ep_disable(ep->ep);
    4080 ++ep;
    4081 - spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
    4082
    4083 if (epfile) {
    4084 epfile->ep = NULL;
    4085 - kfree(epfile->read_buffer);
    4086 - epfile->read_buffer = NULL;
    4087 - mutex_unlock(&epfile->mutex);
    4088 + __ffs_epfile_read_buffer_free(epfile);
    4089 ++epfile;
    4090 }
    4091 } while (--count);
    4092 + spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
    4093 }
    4094
    4095 static int ffs_func_eps_enable(struct ffs_function *func)
    4096 diff --git a/include/net/ip.h b/include/net/ip.h
    4097 index 156b0c11b524..0ccf6daf6f56 100644
    4098 --- a/include/net/ip.h
    4099 +++ b/include/net/ip.h
    4100 @@ -47,7 +47,6 @@ struct inet_skb_parm {
    4101 #define IPSKB_REROUTED BIT(4)
    4102 #define IPSKB_DOREDIRECT BIT(5)
    4103 #define IPSKB_FRAG_PMTU BIT(6)
    4104 -#define IPSKB_FRAG_SEGS BIT(7)
    4105
    4106 u16 frag_max_size;
    4107 };
    4108 diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
    4109 index 43a5a0e4524c..b01d5d1d7439 100644
    4110 --- a/include/net/ip6_tunnel.h
    4111 +++ b/include/net/ip6_tunnel.h
    4112 @@ -145,6 +145,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb,
    4113 {
    4114 int pkt_len, err;
    4115
    4116 + memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
    4117 pkt_len = skb->len - skb_inner_network_offset(skb);
    4118 err = ip6_local_out(dev_net(skb_dst(skb)->dev), sk, skb);
    4119 if (unlikely(net_xmit_eval(err)))
    4120 diff --git a/include/net/sock.h b/include/net/sock.h
    4121 index 8741988e6880..c26eab962ec7 100644
    4122 --- a/include/net/sock.h
    4123 +++ b/include/net/sock.h
    4124 @@ -1587,11 +1587,11 @@ static inline void sock_put(struct sock *sk)
    4125 void sock_gen_put(struct sock *sk);
    4126
    4127 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
    4128 - unsigned int trim_cap);
    4129 + unsigned int trim_cap, bool refcounted);
    4130 static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
    4131 const int nested)
    4132 {
    4133 - return __sk_receive_skb(sk, skb, nested, 1);
    4134 + return __sk_receive_skb(sk, skb, nested, 1, true);
    4135 }
    4136
    4137 static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
    4138 diff --git a/include/net/tcp.h b/include/net/tcp.h
    4139 index 7717302cab91..0de698940793 100644
    4140 --- a/include/net/tcp.h
    4141 +++ b/include/net/tcp.h
    4142 @@ -1164,6 +1164,7 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp)
    4143 }
    4144
    4145 bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
    4146 +int tcp_filter(struct sock *sk, struct sk_buff *skb);
    4147
    4148 #undef STATE_TRACE
    4149
    4150 diff --git a/include/uapi/linux/atm_zatm.h b/include/uapi/linux/atm_zatm.h
    4151 index 5cd4d4d2dd1d..9c9c6ad55f14 100644
    4152 --- a/include/uapi/linux/atm_zatm.h
    4153 +++ b/include/uapi/linux/atm_zatm.h
    4154 @@ -14,7 +14,6 @@
    4155
    4156 #include <linux/atmapi.h>
    4157 #include <linux/atmioc.h>
    4158 -#include <linux/time.h>
    4159
    4160 #define ZATM_GETPOOL _IOW('a',ATMIOC_SARPRV+1,struct atmif_sioc)
    4161 /* get pool statistics */
    4162 diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
    4163 index 570eeca7bdfa..ad1bc67aff1b 100644
    4164 --- a/kernel/bpf/hashtab.c
    4165 +++ b/kernel/bpf/hashtab.c
    4166 @@ -687,7 +687,8 @@ static void delete_all_elements(struct bpf_htab *htab)
    4167
    4168 hlist_for_each_entry_safe(l, n, head, hash_node) {
    4169 hlist_del_rcu(&l->hash_node);
    4170 - htab_elem_free(htab, l);
    4171 + if (l->state != HTAB_EXTRA_ELEM_USED)
    4172 + htab_elem_free(htab, l);
    4173 }
    4174 }
    4175 }
    4176 diff --git a/net/core/dev.c b/net/core/dev.c
    4177 index 44b3ba462ba1..9ce9d7284ea7 100644
    4178 --- a/net/core/dev.c
    4179 +++ b/net/core/dev.c
    4180 @@ -2484,7 +2484,7 @@ int skb_checksum_help(struct sk_buff *skb)
    4181 goto out;
    4182 }
    4183
    4184 - *(__sum16 *)(skb->data + offset) = csum_fold(csum);
    4185 + *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
    4186 out_set_summed:
    4187 skb->ip_summed = CHECKSUM_NONE;
    4188 out:
    4189 diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
    4190 index 52742a02814f..5550a86f7264 100644
    4191 --- a/net/core/flow_dissector.c
    4192 +++ b/net/core/flow_dissector.c
    4193 @@ -118,7 +118,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
    4194 struct flow_dissector_key_tags *key_tags;
    4195 struct flow_dissector_key_keyid *key_keyid;
    4196 u8 ip_proto = 0;
    4197 - bool ret = false;
    4198 + bool ret;
    4199
    4200 if (!data) {
    4201 data = skb->data;
    4202 @@ -481,12 +481,17 @@ ip_proto_again:
    4203 out_good:
    4204 ret = true;
    4205
    4206 -out_bad:
    4207 + key_control->thoff = (u16)nhoff;
    4208 +out:
    4209 key_basic->n_proto = proto;
    4210 key_basic->ip_proto = ip_proto;
    4211 - key_control->thoff = (u16)nhoff;
    4212
    4213 return ret;
    4214 +
    4215 +out_bad:
    4216 + ret = false;
    4217 + key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
    4218 + goto out;
    4219 }
    4220 EXPORT_SYMBOL(__skb_flow_dissect);
    4221
    4222 diff --git a/net/core/sock.c b/net/core/sock.c
    4223 index fd7b41edf1ce..10acaccca5c8 100644
    4224 --- a/net/core/sock.c
    4225 +++ b/net/core/sock.c
    4226 @@ -453,7 +453,7 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
    4227 EXPORT_SYMBOL(sock_queue_rcv_skb);
    4228
    4229 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
    4230 - const int nested, unsigned int trim_cap)
    4231 + const int nested, unsigned int trim_cap, bool refcounted)
    4232 {
    4233 int rc = NET_RX_SUCCESS;
    4234
    4235 @@ -487,7 +487,8 @@ int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
    4236
    4237 bh_unlock_sock(sk);
    4238 out:
    4239 - sock_put(sk);
    4240 + if (refcounted)
    4241 + sock_put(sk);
    4242 return rc;
    4243 discard_and_relse:
    4244 kfree_skb(skb);
    4245 @@ -1563,6 +1564,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
    4246 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
    4247
    4248 newsk->sk_err = 0;
    4249 + newsk->sk_err_soft = 0;
    4250 newsk->sk_priority = 0;
    4251 newsk->sk_incoming_cpu = raw_smp_processor_id();
    4252 atomic64_set(&newsk->sk_cookie, 0);
    4253 diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
    4254 index 345a3aeb8c7e..b567c8725aea 100644
    4255 --- a/net/dccp/ipv4.c
    4256 +++ b/net/dccp/ipv4.c
    4257 @@ -235,7 +235,7 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
    4258 {
    4259 const struct iphdr *iph = (struct iphdr *)skb->data;
    4260 const u8 offset = iph->ihl << 2;
    4261 - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
    4262 + const struct dccp_hdr *dh;
    4263 struct dccp_sock *dp;
    4264 struct inet_sock *inet;
    4265 const int type = icmp_hdr(skb)->type;
    4266 @@ -245,11 +245,13 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
    4267 int err;
    4268 struct net *net = dev_net(skb->dev);
    4269
    4270 - if (skb->len < offset + sizeof(*dh) ||
    4271 - skb->len < offset + __dccp_basic_hdr_len(dh)) {
    4272 - __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
    4273 - return;
    4274 - }
    4275 + /* Only need dccph_dport & dccph_sport which are the first
    4276 + * 4 bytes in dccp header.
    4277 + * Our caller (icmp_socket_deliver()) already pulled 8 bytes for us.
    4278 + */
    4279 + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
    4280 + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
    4281 + dh = (struct dccp_hdr *)(skb->data + offset);
    4282
    4283 sk = __inet_lookup_established(net, &dccp_hashinfo,
    4284 iph->daddr, dh->dccph_dport,
    4285 @@ -868,7 +870,7 @@ lookup:
    4286 goto discard_and_relse;
    4287 nf_reset(skb);
    4288
    4289 - return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4);
    4290 + return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4, refcounted);
    4291
    4292 no_dccp_socket:
    4293 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
    4294 diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
    4295 index 3828f94b234c..715e5d1dc107 100644
    4296 --- a/net/dccp/ipv6.c
    4297 +++ b/net/dccp/ipv6.c
    4298 @@ -70,7 +70,7 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
    4299 u8 type, u8 code, int offset, __be32 info)
    4300 {
    4301 const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
    4302 - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
    4303 + const struct dccp_hdr *dh;
    4304 struct dccp_sock *dp;
    4305 struct ipv6_pinfo *np;
    4306 struct sock *sk;
    4307 @@ -78,12 +78,13 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
    4308 __u64 seq;
    4309 struct net *net = dev_net(skb->dev);
    4310
    4311 - if (skb->len < offset + sizeof(*dh) ||
    4312 - skb->len < offset + __dccp_basic_hdr_len(dh)) {
    4313 - __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
    4314 - ICMP6_MIB_INERRORS);
    4315 - return;
    4316 - }
    4317 + /* Only need dccph_dport & dccph_sport which are the first
    4318 + * 4 bytes in dccp header.
    4319 + * Our caller (icmpv6_notify()) already pulled 8 bytes for us.
    4320 + */
    4321 + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
    4322 + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
    4323 + dh = (struct dccp_hdr *)(skb->data + offset);
    4324
    4325 sk = __inet6_lookup_established(net, &dccp_hashinfo,
    4326 &hdr->daddr, dh->dccph_dport,
    4327 @@ -738,7 +739,8 @@ lookup:
    4328 if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
    4329 goto discard_and_relse;
    4330
    4331 - return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4) ? -1 : 0;
    4332 + return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4,
    4333 + refcounted) ? -1 : 0;
    4334
    4335 no_dccp_socket:
    4336 if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
    4337 @@ -956,6 +958,7 @@ static const struct inet_connection_sock_af_ops dccp_ipv6_mapped = {
    4338 .getsockopt = ipv6_getsockopt,
    4339 .addr2sockaddr = inet6_csk_addr2sockaddr,
    4340 .sockaddr_len = sizeof(struct sockaddr_in6),
    4341 + .bind_conflict = inet6_csk_bind_conflict,
    4342 #ifdef CONFIG_COMPAT
    4343 .compat_setsockopt = compat_ipv6_setsockopt,
    4344 .compat_getsockopt = compat_ipv6_getsockopt,
    4345 diff --git a/net/dccp/proto.c b/net/dccp/proto.c
    4346 index 41e65804ddf5..9fe25bf63296 100644
    4347 --- a/net/dccp/proto.c
    4348 +++ b/net/dccp/proto.c
    4349 @@ -1009,6 +1009,10 @@ void dccp_close(struct sock *sk, long timeout)
    4350 __kfree_skb(skb);
    4351 }
    4352
    4353 + /* If socket has been already reset kill it. */
    4354 + if (sk->sk_state == DCCP_CLOSED)
    4355 + goto adjudge_to_death;
    4356 +
    4357 if (data_was_unread) {
    4358 /* Unread data was tossed, send an appropriate Reset Code */
    4359 DCCP_WARN("ABORT with %u bytes unread\n", data_was_unread);
    4360 diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
    4361 index e2ffc2a5c7db..7ef703102dca 100644
    4362 --- a/net/ipv4/fib_trie.c
    4363 +++ b/net/ipv4/fib_trie.c
    4364 @@ -2455,22 +2455,19 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
    4365 struct key_vector *l, **tp = &iter->tnode;
    4366 t_key key;
    4367
    4368 - /* use cache location of next-to-find key */
    4369 + /* use cached location of previously found key */
    4370 if (iter->pos > 0 && pos >= iter->pos) {
    4371 - pos -= iter->pos;
    4372 key = iter->key;
    4373 } else {
    4374 - iter->pos = 0;
    4375 + iter->pos = 1;
    4376 key = 0;
    4377 }
    4378
    4379 - while ((l = leaf_walk_rcu(tp, key)) != NULL) {
    4380 + pos -= iter->pos;
    4381 +
    4382 + while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) {
    4383 key = l->key + 1;
    4384 iter->pos++;
    4385 -
    4386 - if (--pos <= 0)
    4387 - break;
    4388 -
    4389 l = NULL;
    4390
    4391 /* handle unlikely case of a key wrap */
    4392 @@ -2479,7 +2476,7 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
    4393 }
    4394
    4395 if (l)
    4396 - iter->key = key; /* remember it */
    4397 + iter->key = l->key; /* remember it */
    4398 else
    4399 iter->pos = 0; /* forget it */
    4400
    4401 @@ -2507,7 +2504,7 @@ static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos)
    4402 return fib_route_get_idx(iter, *pos);
    4403
    4404 iter->pos = 0;
    4405 - iter->key = 0;
    4406 + iter->key = KEY_MAX;
    4407
    4408 return SEQ_START_TOKEN;
    4409 }
    4410 @@ -2516,7 +2513,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
    4411 {
    4412 struct fib_route_iter *iter = seq->private;
    4413 struct key_vector *l = NULL;
    4414 - t_key key = iter->key;
    4415 + t_key key = iter->key + 1;
    4416
    4417 ++*pos;
    4418
    4419 @@ -2525,7 +2522,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
    4420 l = leaf_walk_rcu(&iter->tnode, key);
    4421
    4422 if (l) {
    4423 - iter->key = l->key + 1;
    4424 + iter->key = l->key;
    4425 iter->pos++;
    4426 } else {
    4427 iter->pos = 0;
    4428 diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c
    4429 index 38abe70e595f..48734ee6293f 100644
    4430 --- a/net/ipv4/icmp.c
    4431 +++ b/net/ipv4/icmp.c
    4432 @@ -477,7 +477,7 @@ static struct rtable *icmp_route_lookup(struct net *net,
    4433 fl4->flowi4_proto = IPPROTO_ICMP;
    4434 fl4->fl4_icmp_type = type;
    4435 fl4->fl4_icmp_code = code;
    4436 - fl4->flowi4_oif = l3mdev_master_ifindex(skb_in->dev);
    4437 + fl4->flowi4_oif = l3mdev_master_ifindex(skb_dst(skb_in)->dev);
    4438
    4439 security_skb_classify_flow(skb_in, flowi4_to_flowi(fl4));
    4440 rt = __ip_route_output_key_hash(net, fl4,
    4441 @@ -502,7 +502,7 @@ static struct rtable *icmp_route_lookup(struct net *net,
    4442 if (err)
    4443 goto relookup_failed;
    4444
    4445 - if (inet_addr_type_dev_table(net, skb_in->dev,
    4446 + if (inet_addr_type_dev_table(net, skb_dst(skb_in)->dev,
    4447 fl4_dec.saddr) == RTN_LOCAL) {
    4448 rt2 = __ip_route_output_key(net, &fl4_dec);
    4449 if (IS_ERR(rt2))
    4450 diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c
    4451 index 8b4ffd216839..9f0a7b96646f 100644
    4452 --- a/net/ipv4/ip_forward.c
    4453 +++ b/net/ipv4/ip_forward.c
    4454 @@ -117,7 +117,7 @@ int ip_forward(struct sk_buff *skb)
    4455 if (opt->is_strictroute && rt->rt_uses_gateway)
    4456 goto sr_failed;
    4457
    4458 - IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS;
    4459 + IPCB(skb)->flags |= IPSKB_FORWARDED;
    4460 mtu = ip_dst_mtu_maybe_forward(&rt->dst, true);
    4461 if (ip_exceeds_mtu(skb, mtu)) {
    4462 IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
    4463 diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
    4464 index dde37fb340bf..307daed9a4b9 100644
    4465 --- a/net/ipv4/ip_output.c
    4466 +++ b/net/ipv4/ip_output.c
    4467 @@ -223,11 +223,9 @@ static int ip_finish_output_gso(struct net *net, struct sock *sk,
    4468 struct sk_buff *segs;
    4469 int ret = 0;
    4470
    4471 - /* common case: fragmentation of segments is not allowed,
    4472 - * or seglen is <= mtu
    4473 + /* common case: seglen is <= mtu
    4474 */
    4475 - if (((IPCB(skb)->flags & IPSKB_FRAG_SEGS) == 0) ||
    4476 - skb_gso_validate_mtu(skb, mtu))
    4477 + if (skb_gso_validate_mtu(skb, mtu))
    4478 return ip_finish_output2(net, sk, skb);
    4479
    4480 /* Slowpath - GSO segment length is exceeding the dst MTU.
    4481 diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c
    4482 index 0f227db0e9ac..afd6b5968caf 100644
    4483 --- a/net/ipv4/ip_tunnel_core.c
    4484 +++ b/net/ipv4/ip_tunnel_core.c
    4485 @@ -63,7 +63,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
    4486 int pkt_len = skb->len - skb_inner_network_offset(skb);
    4487 struct net *net = dev_net(rt->dst.dev);
    4488 struct net_device *dev = skb->dev;
    4489 - int skb_iif = skb->skb_iif;
    4490 struct iphdr *iph;
    4491 int err;
    4492
    4493 @@ -73,16 +72,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
    4494 skb_dst_set(skb, &rt->dst);
    4495 memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
    4496
    4497 - if (skb_iif && !(df & htons(IP_DF))) {
    4498 - /* Arrived from an ingress interface, got encapsulated, with
    4499 - * fragmentation of encapulating frames allowed.
    4500 - * If skb is gso, the resulting encapsulated network segments
    4501 - * may exceed dst mtu.
    4502 - * Allow IP Fragmentation of segments.
    4503 - */
    4504 - IPCB(skb)->flags |= IPSKB_FRAG_SEGS;
    4505 - }
    4506 -
    4507 /* Push down and install the IP header. */
    4508 skb_push(skb, sizeof(struct iphdr));
    4509 skb_reset_network_header(skb);
    4510 diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
    4511 index 5f006e13de56..27089f5ebbb1 100644
    4512 --- a/net/ipv4/ipmr.c
    4513 +++ b/net/ipv4/ipmr.c
    4514 @@ -1749,7 +1749,7 @@ static void ipmr_queue_xmit(struct net *net, struct mr_table *mrt,
    4515 vif->dev->stats.tx_bytes += skb->len;
    4516 }
    4517
    4518 - IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS;
    4519 + IPCB(skb)->flags |= IPSKB_FORWARDED;
    4520
    4521 /* RFC1584 teaches, that DVMRP/PIM router must deliver packets locally
    4522 * not only before forwarding, but after forwarding on all output
    4523 diff --git a/net/ipv4/route.c b/net/ipv4/route.c
    4524 index 62c3ed0b7556..2f23ef1a8486 100644
    4525 --- a/net/ipv4/route.c
    4526 +++ b/net/ipv4/route.c
    4527 @@ -753,7 +753,9 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
    4528 goto reject_redirect;
    4529 }
    4530
    4531 - n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
    4532 + n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
    4533 + if (!n)
    4534 + n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
    4535 if (!IS_ERR(n)) {
    4536 if (!(n->nud_state & NUD_VALID)) {
    4537 neigh_event_send(n, NULL);
    4538 diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
    4539 index ffbb218de520..c876f5ddc86c 100644
    4540 --- a/net/ipv4/tcp.c
    4541 +++ b/net/ipv4/tcp.c
    4542 @@ -1145,7 +1145,7 @@ restart:
    4543
    4544 err = -EPIPE;
    4545 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
    4546 - goto out_err;
    4547 + goto do_error;
    4548
    4549 sg = !!(sk->sk_route_caps & NETIF_F_SG);
    4550
    4551 @@ -1219,7 +1219,7 @@ new_segment:
    4552
    4553 if (!skb_can_coalesce(skb, i, pfrag->page,
    4554 pfrag->offset)) {
    4555 - if (i == sysctl_max_skb_frags || !sg) {
    4556 + if (i >= sysctl_max_skb_frags || !sg) {
    4557 tcp_mark_push(tp, skb);
    4558 goto new_segment;
    4559 }
    4560 diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c
    4561 index 10d728b6804c..ab37c6775630 100644
    4562 --- a/net/ipv4/tcp_dctcp.c
    4563 +++ b/net/ipv4/tcp_dctcp.c
    4564 @@ -56,6 +56,7 @@ struct dctcp {
    4565 u32 next_seq;
    4566 u32 ce_state;
    4567 u32 delayed_ack_reserved;
    4568 + u32 loss_cwnd;
    4569 };
    4570
    4571 static unsigned int dctcp_shift_g __read_mostly = 4; /* g = 1/2^4 */
    4572 @@ -96,6 +97,7 @@ static void dctcp_init(struct sock *sk)
    4573 ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
    4574
    4575 ca->delayed_ack_reserved = 0;
    4576 + ca->loss_cwnd = 0;
    4577 ca->ce_state = 0;
    4578
    4579 dctcp_reset(tp, ca);
    4580 @@ -111,9 +113,10 @@ static void dctcp_init(struct sock *sk)
    4581
    4582 static u32 dctcp_ssthresh(struct sock *sk)
    4583 {
    4584 - const struct dctcp *ca = inet_csk_ca(sk);
    4585 + struct dctcp *ca = inet_csk_ca(sk);
    4586 struct tcp_sock *tp = tcp_sk(sk);
    4587
    4588 + ca->loss_cwnd = tp->snd_cwnd;
    4589 return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
    4590 }
    4591
    4592 @@ -308,12 +311,20 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr,
    4593 return 0;
    4594 }
    4595
    4596 +static u32 dctcp_cwnd_undo(struct sock *sk)
    4597 +{
    4598 + const struct dctcp *ca = inet_csk_ca(sk);
    4599 +
    4600 + return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
    4601 +}
    4602 +
    4603 static struct tcp_congestion_ops dctcp __read_mostly = {
    4604 .init = dctcp_init,
    4605 .in_ack_event = dctcp_update_alpha,
    4606 .cwnd_event = dctcp_cwnd_event,
    4607 .ssthresh = dctcp_ssthresh,
    4608 .cong_avoid = tcp_reno_cong_avoid,
    4609 + .undo_cwnd = dctcp_cwnd_undo,
    4610 .set_state = dctcp_state,
    4611 .get_info = dctcp_get_info,
    4612 .flags = TCP_CONG_NEEDS_ECN,
    4613 diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
    4614 index 7158d4f8dae4..7b235fa12903 100644
    4615 --- a/net/ipv4/tcp_ipv4.c
    4616 +++ b/net/ipv4/tcp_ipv4.c
    4617 @@ -1537,6 +1537,21 @@ bool tcp_prequeue(struct sock *sk, struct sk_buff *skb)
    4618 }
    4619 EXPORT_SYMBOL(tcp_prequeue);
    4620
    4621 +int tcp_filter(struct sock *sk, struct sk_buff *skb)
    4622 +{
    4623 + struct tcphdr *th = (struct tcphdr *)skb->data;
    4624 + unsigned int eaten = skb->len;
    4625 + int err;
    4626 +
    4627 + err = sk_filter_trim_cap(sk, skb, th->doff * 4);
    4628 + if (!err) {
    4629 + eaten -= skb->len;
    4630 + TCP_SKB_CB(skb)->end_seq -= eaten;
    4631 + }
    4632 + return err;
    4633 +}
    4634 +EXPORT_SYMBOL(tcp_filter);
    4635 +
    4636 /*
    4637 * From tcp_input.c
    4638 */
    4639 @@ -1648,8 +1663,10 @@ process:
    4640
    4641 nf_reset(skb);
    4642
    4643 - if (sk_filter(sk, skb))
    4644 + if (tcp_filter(sk, skb))
    4645 goto discard_and_relse;
    4646 + th = (const struct tcphdr *)skb->data;
    4647 + iph = ip_hdr(skb);
    4648
    4649 skb->dev = NULL;
    4650
    4651 diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c
    4652 index bd59c343d35f..7370ad2e693a 100644
    4653 --- a/net/ipv6/icmp.c
    4654 +++ b/net/ipv6/icmp.c
    4655 @@ -448,7 +448,7 @@ static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info,
    4656 if (__ipv6_addr_needs_scope_id(addr_type))
    4657 iif = skb->dev->ifindex;
    4658 else
    4659 - iif = l3mdev_master_ifindex(skb->dev);
    4660 + iif = l3mdev_master_ifindex(skb_dst(skb)->dev);
    4661
    4662 /*
    4663 * Must not send error if the source does not uniquely
    4664 diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
    4665 index fc67822c42e0..af6a09efad5b 100644
    4666 --- a/net/ipv6/tcp_ipv6.c
    4667 +++ b/net/ipv6/tcp_ipv6.c
    4668 @@ -1228,7 +1228,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
    4669 if (skb->protocol == htons(ETH_P_IP))
    4670 return tcp_v4_do_rcv(sk, skb);
    4671
    4672 - if (sk_filter(sk, skb))
    4673 + if (tcp_filter(sk, skb))
    4674 goto discard;
    4675
    4676 /*
    4677 @@ -1455,8 +1455,10 @@ process:
    4678 if (tcp_v6_inbound_md5_hash(sk, skb))
    4679 goto discard_and_relse;
    4680
    4681 - if (sk_filter(sk, skb))
    4682 + if (tcp_filter(sk, skb))
    4683 goto discard_and_relse;
    4684 + th = (const struct tcphdr *)skb->data;
    4685 + hdr = ipv6_hdr(skb);
    4686
    4687 skb->dev = NULL;
    4688
    4689 diff --git a/net/sctp/socket.c b/net/sctp/socket.c
    4690 index baccbf3c1c60..7b0e059bf13b 100644
    4691 --- a/net/sctp/socket.c
    4692 +++ b/net/sctp/socket.c
    4693 @@ -1214,9 +1214,12 @@ static int __sctp_connect(struct sock *sk,
    4694
    4695 timeo = sock_sndtimeo(sk, f_flags & O_NONBLOCK);
    4696
    4697 - err = sctp_wait_for_connect(asoc, &timeo);
    4698 - if ((err == 0 || err == -EINPROGRESS) && assoc_id)
    4699 + if (assoc_id)
    4700 *assoc_id = asoc->assoc_id;
    4701 + err = sctp_wait_for_connect(asoc, &timeo);
    4702 + /* Note: the asoc may be freed after the return of
    4703 + * sctp_wait_for_connect.
    4704 + */
    4705
    4706 /* Don't free association on exit. */
    4707 asoc = NULL;
    4708 @@ -4278,19 +4281,18 @@ static void sctp_shutdown(struct sock *sk, int how)
    4709 {
    4710 struct net *net = sock_net(sk);
    4711 struct sctp_endpoint *ep;
    4712 - struct sctp_association *asoc;
    4713
    4714 if (!sctp_style(sk, TCP))
    4715 return;
    4716
    4717 - if (how & SEND_SHUTDOWN) {
    4718 + ep = sctp_sk(sk)->ep;
    4719 + if (how & SEND_SHUTDOWN && !list_empty(&ep->asocs)) {
    4720 + struct sctp_association *asoc;
    4721 +
    4722 sk->sk_state = SCTP_SS_CLOSING;
    4723 - ep = sctp_sk(sk)->ep;
    4724 - if (!list_empty(&ep->asocs)) {
    4725 - asoc = list_entry(ep->asocs.next,
    4726 - struct sctp_association, asocs);
    4727 - sctp_primitive_SHUTDOWN(net, asoc, NULL);
    4728 - }
    4729 + asoc = list_entry(ep->asocs.next,
    4730 + struct sctp_association, asocs);
    4731 + sctp_primitive_SHUTDOWN(net, asoc, NULL);
    4732 }
    4733 }
    4734
    4735 diff --git a/net/socket.c b/net/socket.c
    4736 index a1bd16106625..03bc2c289c94 100644
    4737 --- a/net/socket.c
    4738 +++ b/net/socket.c
    4739 @@ -2041,6 +2041,8 @@ int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen,
    4740 if (err)
    4741 break;
    4742 ++datagrams;
    4743 + if (msg_data_left(&msg_sys))
    4744 + break;
    4745 cond_resched();
    4746 }
    4747
    4748 diff --git a/tools/spi/spidev_test.c b/tools/spi/spidev_test.c
    4749 index f3825b676e38..f046b77cfefe 100644
    4750 --- a/tools/spi/spidev_test.c
    4751 +++ b/tools/spi/spidev_test.c
    4752 @@ -19,6 +19,7 @@
    4753 #include <getopt.h>
    4754 #include <fcntl.h>
    4755 #include <sys/ioctl.h>
    4756 +#include <linux/ioctl.h>
    4757 #include <sys/stat.h>
    4758 #include <linux/types.h>
    4759 #include <linux/spi/spidev.h>