Magellan Linux

  • Back to homepage
    • /[pkg-src]/trunk/kernel-magellan/patches-4.8/0109-4.8.10-all-fixes.patch

    Annotation of /trunk/kernel-magellan/patches-4.8/0109-4.8.10-all-fixes.patch

    Parent Directory Parent Directory | Revision Log Revision Log

    Revision 2849 - (hide annotations) (download)
    Tue Nov 22 13:19:55 2016 UTC (7 years, 6 months ago) by niro
    File size: 141674 byte(s) 
    -linux-4.8.10
    1 niro 2849 diff --git a/Makefile b/Makefile
    2     index c1519ab85258..7cf2b4985703 100644
    3     --- a/Makefile
    4     +++ b/Makefile
    5     @@ -1,6 +1,6 @@
    6     VERSION = 4
    7     PATCHLEVEL = 8
    8     -SUBLEVEL = 9
    9     +SUBLEVEL = 10
    10     EXTRAVERSION =
    11     NAME = Psychotic Stoned Sheep
    12    
    13     diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
    14     index 37a315d0ddd4..a6847fc05a6d 100644
    15     --- a/arch/sparc/include/asm/uaccess_64.h
    16     +++ b/arch/sparc/include/asm/uaccess_64.h
    17     @@ -98,7 +98,6 @@ struct exception_table_entry {
    18     unsigned int insn, fixup;
    19     };
    20    
    21     -void __ret_efault(void);
    22     void __retl_efault(void);
    23    
    24     /* Uh, these should become the main single-value transfer routines..
    25     @@ -205,55 +204,34 @@ int __get_user_bad(void);
    26     unsigned long __must_check ___copy_from_user(void *to,
    27     const void __user *from,
    28     unsigned long size);
    29     -unsigned long copy_from_user_fixup(void *to, const void __user *from,
    30     - unsigned long size);
    31     static inline unsigned long __must_check
    32     copy_from_user(void *to, const void __user *from, unsigned long size)
    33     {
    34     - unsigned long ret;
    35     -
    36     check_object_size(to, size, false);
    37    
    38     - ret = ___copy_from_user(to, from, size);
    39     - if (unlikely(ret))
    40     - ret = copy_from_user_fixup(to, from, size);
    41     -
    42     - return ret;
    43     + return ___copy_from_user(to, from, size);
    44     }
    45     #define __copy_from_user copy_from_user
    46    
    47     unsigned long __must_check ___copy_to_user(void __user *to,
    48     const void *from,
    49     unsigned long size);
    50     -unsigned long copy_to_user_fixup(void __user *to, const void *from,
    51     - unsigned long size);
    52     static inline unsigned long __must_check
    53     copy_to_user(void __user *to, const void *from, unsigned long size)
    54     {
    55     - unsigned long ret;
    56     -
    57     check_object_size(from, size, true);
    58    
    59     - ret = ___copy_to_user(to, from, size);
    60     - if (unlikely(ret))
    61     - ret = copy_to_user_fixup(to, from, size);
    62     - return ret;
    63     + return ___copy_to_user(to, from, size);
    64     }
    65     #define __copy_to_user copy_to_user
    66    
    67     unsigned long __must_check ___copy_in_user(void __user *to,
    68     const void __user *from,
    69     unsigned long size);
    70     -unsigned long copy_in_user_fixup(void __user *to, void __user *from,
    71     - unsigned long size);
    72     static inline unsigned long __must_check
    73     copy_in_user(void __user *to, void __user *from, unsigned long size)
    74     {
    75     - unsigned long ret = ___copy_in_user(to, from, size);
    76     -
    77     - if (unlikely(ret))
    78     - ret = copy_in_user_fixup(to, from, size);
    79     - return ret;
    80     + return ___copy_in_user(to, from, size);
    81     }
    82     #define __copy_in_user copy_in_user
    83    
    84     diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
    85     index a076b4249e62..5f1f3ae21657 100644
    86     --- a/arch/sparc/kernel/head_64.S
    87     +++ b/arch/sparc/kernel/head_64.S
    88     @@ -922,47 +922,11 @@ prom_tba: .xword 0
    89     tlb_type: .word 0 /* Must NOT end up in BSS */
    90     .section ".fixup",#alloc,#execinstr
    91    
    92     - .globl __ret_efault, __retl_efault, __ret_one, __retl_one
    93     -ENTRY(__ret_efault)
    94     - ret
    95     - restore %g0, -EFAULT, %o0
    96     -ENDPROC(__ret_efault)
    97     -
    98     ENTRY(__retl_efault)
    99     retl
    100     mov -EFAULT, %o0
    101     ENDPROC(__retl_efault)
    102    
    103     -ENTRY(__retl_one)
    104     - retl
    105     - mov 1, %o0
    106     -ENDPROC(__retl_one)
    107     -
    108     -ENTRY(__retl_one_fp)
    109     - VISExitHalf
    110     - retl
    111     - mov 1, %o0
    112     -ENDPROC(__retl_one_fp)
    113     -
    114     -ENTRY(__ret_one_asi)
    115     - wr %g0, ASI_AIUS, %asi
    116     - ret
    117     - restore %g0, 1, %o0
    118     -ENDPROC(__ret_one_asi)
    119     -
    120     -ENTRY(__retl_one_asi)
    121     - wr %g0, ASI_AIUS, %asi
    122     - retl
    123     - mov 1, %o0
    124     -ENDPROC(__retl_one_asi)
    125     -
    126     -ENTRY(__retl_one_asi_fp)
    127     - wr %g0, ASI_AIUS, %asi
    128     - VISExitHalf
    129     - retl
    130     - mov 1, %o0
    131     -ENDPROC(__retl_one_asi_fp)
    132     -
    133     ENTRY(__retl_o1)
    134     retl
    135     mov %o1, %o0
    136     diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
    137     index 59bbeff55024..07933b9e9ce0 100644
    138     --- a/arch/sparc/kernel/jump_label.c
    139     +++ b/arch/sparc/kernel/jump_label.c
    140     @@ -13,19 +13,30 @@
    141     void arch_jump_label_transform(struct jump_entry *entry,
    142     enum jump_label_type type)
    143     {
    144     - u32 val;
    145     u32 *insn = (u32 *) (unsigned long) entry->code;
    146     + u32 val;
    147    
    148     if (type == JUMP_LABEL_JMP) {
    149     s32 off = (s32)entry->target - (s32)entry->code;
    150     + bool use_v9_branch = false;
    151     +
    152     + BUG_ON(off & 3);
    153    
    154     #ifdef CONFIG_SPARC64
    155     - /* ba,pt %xcc, . + (off << 2) */
    156     - val = 0x10680000 | ((u32) off >> 2);
    157     -#else
    158     - /* ba . + (off << 2) */
    159     - val = 0x10800000 | ((u32) off >> 2);
    160     + if (off <= 0xfffff && off >= -0x100000)
    161     + use_v9_branch = true;
    162     #endif
    163     + if (use_v9_branch) {
    164     + /* WDISP19 - target is . + immed << 2 */
    165     + /* ba,pt %xcc, . + off */
    166     + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff);
    167     + } else {
    168     + /* WDISP22 - target is . + immed << 2 */
    169     + BUG_ON(off > 0x7fffff);
    170     + BUG_ON(off < -0x800000);
    171     + /* ba . + off */
    172     + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff);
    173     + }
    174     } else {
    175     val = 0x01000000;
    176     }
    177     diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c
    178     index 9e034f29dcc5..20ffb052fe38 100644
    179     --- a/arch/sparc/kernel/sparc_ksyms_64.c
    180     +++ b/arch/sparc/kernel/sparc_ksyms_64.c
    181     @@ -27,7 +27,6 @@ EXPORT_SYMBOL(__flushw_user);
    182     EXPORT_SYMBOL_GPL(real_hard_smp_processor_id);
    183    
    184     /* from head_64.S */
    185     -EXPORT_SYMBOL(__ret_efault);
    186     EXPORT_SYMBOL(tlb_type);
    187     EXPORT_SYMBOL(sun4v_chip_type);
    188     EXPORT_SYMBOL(prom_root_node);
    189     diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
    190     index b7d0bd6b1406..69a439fa2fc1 100644
    191     --- a/arch/sparc/lib/GENcopy_from_user.S
    192     +++ b/arch/sparc/lib/GENcopy_from_user.S
    193     @@ -3,11 +3,11 @@
    194     * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    195     */
    196    
    197     -#define EX_LD(x) \
    198     +#define EX_LD(x,y) \
    199     98: x; \
    200     .section __ex_table,"a";\
    201     .align 4; \
    202     - .word 98b, __retl_one; \
    203     + .word 98b, y; \
    204     .text; \
    205     .align 4;
    206    
    207     diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
    208     index 780550e1afc7..9947427ce354 100644
    209     --- a/arch/sparc/lib/GENcopy_to_user.S
    210     +++ b/arch/sparc/lib/GENcopy_to_user.S
    211     @@ -3,11 +3,11 @@
    212     * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    213     */
    214    
    215     -#define EX_ST(x) \
    216     +#define EX_ST(x,y) \
    217     98: x; \
    218     .section __ex_table,"a";\
    219     .align 4; \
    220     - .word 98b, __retl_one; \
    221     + .word 98b, y; \
    222     .text; \
    223     .align 4;
    224    
    225     diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S
    226     index 89358ee94851..059ea24ad73d 100644
    227     --- a/arch/sparc/lib/GENmemcpy.S
    228     +++ b/arch/sparc/lib/GENmemcpy.S
    229     @@ -4,21 +4,18 @@
    230     */
    231    
    232     #ifdef __KERNEL__
    233     +#include <linux/linkage.h>
    234     #define GLOBAL_SPARE %g7
    235     #else
    236     #define GLOBAL_SPARE %g5
    237     #endif
    238    
    239     #ifndef EX_LD
    240     -#define EX_LD(x) x
    241     +#define EX_LD(x,y) x
    242     #endif
    243    
    244     #ifndef EX_ST
    245     -#define EX_ST(x) x
    246     -#endif
    247     -
    248     -#ifndef EX_RETVAL
    249     -#define EX_RETVAL(x) x
    250     +#define EX_ST(x,y) x
    251     #endif
    252    
    253     #ifndef LOAD
    254     @@ -45,6 +42,29 @@
    255     .register %g3,#scratch
    256    
    257     .text
    258     +
    259     +#ifndef EX_RETVAL
    260     +#define EX_RETVAL(x) x
    261     +ENTRY(GEN_retl_o4_1)
    262     + add %o4, %o2, %o4
    263     + retl
    264     + add %o4, 1, %o0
    265     +ENDPROC(GEN_retl_o4_1)
    266     +ENTRY(GEN_retl_g1_8)
    267     + add %g1, %o2, %g1
    268     + retl
    269     + add %g1, 8, %o0
    270     +ENDPROC(GEN_retl_g1_8)
    271     +ENTRY(GEN_retl_o2_4)
    272     + retl
    273     + add %o2, 4, %o0
    274     +ENDPROC(GEN_retl_o2_4)
    275     +ENTRY(GEN_retl_o2_1)
    276     + retl
    277     + add %o2, 1, %o0
    278     +ENDPROC(GEN_retl_o2_1)
    279     +#endif
    280     +
    281     .align 64
    282    
    283     .globl FUNC_NAME
    284     @@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    285     sub %g0, %o4, %o4
    286     sub %o2, %o4, %o2
    287     1: subcc %o4, 1, %o4
    288     - EX_LD(LOAD(ldub, %o1, %g1))
    289     - EX_ST(STORE(stb, %g1, %o0))
    290     + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1)
    291     + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1)
    292     add %o1, 1, %o1
    293     bne,pt %XCC, 1b
    294     add %o0, 1, %o0
    295     @@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    296     andn %o2, 0x7, %g1
    297     sub %o2, %g1, %o2
    298     1: subcc %g1, 0x8, %g1
    299     - EX_LD(LOAD(ldx, %o1, %g2))
    300     - EX_ST(STORE(stx, %g2, %o0))
    301     + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8)
    302     + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8)
    303     add %o1, 0x8, %o1
    304     bne,pt %XCC, 1b
    305     add %o0, 0x8, %o0
    306     @@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    307    
    308     1:
    309     subcc %o2, 4, %o2
    310     - EX_LD(LOAD(lduw, %o1, %g1))
    311     - EX_ST(STORE(stw, %g1, %o1 + %o3))
    312     + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4)
    313     + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4)
    314     bgu,pt %XCC, 1b
    315     add %o1, 4, %o1
    316    
    317     @@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    318     .align 32
    319     90:
    320     subcc %o2, 1, %o2
    321     - EX_LD(LOAD(ldub, %o1, %g1))
    322     - EX_ST(STORE(stb, %g1, %o1 + %o3))
    323     + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1)
    324     + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1)
    325     bgu,pt %XCC, 90b
    326     add %o1, 1, %o1
    327     retl
    328     diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
    329     index 3269b0234093..4f2384a4286a 100644
    330     --- a/arch/sparc/lib/Makefile
    331     +++ b/arch/sparc/lib/Makefile
    332     @@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o
    333     lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
    334     lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
    335    
    336     -lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o
    337     +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
    338     lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
    339    
    340     obj-$(CONFIG_SPARC64) += iomap.o
    341     diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
    342     index d5242b8c4f94..b79a6998d87c 100644
    343     --- a/arch/sparc/lib/NG2copy_from_user.S
    344     +++ b/arch/sparc/lib/NG2copy_from_user.S
    345     @@ -3,19 +3,19 @@
    346     * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    347     */
    348    
    349     -#define EX_LD(x) \
    350     +#define EX_LD(x,y) \
    351     98: x; \
    352     .section __ex_table,"a";\
    353     .align 4; \
    354     - .word 98b, __retl_one_asi;\
    355     + .word 98b, y; \
    356     .text; \
    357     .align 4;
    358    
    359     -#define EX_LD_FP(x) \
    360     +#define EX_LD_FP(x,y) \
    361     98: x; \
    362     .section __ex_table,"a";\
    363     .align 4; \
    364     - .word 98b, __retl_one_asi_fp;\
    365     + .word 98b, y##_fp; \
    366     .text; \
    367     .align 4;
    368    
    369     diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
    370     index 4e962d993b10..dcec55f254ab 100644
    371     --- a/arch/sparc/lib/NG2copy_to_user.S
    372     +++ b/arch/sparc/lib/NG2copy_to_user.S
    373     @@ -3,19 +3,19 @@
    374     * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    375     */
    376    
    377     -#define EX_ST(x) \
    378     +#define EX_ST(x,y) \
    379     98: x; \
    380     .section __ex_table,"a";\
    381     .align 4; \
    382     - .word 98b, __retl_one_asi;\
    383     + .word 98b, y; \
    384     .text; \
    385     .align 4;
    386    
    387     -#define EX_ST_FP(x) \
    388     +#define EX_ST_FP(x,y) \
    389     98: x; \
    390     .section __ex_table,"a";\
    391     .align 4; \
    392     - .word 98b, __retl_one_asi_fp;\
    393     + .word 98b, y##_fp; \
    394     .text; \
    395     .align 4;
    396    
    397     diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
    398     index d5f585df2f3f..c629dbd121b6 100644
    399     --- a/arch/sparc/lib/NG2memcpy.S
    400     +++ b/arch/sparc/lib/NG2memcpy.S
    401     @@ -4,6 +4,7 @@
    402     */
    403    
    404     #ifdef __KERNEL__
    405     +#include <linux/linkage.h>
    406     #include <asm/visasm.h>
    407     #include <asm/asi.h>
    408     #define GLOBAL_SPARE %g7
    409     @@ -32,21 +33,17 @@
    410     #endif
    411    
    412     #ifndef EX_LD
    413     -#define EX_LD(x) x
    414     +#define EX_LD(x,y) x
    415     #endif
    416     #ifndef EX_LD_FP
    417     -#define EX_LD_FP(x) x
    418     +#define EX_LD_FP(x,y) x
    419     #endif
    420    
    421     #ifndef EX_ST
    422     -#define EX_ST(x) x
    423     +#define EX_ST(x,y) x
    424     #endif
    425     #ifndef EX_ST_FP
    426     -#define EX_ST_FP(x) x
    427     -#endif
    428     -
    429     -#ifndef EX_RETVAL
    430     -#define EX_RETVAL(x) x
    431     +#define EX_ST_FP(x,y) x
    432     #endif
    433    
    434     #ifndef LOAD
    435     @@ -140,45 +137,110 @@
    436     fsrc2 %x6, %f12; \
    437     fsrc2 %x7, %f14;
    438     #define FREG_LOAD_1(base, x0) \
    439     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0))
    440     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1)
    441     #define FREG_LOAD_2(base, x0, x1) \
    442     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    443     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1));
    444     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    445     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1);
    446     #define FREG_LOAD_3(base, x0, x1, x2) \
    447     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    448     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    449     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2));
    450     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    451     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    452     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1);
    453     #define FREG_LOAD_4(base, x0, x1, x2, x3) \
    454     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    455     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    456     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    457     - EX_LD_FP(LOAD(ldd, base + 0x18, %x3));
    458     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    459     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    460     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    461     + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1);
    462     #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
    463     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    464     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    465     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    466     - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    467     - EX_LD_FP(LOAD(ldd, base + 0x20, %x4));
    468     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    469     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    470     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    471     + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    472     + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1);
    473     #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
    474     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    475     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    476     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    477     - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    478     - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
    479     - EX_LD_FP(LOAD(ldd, base + 0x28, %x5));
    480     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    481     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    482     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    483     + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    484     + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
    485     + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1);
    486     #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
    487     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    488     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    489     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    490     - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    491     - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
    492     - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \
    493     - EX_LD_FP(LOAD(ldd, base + 0x30, %x6));
    494     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    495     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    496     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    497     + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    498     + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
    499     + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \
    500     + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1);
    501    
    502     .register %g2,#scratch
    503     .register %g3,#scratch
    504    
    505     .text
    506     +#ifndef EX_RETVAL
    507     +#define EX_RETVAL(x) x
    508     +__restore_fp:
    509     + VISExitHalf
    510     +__restore_asi:
    511     + retl
    512     + wr %g0, ASI_AIUS, %asi
    513     +ENTRY(NG2_retl_o2)
    514     + ba,pt %xcc, __restore_asi
    515     + mov %o2, %o0
    516     +ENDPROC(NG2_retl_o2)
    517     +ENTRY(NG2_retl_o2_plus_1)
    518     + ba,pt %xcc, __restore_asi
    519     + add %o2, 1, %o0
    520     +ENDPROC(NG2_retl_o2_plus_1)
    521     +ENTRY(NG2_retl_o2_plus_4)
    522     + ba,pt %xcc, __restore_asi
    523     + add %o2, 4, %o0
    524     +ENDPROC(NG2_retl_o2_plus_4)
    525     +ENTRY(NG2_retl_o2_plus_8)
    526     + ba,pt %xcc, __restore_asi
    527     + add %o2, 8, %o0
    528     +ENDPROC(NG2_retl_o2_plus_8)
    529     +ENTRY(NG2_retl_o2_plus_o4_plus_1)
    530     + add %o4, 1, %o4
    531     + ba,pt %xcc, __restore_asi
    532     + add %o2, %o4, %o0
    533     +ENDPROC(NG2_retl_o2_plus_o4_plus_1)
    534     +ENTRY(NG2_retl_o2_plus_o4_plus_8)
    535     + add %o4, 8, %o4
    536     + ba,pt %xcc, __restore_asi
    537     + add %o2, %o4, %o0
    538     +ENDPROC(NG2_retl_o2_plus_o4_plus_8)
    539     +ENTRY(NG2_retl_o2_plus_o4_plus_16)
    540     + add %o4, 16, %o4
    541     + ba,pt %xcc, __restore_asi
    542     + add %o2, %o4, %o0
    543     +ENDPROC(NG2_retl_o2_plus_o4_plus_16)
    544     +ENTRY(NG2_retl_o2_plus_g1_fp)
    545     + ba,pt %xcc, __restore_fp
    546     + add %o2, %g1, %o0
    547     +ENDPROC(NG2_retl_o2_plus_g1_fp)
    548     +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp)
    549     + add %g1, 64, %g1
    550     + ba,pt %xcc, __restore_fp
    551     + add %o2, %g1, %o0
    552     +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp)
    553     +ENTRY(NG2_retl_o2_plus_g1_plus_1)
    554     + add %g1, 1, %g1
    555     + ba,pt %xcc, __restore_asi
    556     + add %o2, %g1, %o0
    557     +ENDPROC(NG2_retl_o2_plus_g1_plus_1)
    558     +ENTRY(NG2_retl_o2_and_7_plus_o4)
    559     + and %o2, 7, %o2
    560     + ba,pt %xcc, __restore_asi
    561     + add %o2, %o4, %o0
    562     +ENDPROC(NG2_retl_o2_and_7_plus_o4)
    563     +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8)
    564     + and %o2, 7, %o2
    565     + add %o4, 8, %o4
    566     + ba,pt %xcc, __restore_asi
    567     + add %o2, %o4, %o0
    568     +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8)
    569     +#endif
    570     +
    571     .align 64
    572    
    573     .globl FUNC_NAME
    574     @@ -230,8 +292,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    575     sub %g0, %o4, %o4 ! bytes to align dst
    576     sub %o2, %o4, %o2
    577     1: subcc %o4, 1, %o4
    578     - EX_LD(LOAD(ldub, %o1, %g1))
    579     - EX_ST(STORE(stb, %g1, %o0))
    580     + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1)
    581     + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1)
    582     add %o1, 1, %o1
    583     bne,pt %XCC, 1b
    584     add %o0, 1, %o0
    585     @@ -281,11 +343,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    586     nop
    587     /* fall through for 0 < low bits < 8 */
    588     110: sub %o4, 64, %g2
    589     - EX_LD_FP(LOAD_BLK(%g2, %f0))
    590     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    591     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    592     + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1)
    593     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    594     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    595     FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
    596     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    597     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    598     FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
    599     subcc %g1, 64, %g1
    600     add %o4, 64, %o4
    601     @@ -296,10 +358,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    602    
    603     120: sub %o4, 56, %g2
    604     FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
    605     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    606     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    607     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    608     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    609     FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
    610     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    611     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    612     FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
    613     subcc %g1, 64, %g1
    614     add %o4, 64, %o4
    615     @@ -310,10 +372,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    616    
    617     130: sub %o4, 48, %g2
    618     FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
    619     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    620     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    621     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    622     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    623     FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
    624     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    625     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    626     FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
    627     subcc %g1, 64, %g1
    628     add %o4, 64, %o4
    629     @@ -324,10 +386,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    630    
    631     140: sub %o4, 40, %g2
    632     FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
    633     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    634     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    635     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    636     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    637     FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
    638     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    639     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    640     FREG_MOVE_5(f22, f24, f26, f28, f30)
    641     subcc %g1, 64, %g1
    642     add %o4, 64, %o4
    643     @@ -338,10 +400,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    644    
    645     150: sub %o4, 32, %g2
    646     FREG_LOAD_4(%g2, f0, f2, f4, f6)
    647     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    648     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    649     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    650     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    651     FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
    652     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    653     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    654     FREG_MOVE_4(f24, f26, f28, f30)
    655     subcc %g1, 64, %g1
    656     add %o4, 64, %o4
    657     @@ -352,10 +414,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    658    
    659     160: sub %o4, 24, %g2
    660     FREG_LOAD_3(%g2, f0, f2, f4)
    661     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    662     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    663     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    664     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    665     FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
    666     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    667     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    668     FREG_MOVE_3(f26, f28, f30)
    669     subcc %g1, 64, %g1
    670     add %o4, 64, %o4
    671     @@ -366,10 +428,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    672    
    673     170: sub %o4, 16, %g2
    674     FREG_LOAD_2(%g2, f0, f2)
    675     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    676     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    677     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    678     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    679     FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
    680     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    681     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    682     FREG_MOVE_2(f28, f30)
    683     subcc %g1, 64, %g1
    684     add %o4, 64, %o4
    685     @@ -380,10 +442,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    686    
    687     180: sub %o4, 8, %g2
    688     FREG_LOAD_1(%g2, f0)
    689     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    690     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    691     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    692     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    693     FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
    694     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    695     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    696     FREG_MOVE_1(f30)
    697     subcc %g1, 64, %g1
    698     add %o4, 64, %o4
    699     @@ -393,10 +455,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    700     nop
    701    
    702     190:
    703     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    704     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    705     subcc %g1, 64, %g1
    706     - EX_LD_FP(LOAD_BLK(%o4, %f0))
    707     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    708     + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64)
    709     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64)
    710     add %o4, 64, %o4
    711     bne,pt %xcc, 1b
    712     LOAD(prefetch, %o4 + 64, #one_read)
    713     @@ -423,28 +485,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    714     andn %o2, 0xf, %o4
    715     and %o2, 0xf, %o2
    716     1: subcc %o4, 0x10, %o4
    717     - EX_LD(LOAD(ldx, %o1, %o5))
    718     + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16)
    719     add %o1, 0x08, %o1
    720     - EX_LD(LOAD(ldx, %o1, %g1))
    721     + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16)
    722     sub %o1, 0x08, %o1
    723     - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
    724     + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16)
    725     add %o1, 0x8, %o1
    726     - EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE))
    727     + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8)
    728     bgu,pt %XCC, 1b
    729     add %o1, 0x8, %o1
    730     73: andcc %o2, 0x8, %g0
    731     be,pt %XCC, 1f
    732     nop
    733     sub %o2, 0x8, %o2
    734     - EX_LD(LOAD(ldx, %o1, %o5))
    735     - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
    736     + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8)
    737     + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8)
    738     add %o1, 0x8, %o1
    739     1: andcc %o2, 0x4, %g0
    740     be,pt %XCC, 1f
    741     nop
    742     sub %o2, 0x4, %o2
    743     - EX_LD(LOAD(lduw, %o1, %o5))
    744     - EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE))
    745     + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4)
    746     + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
    747     add %o1, 0x4, %o1
    748     1: cmp %o2, 0
    749     be,pt %XCC, 85f
    750     @@ -460,8 +522,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    751     sub %o2, %g1, %o2
    752    
    753     1: subcc %g1, 1, %g1
    754     - EX_LD(LOAD(ldub, %o1, %o5))
    755     - EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE))
    756     + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1)
    757     + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1)
    758     bgu,pt %icc, 1b
    759     add %o1, 1, %o1
    760    
    761     @@ -477,16 +539,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    762    
    763     8: mov 64, GLOBAL_SPARE
    764     andn %o1, 0x7, %o1
    765     - EX_LD(LOAD(ldx, %o1, %g2))
    766     + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2)
    767     sub GLOBAL_SPARE, %g1, GLOBAL_SPARE
    768     andn %o2, 0x7, %o4
    769     sllx %g2, %g1, %g2
    770     1: add %o1, 0x8, %o1
    771     - EX_LD(LOAD(ldx, %o1, %g3))
    772     + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4)
    773     subcc %o4, 0x8, %o4
    774     srlx %g3, GLOBAL_SPARE, %o5
    775     or %o5, %g2, %o5
    776     - EX_ST(STORE(stx, %o5, %o0))
    777     + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8)
    778     add %o0, 0x8, %o0
    779     bgu,pt %icc, 1b
    780     sllx %g3, %g1, %g2
    781     @@ -506,8 +568,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    782    
    783     1:
    784     subcc %o2, 4, %o2
    785     - EX_LD(LOAD(lduw, %o1, %g1))
    786     - EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE))
    787     + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4)
    788     + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
    789     bgu,pt %XCC, 1b
    790     add %o1, 4, %o1
    791    
    792     @@ -517,8 +579,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    793     .align 32
    794     90:
    795     subcc %o2, 1, %o2
    796     - EX_LD(LOAD(ldub, %o1, %g1))
    797     - EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE))
    798     + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1)
    799     + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1)
    800     bgu,pt %XCC, 90b
    801     add %o1, 1, %o1
    802     retl
    803     diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S
    804     index 2e8ee7ad07a9..16a286c1a528 100644
    805     --- a/arch/sparc/lib/NG4copy_from_user.S
    806     +++ b/arch/sparc/lib/NG4copy_from_user.S
    807     @@ -3,19 +3,19 @@
    808     * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
    809     */
    810    
    811     -#define EX_LD(x) \
    812     +#define EX_LD(x, y) \
    813     98: x; \
    814     .section __ex_table,"a";\
    815     .align 4; \
    816     - .word 98b, __retl_one_asi;\
    817     + .word 98b, y; \
    818     .text; \
    819     .align 4;
    820    
    821     -#define EX_LD_FP(x) \
    822     +#define EX_LD_FP(x,y) \
    823     98: x; \
    824     .section __ex_table,"a";\
    825     .align 4; \
    826     - .word 98b, __retl_one_asi_fp;\
    827     + .word 98b, y##_fp; \
    828     .text; \
    829     .align 4;
    830    
    831     diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S
    832     index be0bf4590df8..6b0276ffc858 100644
    833     --- a/arch/sparc/lib/NG4copy_to_user.S
    834     +++ b/arch/sparc/lib/NG4copy_to_user.S
    835     @@ -3,19 +3,19 @@
    836     * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
    837     */
    838    
    839     -#define EX_ST(x) \
    840     +#define EX_ST(x,y) \
    841     98: x; \
    842     .section __ex_table,"a";\
    843     .align 4; \
    844     - .word 98b, __retl_one_asi;\
    845     + .word 98b, y; \
    846     .text; \
    847     .align 4;
    848    
    849     -#define EX_ST_FP(x) \
    850     +#define EX_ST_FP(x,y) \
    851     98: x; \
    852     .section __ex_table,"a";\
    853     .align 4; \
    854     - .word 98b, __retl_one_asi_fp;\
    855     + .word 98b, y##_fp; \
    856     .text; \
    857     .align 4;
    858    
    859     diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S
    860     index 8e13ee1f4454..75bb93b1437f 100644
    861     --- a/arch/sparc/lib/NG4memcpy.S
    862     +++ b/arch/sparc/lib/NG4memcpy.S
    863     @@ -4,6 +4,7 @@
    864     */
    865    
    866     #ifdef __KERNEL__
    867     +#include <linux/linkage.h>
    868     #include <asm/visasm.h>
    869     #include <asm/asi.h>
    870     #define GLOBAL_SPARE %g7
    871     @@ -46,22 +47,19 @@
    872     #endif
    873    
    874     #ifndef EX_LD
    875     -#define EX_LD(x) x
    876     +#define EX_LD(x,y) x
    877     #endif
    878     #ifndef EX_LD_FP
    879     -#define EX_LD_FP(x) x
    880     +#define EX_LD_FP(x,y) x
    881     #endif
    882    
    883     #ifndef EX_ST
    884     -#define EX_ST(x) x
    885     +#define EX_ST(x,y) x
    886     #endif
    887     #ifndef EX_ST_FP
    888     -#define EX_ST_FP(x) x
    889     +#define EX_ST_FP(x,y) x
    890     #endif
    891    
    892     -#ifndef EX_RETVAL
    893     -#define EX_RETVAL(x) x
    894     -#endif
    895    
    896     #ifndef LOAD
    897     #define LOAD(type,addr,dest) type [addr], dest
    898     @@ -94,6 +92,158 @@
    899     .register %g3,#scratch
    900    
    901     .text
    902     +#ifndef EX_RETVAL
    903     +#define EX_RETVAL(x) x
    904     +__restore_asi_fp:
    905     + VISExitHalf
    906     +__restore_asi:
    907     + retl
    908     + wr %g0, ASI_AIUS, %asi
    909     +
    910     +ENTRY(NG4_retl_o2)
    911     + ba,pt %xcc, __restore_asi
    912     + mov %o2, %o0
    913     +ENDPROC(NG4_retl_o2)
    914     +ENTRY(NG4_retl_o2_plus_1)
    915     + ba,pt %xcc, __restore_asi
    916     + add %o2, 1, %o0
    917     +ENDPROC(NG4_retl_o2_plus_1)
    918     +ENTRY(NG4_retl_o2_plus_4)
    919     + ba,pt %xcc, __restore_asi
    920     + add %o2, 4, %o0
    921     +ENDPROC(NG4_retl_o2_plus_4)
    922     +ENTRY(NG4_retl_o2_plus_o5)
    923     + ba,pt %xcc, __restore_asi
    924     + add %o2, %o5, %o0
    925     +ENDPROC(NG4_retl_o2_plus_o5)
    926     +ENTRY(NG4_retl_o2_plus_o5_plus_4)
    927     + add %o5, 4, %o5
    928     + ba,pt %xcc, __restore_asi
    929     + add %o2, %o5, %o0
    930     +ENDPROC(NG4_retl_o2_plus_o5_plus_4)
    931     +ENTRY(NG4_retl_o2_plus_o5_plus_8)
    932     + add %o5, 8, %o5
    933     + ba,pt %xcc, __restore_asi
    934     + add %o2, %o5, %o0
    935     +ENDPROC(NG4_retl_o2_plus_o5_plus_8)
    936     +ENTRY(NG4_retl_o2_plus_o5_plus_16)
    937     + add %o5, 16, %o5
    938     + ba,pt %xcc, __restore_asi
    939     + add %o2, %o5, %o0
    940     +ENDPROC(NG4_retl_o2_plus_o5_plus_16)
    941     +ENTRY(NG4_retl_o2_plus_o5_plus_24)
    942     + add %o5, 24, %o5
    943     + ba,pt %xcc, __restore_asi
    944     + add %o2, %o5, %o0
    945     +ENDPROC(NG4_retl_o2_plus_o5_plus_24)
    946     +ENTRY(NG4_retl_o2_plus_o5_plus_32)
    947     + add %o5, 32, %o5
    948     + ba,pt %xcc, __restore_asi
    949     + add %o2, %o5, %o0
    950     +ENDPROC(NG4_retl_o2_plus_o5_plus_32)
    951     +ENTRY(NG4_retl_o2_plus_g1)
    952     + ba,pt %xcc, __restore_asi
    953     + add %o2, %g1, %o0
    954     +ENDPROC(NG4_retl_o2_plus_g1)
    955     +ENTRY(NG4_retl_o2_plus_g1_plus_1)
    956     + add %g1, 1, %g1
    957     + ba,pt %xcc, __restore_asi
    958     + add %o2, %g1, %o0
    959     +ENDPROC(NG4_retl_o2_plus_g1_plus_1)
    960     +ENTRY(NG4_retl_o2_plus_g1_plus_8)
    961     + add %g1, 8, %g1
    962     + ba,pt %xcc, __restore_asi
    963     + add %o2, %g1, %o0
    964     +ENDPROC(NG4_retl_o2_plus_g1_plus_8)
    965     +ENTRY(NG4_retl_o2_plus_o4)
    966     + ba,pt %xcc, __restore_asi
    967     + add %o2, %o4, %o0
    968     +ENDPROC(NG4_retl_o2_plus_o4)
    969     +ENTRY(NG4_retl_o2_plus_o4_plus_8)
    970     + add %o4, 8, %o4
    971     + ba,pt %xcc, __restore_asi
    972     + add %o2, %o4, %o0
    973     +ENDPROC(NG4_retl_o2_plus_o4_plus_8)
    974     +ENTRY(NG4_retl_o2_plus_o4_plus_16)
    975     + add %o4, 16, %o4
    976     + ba,pt %xcc, __restore_asi
    977     + add %o2, %o4, %o0
    978     +ENDPROC(NG4_retl_o2_plus_o4_plus_16)
    979     +ENTRY(NG4_retl_o2_plus_o4_plus_24)
    980     + add %o4, 24, %o4
    981     + ba,pt %xcc, __restore_asi
    982     + add %o2, %o4, %o0
    983     +ENDPROC(NG4_retl_o2_plus_o4_plus_24)
    984     +ENTRY(NG4_retl_o2_plus_o4_plus_32)
    985     + add %o4, 32, %o4
    986     + ba,pt %xcc, __restore_asi
    987     + add %o2, %o4, %o0
    988     +ENDPROC(NG4_retl_o2_plus_o4_plus_32)
    989     +ENTRY(NG4_retl_o2_plus_o4_plus_40)
    990     + add %o4, 40, %o4
    991     + ba,pt %xcc, __restore_asi
    992     + add %o2, %o4, %o0
    993     +ENDPROC(NG4_retl_o2_plus_o4_plus_40)
    994     +ENTRY(NG4_retl_o2_plus_o4_plus_48)
    995     + add %o4, 48, %o4
    996     + ba,pt %xcc, __restore_asi
    997     + add %o2, %o4, %o0
    998     +ENDPROC(NG4_retl_o2_plus_o4_plus_48)
    999     +ENTRY(NG4_retl_o2_plus_o4_plus_56)
    1000     + add %o4, 56, %o4
    1001     + ba,pt %xcc, __restore_asi
    1002     + add %o2, %o4, %o0
    1003     +ENDPROC(NG4_retl_o2_plus_o4_plus_56)
    1004     +ENTRY(NG4_retl_o2_plus_o4_plus_64)
    1005     + add %o4, 64, %o4
    1006     + ba,pt %xcc, __restore_asi
    1007     + add %o2, %o4, %o0
    1008     +ENDPROC(NG4_retl_o2_plus_o4_plus_64)
    1009     +ENTRY(NG4_retl_o2_plus_o4_fp)
    1010     + ba,pt %xcc, __restore_asi_fp
    1011     + add %o2, %o4, %o0
    1012     +ENDPROC(NG4_retl_o2_plus_o4_fp)
    1013     +ENTRY(NG4_retl_o2_plus_o4_plus_8_fp)
    1014     + add %o4, 8, %o4
    1015     + ba,pt %xcc, __restore_asi_fp
    1016     + add %o2, %o4, %o0
    1017     +ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp)
    1018     +ENTRY(NG4_retl_o2_plus_o4_plus_16_fp)
    1019     + add %o4, 16, %o4
    1020     + ba,pt %xcc, __restore_asi_fp
    1021     + add %o2, %o4, %o0
    1022     +ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp)
    1023     +ENTRY(NG4_retl_o2_plus_o4_plus_24_fp)
    1024     + add %o4, 24, %o4
    1025     + ba,pt %xcc, __restore_asi_fp
    1026     + add %o2, %o4, %o0
    1027     +ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp)
    1028     +ENTRY(NG4_retl_o2_plus_o4_plus_32_fp)
    1029     + add %o4, 32, %o4
    1030     + ba,pt %xcc, __restore_asi_fp
    1031     + add %o2, %o4, %o0
    1032     +ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp)
    1033     +ENTRY(NG4_retl_o2_plus_o4_plus_40_fp)
    1034     + add %o4, 40, %o4
    1035     + ba,pt %xcc, __restore_asi_fp
    1036     + add %o2, %o4, %o0
    1037     +ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp)
    1038     +ENTRY(NG4_retl_o2_plus_o4_plus_48_fp)
    1039     + add %o4, 48, %o4
    1040     + ba,pt %xcc, __restore_asi_fp
    1041     + add %o2, %o4, %o0
    1042     +ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp)
    1043     +ENTRY(NG4_retl_o2_plus_o4_plus_56_fp)
    1044     + add %o4, 56, %o4
    1045     + ba,pt %xcc, __restore_asi_fp
    1046     + add %o2, %o4, %o0
    1047     +ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp)
    1048     +ENTRY(NG4_retl_o2_plus_o4_plus_64_fp)
    1049     + add %o4, 64, %o4
    1050     + ba,pt %xcc, __restore_asi_fp
    1051     + add %o2, %o4, %o0
    1052     +ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp)
    1053     +#endif
    1054     .align 64
    1055    
    1056     .globl FUNC_NAME
    1057     @@ -124,12 +274,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1058     brz,pt %g1, 51f
    1059     sub %o2, %g1, %o2
    1060    
    1061     -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
    1062     +
    1063     +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1064     add %o1, 1, %o1
    1065     subcc %g1, 1, %g1
    1066     add %o0, 1, %o0
    1067     bne,pt %icc, 1b
    1068     - EX_ST(STORE(stb, %g2, %o0 - 0x01))
    1069     + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
    1070    
    1071     51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong)
    1072     LOAD(prefetch, %o1 + 0x080, #n_reads_strong)
    1073     @@ -154,43 +305,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1074     brz,pt %g1, .Llarge_aligned
    1075     sub %o2, %g1, %o2
    1076    
    1077     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2))
    1078     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1079     add %o1, 8, %o1
    1080     subcc %g1, 8, %g1
    1081     add %o0, 8, %o0
    1082     bne,pt %icc, 1b
    1083     - EX_ST(STORE(stx, %g2, %o0 - 0x08))
    1084     + EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8)
    1085    
    1086     .Llarge_aligned:
    1087     /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */
    1088     andn %o2, 0x3f, %o4
    1089     sub %o2, %o4, %o2
    1090    
    1091     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1092     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4)
    1093     add %o1, 0x40, %o1
    1094     - EX_LD(LOAD(ldx, %o1 - 0x38, %g2))
    1095     + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4)
    1096     subcc %o4, 0x40, %o4
    1097     - EX_LD(LOAD(ldx, %o1 - 0x30, %g3))
    1098     - EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE))
    1099     - EX_LD(LOAD(ldx, %o1 - 0x20, %o5))
    1100     - EX_ST(STORE_INIT(%g1, %o0))
    1101     + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64)
    1102     + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64)
    1103     + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64)
    1104     + EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64)
    1105     add %o0, 0x08, %o0
    1106     - EX_ST(STORE_INIT(%g2, %o0))
    1107     + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56)
    1108     add %o0, 0x08, %o0
    1109     - EX_LD(LOAD(ldx, %o1 - 0x18, %g2))
    1110     - EX_ST(STORE_INIT(%g3, %o0))
    1111     + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48)
    1112     + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48)
    1113     add %o0, 0x08, %o0
    1114     - EX_LD(LOAD(ldx, %o1 - 0x10, %g3))
    1115     - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
    1116     + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40)
    1117     + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40)
    1118     add %o0, 0x08, %o0
    1119     - EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE))
    1120     - EX_ST(STORE_INIT(%o5, %o0))
    1121     + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32)
    1122     + EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32)
    1123     add %o0, 0x08, %o0
    1124     - EX_ST(STORE_INIT(%g2, %o0))
    1125     + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24)
    1126     add %o0, 0x08, %o0
    1127     - EX_ST(STORE_INIT(%g3, %o0))
    1128     + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16)
    1129     add %o0, 0x08, %o0
    1130     - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
    1131     + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8)
    1132     add %o0, 0x08, %o0
    1133     bne,pt %icc, 1b
    1134     LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
    1135     @@ -216,17 +367,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1136     sub %o2, %o4, %o2
    1137     alignaddr %o1, %g0, %g1
    1138     add %o1, %o4, %o1
    1139     - EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0))
    1140     -1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2))
    1141     + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4)
    1142     +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4)
    1143     subcc %o4, 0x40, %o4
    1144     - EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4))
    1145     - EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6))
    1146     - EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8))
    1147     - EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10))
    1148     - EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12))
    1149     - EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14))
    1150     + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64)
    1151     + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64)
    1152     + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64)
    1153     + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64)
    1154     + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64)
    1155     + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64)
    1156     faligndata %f0, %f2, %f16
    1157     - EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0))
    1158     + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64)
    1159     faligndata %f2, %f4, %f18
    1160     add %g1, 0x40, %g1
    1161     faligndata %f4, %f6, %f20
    1162     @@ -235,14 +386,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1163     faligndata %f10, %f12, %f26
    1164     faligndata %f12, %f14, %f28
    1165     faligndata %f14, %f0, %f30
    1166     - EX_ST_FP(STORE(std, %f16, %o0 + 0x00))
    1167     - EX_ST_FP(STORE(std, %f18, %o0 + 0x08))
    1168     - EX_ST_FP(STORE(std, %f20, %o0 + 0x10))
    1169     - EX_ST_FP(STORE(std, %f22, %o0 + 0x18))
    1170     - EX_ST_FP(STORE(std, %f24, %o0 + 0x20))
    1171     - EX_ST_FP(STORE(std, %f26, %o0 + 0x28))
    1172     - EX_ST_FP(STORE(std, %f28, %o0 + 0x30))
    1173     - EX_ST_FP(STORE(std, %f30, %o0 + 0x38))
    1174     + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64)
    1175     + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56)
    1176     + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48)
    1177     + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40)
    1178     + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32)
    1179     + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24)
    1180     + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16)
    1181     + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8)
    1182     add %o0, 0x40, %o0
    1183     bne,pt %icc, 1b
    1184     LOAD(prefetch, %g1 + 0x200, #n_reads_strong)
    1185     @@ -270,37 +421,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1186     andncc %o2, 0x20 - 1, %o5
    1187     be,pn %icc, 2f
    1188     sub %o2, %o5, %o2
    1189     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1190     - EX_LD(LOAD(ldx, %o1 + 0x08, %g2))
    1191     - EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE))
    1192     - EX_LD(LOAD(ldx, %o1 + 0x18, %o4))
    1193     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1194     + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5)
    1195     + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5)
    1196     + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5)
    1197     add %o1, 0x20, %o1
    1198     subcc %o5, 0x20, %o5
    1199     - EX_ST(STORE(stx, %g1, %o0 + 0x00))
    1200     - EX_ST(STORE(stx, %g2, %o0 + 0x08))
    1201     - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10))
    1202     - EX_ST(STORE(stx, %o4, %o0 + 0x18))
    1203     + EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32)
    1204     + EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24)
    1205     + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24)
    1206     + EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8)
    1207     bne,pt %icc, 1b
    1208     add %o0, 0x20, %o0
    1209     2: andcc %o2, 0x18, %o5
    1210     be,pt %icc, 3f
    1211     sub %o2, %o5, %o2
    1212     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1213     +
    1214     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1215     add %o1, 0x08, %o1
    1216     add %o0, 0x08, %o0
    1217     subcc %o5, 0x08, %o5
    1218     bne,pt %icc, 1b
    1219     - EX_ST(STORE(stx, %g1, %o0 - 0x08))
    1220     + EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8)
    1221     3: brz,pt %o2, .Lexit
    1222     cmp %o2, 0x04
    1223     bl,pn %icc, .Ltiny
    1224     nop
    1225     - EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
    1226     + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2)
    1227     add %o1, 0x04, %o1
    1228     add %o0, 0x04, %o0
    1229     subcc %o2, 0x04, %o2
    1230     bne,pn %icc, .Ltiny
    1231     - EX_ST(STORE(stw, %g1, %o0 - 0x04))
    1232     + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4)
    1233     ba,a,pt %icc, .Lexit
    1234     .Lmedium_unaligned:
    1235     /* First get dest 8 byte aligned. */
    1236     @@ -309,12 +461,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1237     brz,pt %g1, 2f
    1238     sub %o2, %g1, %o2
    1239    
    1240     -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
    1241     +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1242     add %o1, 1, %o1
    1243     subcc %g1, 1, %g1
    1244     add %o0, 1, %o0
    1245     bne,pt %icc, 1b
    1246     - EX_ST(STORE(stb, %g2, %o0 - 0x01))
    1247     + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
    1248     2:
    1249     and %o1, 0x7, %g1
    1250     brz,pn %g1, .Lmedium_noprefetch
    1251     @@ -322,16 +474,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1252     mov 64, %g2
    1253     sub %g2, %g1, %g2
    1254     andn %o1, 0x7, %o1
    1255     - EX_LD(LOAD(ldx, %o1 + 0x00, %o4))
    1256     + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2)
    1257     sllx %o4, %g1, %o4
    1258     andn %o2, 0x08 - 1, %o5
    1259     sub %o2, %o5, %o2
    1260     -1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3))
    1261     +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5)
    1262     add %o1, 0x08, %o1
    1263     subcc %o5, 0x08, %o5
    1264     srlx %g3, %g2, GLOBAL_SPARE
    1265     or GLOBAL_SPARE, %o4, GLOBAL_SPARE
    1266     - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00))
    1267     + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8)
    1268     add %o0, 0x08, %o0
    1269     bne,pt %icc, 1b
    1270     sllx %g3, %g1, %o4
    1271     @@ -342,17 +494,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1272     ba,pt %icc, .Lsmall_unaligned
    1273    
    1274     .Ltiny:
    1275     - EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
    1276     + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
    1277     subcc %o2, 1, %o2
    1278     be,pn %icc, .Lexit
    1279     - EX_ST(STORE(stb, %g1, %o0 + 0x00))
    1280     - EX_LD(LOAD(ldub, %o1 + 0x01, %g1))
    1281     + EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1)
    1282     + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2)
    1283     subcc %o2, 1, %o2
    1284     be,pn %icc, .Lexit
    1285     - EX_ST(STORE(stb, %g1, %o0 + 0x01))
    1286     - EX_LD(LOAD(ldub, %o1 + 0x02, %g1))
    1287     + EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1)
    1288     + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2)
    1289     ba,pt %icc, .Lexit
    1290     - EX_ST(STORE(stb, %g1, %o0 + 0x02))
    1291     + EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2)
    1292    
    1293     .Lsmall:
    1294     andcc %g2, 0x3, %g0
    1295     @@ -360,22 +512,22 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1296     andn %o2, 0x4 - 1, %o5
    1297     sub %o2, %o5, %o2
    1298     1:
    1299     - EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
    1300     + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1301     add %o1, 0x04, %o1
    1302     subcc %o5, 0x04, %o5
    1303     add %o0, 0x04, %o0
    1304     bne,pt %icc, 1b
    1305     - EX_ST(STORE(stw, %g1, %o0 - 0x04))
    1306     + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4)
    1307     brz,pt %o2, .Lexit
    1308     nop
    1309     ba,a,pt %icc, .Ltiny
    1310    
    1311     .Lsmall_unaligned:
    1312     -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
    1313     +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
    1314     add %o1, 1, %o1
    1315     add %o0, 1, %o0
    1316     subcc %o2, 1, %o2
    1317     bne,pt %icc, 1b
    1318     - EX_ST(STORE(stb, %g1, %o0 - 0x01))
    1319     + EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1)
    1320     ba,a,pt %icc, .Lexit
    1321     .size FUNC_NAME, .-FUNC_NAME
    1322     diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
    1323     index 5d1e4d1ac21e..9cd42fcbc781 100644
    1324     --- a/arch/sparc/lib/NGcopy_from_user.S
    1325     +++ b/arch/sparc/lib/NGcopy_from_user.S
    1326     @@ -3,11 +3,11 @@
    1327     * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
    1328     */
    1329    
    1330     -#define EX_LD(x) \
    1331     +#define EX_LD(x,y) \
    1332     98: x; \
    1333     .section __ex_table,"a";\
    1334     .align 4; \
    1335     - .word 98b, __ret_one_asi;\
    1336     + .word 98b, y; \
    1337     .text; \
    1338     .align 4;
    1339    
    1340     diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
    1341     index ff630dcb273c..5c358afd464e 100644
    1342     --- a/arch/sparc/lib/NGcopy_to_user.S
    1343     +++ b/arch/sparc/lib/NGcopy_to_user.S
    1344     @@ -3,11 +3,11 @@
    1345     * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
    1346     */
    1347    
    1348     -#define EX_ST(x) \
    1349     +#define EX_ST(x,y) \
    1350     98: x; \
    1351     .section __ex_table,"a";\
    1352     .align 4; \
    1353     - .word 98b, __ret_one_asi;\
    1354     + .word 98b, y; \
    1355     .text; \
    1356     .align 4;
    1357    
    1358     diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S
    1359     index 96a14caf6966..d88c4ed50a00 100644
    1360     --- a/arch/sparc/lib/NGmemcpy.S
    1361     +++ b/arch/sparc/lib/NGmemcpy.S
    1362     @@ -4,6 +4,7 @@
    1363     */
    1364    
    1365     #ifdef __KERNEL__
    1366     +#include <linux/linkage.h>
    1367     #include <asm/asi.h>
    1368     #include <asm/thread_info.h>
    1369     #define GLOBAL_SPARE %g7
    1370     @@ -27,15 +28,11 @@
    1371     #endif
    1372    
    1373     #ifndef EX_LD
    1374     -#define EX_LD(x) x
    1375     +#define EX_LD(x,y) x
    1376     #endif
    1377    
    1378     #ifndef EX_ST
    1379     -#define EX_ST(x) x
    1380     -#endif
    1381     -
    1382     -#ifndef EX_RETVAL
    1383     -#define EX_RETVAL(x) x
    1384     +#define EX_ST(x,y) x
    1385     #endif
    1386    
    1387     #ifndef LOAD
    1388     @@ -79,6 +76,92 @@
    1389     .register %g3,#scratch
    1390    
    1391     .text
    1392     +#ifndef EX_RETVAL
    1393     +#define EX_RETVAL(x) x
    1394     +__restore_asi:
    1395     + ret
    1396     + wr %g0, ASI_AIUS, %asi
    1397     + restore
    1398     +ENTRY(NG_ret_i2_plus_i4_plus_1)
    1399     + ba,pt %xcc, __restore_asi
    1400     + add %i2, %i5, %i0
    1401     +ENDPROC(NG_ret_i2_plus_i4_plus_1)
    1402     +ENTRY(NG_ret_i2_plus_g1)
    1403     + ba,pt %xcc, __restore_asi
    1404     + add %i2, %g1, %i0
    1405     +ENDPROC(NG_ret_i2_plus_g1)
    1406     +ENTRY(NG_ret_i2_plus_g1_minus_8)
    1407     + sub %g1, 8, %g1
    1408     + ba,pt %xcc, __restore_asi
    1409     + add %i2, %g1, %i0
    1410     +ENDPROC(NG_ret_i2_plus_g1_minus_8)
    1411     +ENTRY(NG_ret_i2_plus_g1_minus_16)
    1412     + sub %g1, 16, %g1
    1413     + ba,pt %xcc, __restore_asi
    1414     + add %i2, %g1, %i0
    1415     +ENDPROC(NG_ret_i2_plus_g1_minus_16)
    1416     +ENTRY(NG_ret_i2_plus_g1_minus_24)
    1417     + sub %g1, 24, %g1
    1418     + ba,pt %xcc, __restore_asi
    1419     + add %i2, %g1, %i0
    1420     +ENDPROC(NG_ret_i2_plus_g1_minus_24)
    1421     +ENTRY(NG_ret_i2_plus_g1_minus_32)
    1422     + sub %g1, 32, %g1
    1423     + ba,pt %xcc, __restore_asi
    1424     + add %i2, %g1, %i0
    1425     +ENDPROC(NG_ret_i2_plus_g1_minus_32)
    1426     +ENTRY(NG_ret_i2_plus_g1_minus_40)
    1427     + sub %g1, 40, %g1
    1428     + ba,pt %xcc, __restore_asi
    1429     + add %i2, %g1, %i0
    1430     +ENDPROC(NG_ret_i2_plus_g1_minus_40)
    1431     +ENTRY(NG_ret_i2_plus_g1_minus_48)
    1432     + sub %g1, 48, %g1
    1433     + ba,pt %xcc, __restore_asi
    1434     + add %i2, %g1, %i0
    1435     +ENDPROC(NG_ret_i2_plus_g1_minus_48)
    1436     +ENTRY(NG_ret_i2_plus_g1_minus_56)
    1437     + sub %g1, 56, %g1
    1438     + ba,pt %xcc, __restore_asi
    1439     + add %i2, %g1, %i0
    1440     +ENDPROC(NG_ret_i2_plus_g1_minus_56)
    1441     +ENTRY(NG_ret_i2_plus_i4)
    1442     + ba,pt %xcc, __restore_asi
    1443     + add %i2, %i4, %i0
    1444     +ENDPROC(NG_ret_i2_plus_i4)
    1445     +ENTRY(NG_ret_i2_plus_i4_minus_8)
    1446     + sub %i4, 8, %i4
    1447     + ba,pt %xcc, __restore_asi
    1448     + add %i2, %i4, %i0
    1449     +ENDPROC(NG_ret_i2_plus_i4_minus_8)
    1450     +ENTRY(NG_ret_i2_plus_8)
    1451     + ba,pt %xcc, __restore_asi
    1452     + add %i2, 8, %i0
    1453     +ENDPROC(NG_ret_i2_plus_8)
    1454     +ENTRY(NG_ret_i2_plus_4)
    1455     + ba,pt %xcc, __restore_asi
    1456     + add %i2, 4, %i0
    1457     +ENDPROC(NG_ret_i2_plus_4)
    1458     +ENTRY(NG_ret_i2_plus_1)
    1459     + ba,pt %xcc, __restore_asi
    1460     + add %i2, 1, %i0
    1461     +ENDPROC(NG_ret_i2_plus_1)
    1462     +ENTRY(NG_ret_i2_plus_g1_plus_1)
    1463     + add %g1, 1, %g1
    1464     + ba,pt %xcc, __restore_asi
    1465     + add %i2, %g1, %i0
    1466     +ENDPROC(NG_ret_i2_plus_g1_plus_1)
    1467     +ENTRY(NG_ret_i2)
    1468     + ba,pt %xcc, __restore_asi
    1469     + mov %i2, %i0
    1470     +ENDPROC(NG_ret_i2)
    1471     +ENTRY(NG_ret_i2_and_7_plus_i4)
    1472     + and %i2, 7, %i2
    1473     + ba,pt %xcc, __restore_asi
    1474     + add %i2, %i4, %i0
    1475     +ENDPROC(NG_ret_i2_and_7_plus_i4)
    1476     +#endif
    1477     +
    1478     .align 64
    1479    
    1480     .globl FUNC_NAME
    1481     @@ -126,8 +209,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1482     sub %g0, %i4, %i4 ! bytes to align dst
    1483     sub %i2, %i4, %i2
    1484     1: subcc %i4, 1, %i4
    1485     - EX_LD(LOAD(ldub, %i1, %g1))
    1486     - EX_ST(STORE(stb, %g1, %o0))
    1487     + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1)
    1488     + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1)
    1489     add %i1, 1, %i1
    1490     bne,pt %XCC, 1b
    1491     add %o0, 1, %o0
    1492     @@ -160,7 +243,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1493     and %i4, 0x7, GLOBAL_SPARE
    1494     sll GLOBAL_SPARE, 3, GLOBAL_SPARE
    1495     mov 64, %i5
    1496     - EX_LD(LOAD_TWIN(%i1, %g2, %g3))
    1497     + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1)
    1498     sub %i5, GLOBAL_SPARE, %i5
    1499     mov 16, %o4
    1500     mov 32, %o5
    1501     @@ -178,31 +261,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1502     srlx WORD3, PRE_SHIFT, TMP; \
    1503     or WORD2, TMP, WORD2;
    1504    
    1505     -8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
    1506     +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
    1507     MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
    1508     LOAD(prefetch, %i1 + %i3, #one_read)
    1509    
    1510     - EX_ST(STORE_INIT(%g2, %o0 + 0x00))
    1511     - EX_ST(STORE_INIT(%g3, %o0 + 0x08))
    1512     + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1)
    1513     + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1514    
    1515     - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
    1516     + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
    1517     MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
    1518    
    1519     - EX_ST(STORE_INIT(%o2, %o0 + 0x10))
    1520     - EX_ST(STORE_INIT(%o3, %o0 + 0x18))
    1521     + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1522     + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1523    
    1524     - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1525     + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1526     MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
    1527    
    1528     - EX_ST(STORE_INIT(%g2, %o0 + 0x20))
    1529     - EX_ST(STORE_INIT(%g3, %o0 + 0x28))
    1530     + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1531     + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1532    
    1533     - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
    1534     + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
    1535     add %i1, 64, %i1
    1536     MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
    1537    
    1538     - EX_ST(STORE_INIT(%o2, %o0 + 0x30))
    1539     - EX_ST(STORE_INIT(%o3, %o0 + 0x38))
    1540     + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1541     + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1542    
    1543     subcc %g1, 64, %g1
    1544     bne,pt %XCC, 8b
    1545     @@ -211,31 +294,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1546     ba,pt %XCC, 60f
    1547     add %i1, %i4, %i1
    1548    
    1549     -9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
    1550     +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
    1551     MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
    1552     LOAD(prefetch, %i1 + %i3, #one_read)
    1553    
    1554     - EX_ST(STORE_INIT(%g3, %o0 + 0x00))
    1555     - EX_ST(STORE_INIT(%o2, %o0 + 0x08))
    1556     + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1)
    1557     + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1558    
    1559     - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
    1560     + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
    1561     MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
    1562    
    1563     - EX_ST(STORE_INIT(%o3, %o0 + 0x10))
    1564     - EX_ST(STORE_INIT(%g2, %o0 + 0x18))
    1565     + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1566     + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1567    
    1568     - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1569     + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1570     MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
    1571    
    1572     - EX_ST(STORE_INIT(%g3, %o0 + 0x20))
    1573     - EX_ST(STORE_INIT(%o2, %o0 + 0x28))
    1574     + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1575     + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1576    
    1577     - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
    1578     + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
    1579     add %i1, 64, %i1
    1580     MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
    1581    
    1582     - EX_ST(STORE_INIT(%o3, %o0 + 0x30))
    1583     - EX_ST(STORE_INIT(%g2, %o0 + 0x38))
    1584     + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1585     + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1586    
    1587     subcc %g1, 64, %g1
    1588     bne,pt %XCC, 9b
    1589     @@ -249,25 +332,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1590     * one twin load ahead, then add 8 back into source when
    1591     * we finish the loop.
    1592     */
    1593     - EX_LD(LOAD_TWIN(%i1, %o4, %o5))
    1594     + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1)
    1595     mov 16, %o7
    1596     mov 32, %g2
    1597     mov 48, %g3
    1598     mov 64, %o1
    1599     -1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1600     +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
    1601     LOAD(prefetch, %i1 + %o1, #one_read)
    1602     - EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line
    1603     - EX_ST(STORE_INIT(%o2, %o0 + 0x08))
    1604     - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
    1605     - EX_ST(STORE_INIT(%o3, %o0 + 0x10))
    1606     - EX_ST(STORE_INIT(%o4, %o0 + 0x18))
    1607     - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
    1608     - EX_ST(STORE_INIT(%o5, %o0 + 0x20))
    1609     - EX_ST(STORE_INIT(%o2, %o0 + 0x28))
    1610     - EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5))
    1611     + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line
    1612     + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1613     + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
    1614     + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1615     + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1616     + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1617     + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1618     + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1619     + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48)
    1620     add %i1, 64, %i1
    1621     - EX_ST(STORE_INIT(%o3, %o0 + 0x30))
    1622     - EX_ST(STORE_INIT(%o4, %o0 + 0x38))
    1623     + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1624     + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1625     subcc %g1, 64, %g1
    1626     bne,pt %XCC, 1b
    1627     add %o0, 64, %o0
    1628     @@ -282,20 +365,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1629     mov 32, %g2
    1630     mov 48, %g3
    1631     mov 64, %o1
    1632     -1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5))
    1633     - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1634     +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1)
    1635     + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
    1636     LOAD(prefetch, %i1 + %o1, #one_read)
    1637     - EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line
    1638     - EX_ST(STORE_INIT(%o5, %o0 + 0x08))
    1639     - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
    1640     - EX_ST(STORE_INIT(%o2, %o0 + 0x10))
    1641     - EX_ST(STORE_INIT(%o3, %o0 + 0x18))
    1642     - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
    1643     + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line
    1644     + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1645     + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
    1646     + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1647     + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1648     + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1649     add %i1, 64, %i1
    1650     - EX_ST(STORE_INIT(%o4, %o0 + 0x20))
    1651     - EX_ST(STORE_INIT(%o5, %o0 + 0x28))
    1652     - EX_ST(STORE_INIT(%o2, %o0 + 0x30))
    1653     - EX_ST(STORE_INIT(%o3, %o0 + 0x38))
    1654     + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1655     + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1656     + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1657     + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1658     subcc %g1, 64, %g1
    1659     bne,pt %XCC, 1b
    1660     add %o0, 64, %o0
    1661     @@ -321,28 +404,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1662     andn %i2, 0xf, %i4
    1663     and %i2, 0xf, %i2
    1664     1: subcc %i4, 0x10, %i4
    1665     - EX_LD(LOAD(ldx, %i1, %o4))
    1666     + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4)
    1667     add %i1, 0x08, %i1
    1668     - EX_LD(LOAD(ldx, %i1, %g1))
    1669     + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4)
    1670     sub %i1, 0x08, %i1
    1671     - EX_ST(STORE(stx, %o4, %i1 + %i3))
    1672     + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4)
    1673     add %i1, 0x8, %i1
    1674     - EX_ST(STORE(stx, %g1, %i1 + %i3))
    1675     + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8)
    1676     bgu,pt %XCC, 1b
    1677     add %i1, 0x8, %i1
    1678     73: andcc %i2, 0x8, %g0
    1679     be,pt %XCC, 1f
    1680     nop
    1681     sub %i2, 0x8, %i2
    1682     - EX_LD(LOAD(ldx, %i1, %o4))
    1683     - EX_ST(STORE(stx, %o4, %i1 + %i3))
    1684     + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8)
    1685     + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8)
    1686     add %i1, 0x8, %i1
    1687     1: andcc %i2, 0x4, %g0
    1688     be,pt %XCC, 1f
    1689     nop
    1690     sub %i2, 0x4, %i2
    1691     - EX_LD(LOAD(lduw, %i1, %i5))
    1692     - EX_ST(STORE(stw, %i5, %i1 + %i3))
    1693     + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4)
    1694     + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4)
    1695     add %i1, 0x4, %i1
    1696     1: cmp %i2, 0
    1697     be,pt %XCC, 85f
    1698     @@ -358,8 +441,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1699     sub %i2, %g1, %i2
    1700    
    1701     1: subcc %g1, 1, %g1
    1702     - EX_LD(LOAD(ldub, %i1, %i5))
    1703     - EX_ST(STORE(stb, %i5, %i1 + %i3))
    1704     + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1)
    1705     + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1)
    1706     bgu,pt %icc, 1b
    1707     add %i1, 1, %i1
    1708    
    1709     @@ -375,16 +458,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1710    
    1711     8: mov 64, %i3
    1712     andn %i1, 0x7, %i1
    1713     - EX_LD(LOAD(ldx, %i1, %g2))
    1714     + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2)
    1715     sub %i3, %g1, %i3
    1716     andn %i2, 0x7, %i4
    1717     sllx %g2, %g1, %g2
    1718     1: add %i1, 0x8, %i1
    1719     - EX_LD(LOAD(ldx, %i1, %g3))
    1720     + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4)
    1721     subcc %i4, 0x8, %i4
    1722     srlx %g3, %i3, %i5
    1723     or %i5, %g2, %i5
    1724     - EX_ST(STORE(stx, %i5, %o0))
    1725     + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4)
    1726     add %o0, 0x8, %o0
    1727     bgu,pt %icc, 1b
    1728     sllx %g3, %g1, %g2
    1729     @@ -404,8 +487,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1730    
    1731     1:
    1732     subcc %i2, 4, %i2
    1733     - EX_LD(LOAD(lduw, %i1, %g1))
    1734     - EX_ST(STORE(stw, %g1, %i1 + %i3))
    1735     + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4)
    1736     + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4)
    1737     bgu,pt %XCC, 1b
    1738     add %i1, 4, %i1
    1739    
    1740     @@ -415,8 +498,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1741     .align 32
    1742     90:
    1743     subcc %i2, 1, %i2
    1744     - EX_LD(LOAD(ldub, %i1, %g1))
    1745     - EX_ST(STORE(stb, %g1, %i1 + %i3))
    1746     + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1)
    1747     + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1)
    1748     bgu,pt %XCC, 90b
    1749     add %i1, 1, %i1
    1750     ret
    1751     diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
    1752     index ecc5692fa2b4..bb6ff73229e3 100644
    1753     --- a/arch/sparc/lib/U1copy_from_user.S
    1754     +++ b/arch/sparc/lib/U1copy_from_user.S
    1755     @@ -3,19 +3,19 @@
    1756     * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    1757     */
    1758    
    1759     -#define EX_LD(x) \
    1760     +#define EX_LD(x,y) \
    1761     98: x; \
    1762     .section __ex_table,"a";\
    1763     .align 4; \
    1764     - .word 98b, __retl_one; \
    1765     + .word 98b, y; \
    1766     .text; \
    1767     .align 4;
    1768    
    1769     -#define EX_LD_FP(x) \
    1770     +#define EX_LD_FP(x,y) \
    1771     98: x; \
    1772     .section __ex_table,"a";\
    1773     .align 4; \
    1774     - .word 98b, __retl_one_fp;\
    1775     + .word 98b, y; \
    1776     .text; \
    1777     .align 4;
    1778    
    1779     diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
    1780     index 9eea392e44d4..ed92ce739558 100644
    1781     --- a/arch/sparc/lib/U1copy_to_user.S
    1782     +++ b/arch/sparc/lib/U1copy_to_user.S
    1783     @@ -3,19 +3,19 @@
    1784     * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    1785     */
    1786    
    1787     -#define EX_ST(x) \
    1788     +#define EX_ST(x,y) \
    1789     98: x; \
    1790     .section __ex_table,"a";\
    1791     .align 4; \
    1792     - .word 98b, __retl_one; \
    1793     + .word 98b, y; \
    1794     .text; \
    1795     .align 4;
    1796    
    1797     -#define EX_ST_FP(x) \
    1798     +#define EX_ST_FP(x,y) \
    1799     98: x; \
    1800     .section __ex_table,"a";\
    1801     .align 4; \
    1802     - .word 98b, __retl_one_fp;\
    1803     + .word 98b, y; \
    1804     .text; \
    1805     .align 4;
    1806    
    1807     diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S
    1808     index 3e6209ebb7d7..f30d2ab2c371 100644
    1809     --- a/arch/sparc/lib/U1memcpy.S
    1810     +++ b/arch/sparc/lib/U1memcpy.S
    1811     @@ -5,6 +5,7 @@
    1812     */
    1813    
    1814     #ifdef __KERNEL__
    1815     +#include <linux/linkage.h>
    1816     #include <asm/visasm.h>
    1817     #include <asm/asi.h>
    1818     #define GLOBAL_SPARE g7
    1819     @@ -23,21 +24,17 @@
    1820     #endif
    1821    
    1822     #ifndef EX_LD
    1823     -#define EX_LD(x) x
    1824     +#define EX_LD(x,y) x
    1825     #endif
    1826     #ifndef EX_LD_FP
    1827     -#define EX_LD_FP(x) x
    1828     +#define EX_LD_FP(x,y) x
    1829     #endif
    1830    
    1831     #ifndef EX_ST
    1832     -#define EX_ST(x) x
    1833     +#define EX_ST(x,y) x
    1834     #endif
    1835     #ifndef EX_ST_FP
    1836     -#define EX_ST_FP(x) x
    1837     -#endif
    1838     -
    1839     -#ifndef EX_RETVAL
    1840     -#define EX_RETVAL(x) x
    1841     +#define EX_ST_FP(x,y) x
    1842     #endif
    1843    
    1844     #ifndef LOAD
    1845     @@ -78,53 +75,169 @@
    1846     faligndata %f7, %f8, %f60; \
    1847     faligndata %f8, %f9, %f62;
    1848    
    1849     -#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
    1850     - EX_LD_FP(LOAD_BLK(%src, %fdest)); \
    1851     - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    1852     - add %src, 0x40, %src; \
    1853     - subcc %len, 0x40, %len; \
    1854     - be,pn %xcc, jmptgt; \
    1855     - add %dest, 0x40, %dest; \
    1856     -
    1857     -#define LOOP_CHUNK1(src, dest, len, branch_dest) \
    1858     - MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
    1859     -#define LOOP_CHUNK2(src, dest, len, branch_dest) \
    1860     - MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
    1861     -#define LOOP_CHUNK3(src, dest, len, branch_dest) \
    1862     - MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
    1863     +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \
    1864     + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \
    1865     + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
    1866     + add %src, 0x40, %src; \
    1867     + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \
    1868     + be,pn %xcc, jmptgt; \
    1869     + add %dest, 0x40, %dest; \
    1870     +
    1871     +#define LOOP_CHUNK1(src, dest, branch_dest) \
    1872     + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest)
    1873     +#define LOOP_CHUNK2(src, dest, branch_dest) \
    1874     + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest)
    1875     +#define LOOP_CHUNK3(src, dest, branch_dest) \
    1876     + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest)
    1877    
    1878     #define DO_SYNC membar #Sync;
    1879     #define STORE_SYNC(dest, fsrc) \
    1880     - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    1881     + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
    1882     add %dest, 0x40, %dest; \
    1883     DO_SYNC
    1884    
    1885     #define STORE_JUMP(dest, fsrc, target) \
    1886     - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    1887     + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \
    1888     add %dest, 0x40, %dest; \
    1889     ba,pt %xcc, target; \
    1890     nop;
    1891    
    1892     -#define FINISH_VISCHUNK(dest, f0, f1, left) \
    1893     - subcc %left, 8, %left;\
    1894     - bl,pn %xcc, 95f; \
    1895     - faligndata %f0, %f1, %f48; \
    1896     - EX_ST_FP(STORE(std, %f48, %dest)); \
    1897     +#define FINISH_VISCHUNK(dest, f0, f1) \
    1898     + subcc %g3, 8, %g3; \
    1899     + bl,pn %xcc, 95f; \
    1900     + faligndata %f0, %f1, %f48; \
    1901     + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \
    1902     add %dest, 8, %dest;
    1903    
    1904     -#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
    1905     - subcc %left, 8, %left; \
    1906     - bl,pn %xcc, 95f; \
    1907     +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
    1908     + subcc %g3, 8, %g3; \
    1909     + bl,pn %xcc, 95f; \
    1910     fsrc2 %f0, %f1;
    1911    
    1912     -#define UNEVEN_VISCHUNK(dest, f0, f1, left) \
    1913     - UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
    1914     +#define UNEVEN_VISCHUNK(dest, f0, f1) \
    1915     + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
    1916     ba,a,pt %xcc, 93f;
    1917    
    1918     .register %g2,#scratch
    1919     .register %g3,#scratch
    1920    
    1921     .text
    1922     +#ifndef EX_RETVAL
    1923     +#define EX_RETVAL(x) x
    1924     +ENTRY(U1_g1_1_fp)
    1925     + VISExitHalf
    1926     + add %g1, 1, %g1
    1927     + add %g1, %g2, %g1
    1928     + retl
    1929     + add %g1, %o2, %o0
    1930     +ENDPROC(U1_g1_1_fp)
    1931     +ENTRY(U1_g2_0_fp)
    1932     + VISExitHalf
    1933     + retl
    1934     + add %g2, %o2, %o0
    1935     +ENDPROC(U1_g2_0_fp)
    1936     +ENTRY(U1_g2_8_fp)
    1937     + VISExitHalf
    1938     + add %g2, 8, %g2
    1939     + retl
    1940     + add %g2, %o2, %o0
    1941     +ENDPROC(U1_g2_8_fp)
    1942     +ENTRY(U1_gs_0_fp)
    1943     + VISExitHalf
    1944     + add %GLOBAL_SPARE, %g3, %o0
    1945     + retl
    1946     + add %o0, %o2, %o0
    1947     +ENDPROC(U1_gs_0_fp)
    1948     +ENTRY(U1_gs_80_fp)
    1949     + VISExitHalf
    1950     + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
    1951     + add %GLOBAL_SPARE, %g3, %o0
    1952     + retl
    1953     + add %o0, %o2, %o0
    1954     +ENDPROC(U1_gs_80_fp)
    1955     +ENTRY(U1_gs_40_fp)
    1956     + VISExitHalf
    1957     + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE
    1958     + add %GLOBAL_SPARE, %g3, %o0
    1959     + retl
    1960     + add %o0, %o2, %o0
    1961     +ENDPROC(U1_gs_40_fp)
    1962     +ENTRY(U1_g3_0_fp)
    1963     + VISExitHalf
    1964     + retl
    1965     + add %g3, %o2, %o0
    1966     +ENDPROC(U1_g3_0_fp)
    1967     +ENTRY(U1_g3_8_fp)
    1968     + VISExitHalf
    1969     + add %g3, 8, %g3
    1970     + retl
    1971     + add %g3, %o2, %o0
    1972     +ENDPROC(U1_g3_8_fp)
    1973     +ENTRY(U1_o2_0_fp)
    1974     + VISExitHalf
    1975     + retl
    1976     + mov %o2, %o0
    1977     +ENDPROC(U1_o2_0_fp)
    1978     +ENTRY(U1_o2_1_fp)
    1979     + VISExitHalf
    1980     + retl
    1981     + add %o2, 1, %o0
    1982     +ENDPROC(U1_o2_1_fp)
    1983     +ENTRY(U1_gs_0)
    1984     + VISExitHalf
    1985     + retl
    1986     + add %GLOBAL_SPARE, %o2, %o0
    1987     +ENDPROC(U1_gs_0)
    1988     +ENTRY(U1_gs_8)
    1989     + VISExitHalf
    1990     + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
    1991     + retl
    1992     + add %GLOBAL_SPARE, 0x8, %o0
    1993     +ENDPROC(U1_gs_8)
    1994     +ENTRY(U1_gs_10)
    1995     + VISExitHalf
    1996     + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
    1997     + retl
    1998     + add %GLOBAL_SPARE, 0x10, %o0
    1999     +ENDPROC(U1_gs_10)
    2000     +ENTRY(U1_o2_0)
    2001     + retl
    2002     + mov %o2, %o0
    2003     +ENDPROC(U1_o2_0)
    2004     +ENTRY(U1_o2_8)
    2005     + retl
    2006     + add %o2, 8, %o0
    2007     +ENDPROC(U1_o2_8)
    2008     +ENTRY(U1_o2_4)
    2009     + retl
    2010     + add %o2, 4, %o0
    2011     +ENDPROC(U1_o2_4)
    2012     +ENTRY(U1_o2_1)
    2013     + retl
    2014     + add %o2, 1, %o0
    2015     +ENDPROC(U1_o2_1)
    2016     +ENTRY(U1_g1_0)
    2017     + retl
    2018     + add %g1, %o2, %o0
    2019     +ENDPROC(U1_g1_0)
    2020     +ENTRY(U1_g1_1)
    2021     + add %g1, 1, %g1
    2022     + retl
    2023     + add %g1, %o2, %o0
    2024     +ENDPROC(U1_g1_1)
    2025     +ENTRY(U1_gs_0_o2_adj)
    2026     + and %o2, 7, %o2
    2027     + retl
    2028     + add %GLOBAL_SPARE, %o2, %o0
    2029     +ENDPROC(U1_gs_0_o2_adj)
    2030     +ENTRY(U1_gs_8_o2_adj)
    2031     + and %o2, 7, %o2
    2032     + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE
    2033     + retl
    2034     + add %GLOBAL_SPARE, %o2, %o0
    2035     +ENDPROC(U1_gs_8_o2_adj)
    2036     +#endif
    2037     +
    2038     .align 64
    2039    
    2040     .globl FUNC_NAME
    2041     @@ -166,8 +279,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2042     and %g2, 0x38, %g2
    2043    
    2044     1: subcc %g1, 0x1, %g1
    2045     - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
    2046     - EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE))
    2047     + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp)
    2048     + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp)
    2049     bgu,pt %XCC, 1b
    2050     add %o1, 0x1, %o1
    2051    
    2052     @@ -178,20 +291,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2053     be,pt %icc, 3f
    2054     alignaddr %o1, %g0, %o1
    2055    
    2056     - EX_LD_FP(LOAD(ldd, %o1, %f4))
    2057     -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
    2058     + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp)
    2059     +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp)
    2060     add %o1, 0x8, %o1
    2061     subcc %g2, 0x8, %g2
    2062     faligndata %f4, %f6, %f0
    2063     - EX_ST_FP(STORE(std, %f0, %o0))
    2064     + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
    2065     be,pn %icc, 3f
    2066     add %o0, 0x8, %o0
    2067    
    2068     - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
    2069     + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp)
    2070     add %o1, 0x8, %o1
    2071     subcc %g2, 0x8, %g2
    2072     faligndata %f6, %f4, %f0
    2073     - EX_ST_FP(STORE(std, %f0, %o0))
    2074     + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
    2075     bne,pt %icc, 1b
    2076     add %o0, 0x8, %o0
    2077    
    2078     @@ -214,13 +327,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2079     add %g1, %GLOBAL_SPARE, %g1
    2080     subcc %o2, %g3, %o2
    2081    
    2082     - EX_LD_FP(LOAD_BLK(%o1, %f0))
    2083     + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp)
    2084     add %o1, 0x40, %o1
    2085     add %g1, %g3, %g1
    2086     - EX_LD_FP(LOAD_BLK(%o1, %f16))
    2087     + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp)
    2088     add %o1, 0x40, %o1
    2089     sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
    2090     - EX_LD_FP(LOAD_BLK(%o1, %f32))
    2091     + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp)
    2092     add %o1, 0x40, %o1
    2093    
    2094     /* There are 8 instances of the unrolled loop,
    2095     @@ -240,11 +353,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2096    
    2097     .align 64
    2098     1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
    2099     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2100     + LOOP_CHUNK1(o1, o0, 1f)
    2101     FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
    2102     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2103     + LOOP_CHUNK2(o1, o0, 2f)
    2104     FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
    2105     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2106     + LOOP_CHUNK3(o1, o0, 3f)
    2107     ba,pt %xcc, 1b+4
    2108     faligndata %f0, %f2, %f48
    2109     1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
    2110     @@ -261,11 +374,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2111     STORE_JUMP(o0, f48, 56f)
    2112    
    2113     1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
    2114     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2115     + LOOP_CHUNK1(o1, o0, 1f)
    2116     FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
    2117     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2118     + LOOP_CHUNK2(o1, o0, 2f)
    2119     FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
    2120     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2121     + LOOP_CHUNK3(o1, o0, 3f)
    2122     ba,pt %xcc, 1b+4
    2123     faligndata %f2, %f4, %f48
    2124     1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
    2125     @@ -282,11 +395,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2126     STORE_JUMP(o0, f48, 57f)
    2127    
    2128     1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
    2129     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2130     + LOOP_CHUNK1(o1, o0, 1f)
    2131     FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
    2132     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2133     + LOOP_CHUNK2(o1, o0, 2f)
    2134     FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
    2135     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2136     + LOOP_CHUNK3(o1, o0, 3f)
    2137     ba,pt %xcc, 1b+4
    2138     faligndata %f4, %f6, %f48
    2139     1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
    2140     @@ -303,11 +416,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2141     STORE_JUMP(o0, f48, 58f)
    2142    
    2143     1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
    2144     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2145     + LOOP_CHUNK1(o1, o0, 1f)
    2146     FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
    2147     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2148     + LOOP_CHUNK2(o1, o0, 2f)
    2149     FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
    2150     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2151     + LOOP_CHUNK3(o1, o0, 3f)
    2152     ba,pt %xcc, 1b+4
    2153     faligndata %f6, %f8, %f48
    2154     1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
    2155     @@ -324,11 +437,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2156     STORE_JUMP(o0, f48, 59f)
    2157    
    2158     1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
    2159     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2160     + LOOP_CHUNK1(o1, o0, 1f)
    2161     FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
    2162     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2163     + LOOP_CHUNK2(o1, o0, 2f)
    2164     FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
    2165     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2166     + LOOP_CHUNK3(o1, o0, 3f)
    2167     ba,pt %xcc, 1b+4
    2168     faligndata %f8, %f10, %f48
    2169     1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
    2170     @@ -345,11 +458,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2171     STORE_JUMP(o0, f48, 60f)
    2172    
    2173     1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
    2174     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2175     + LOOP_CHUNK1(o1, o0, 1f)
    2176     FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
    2177     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2178     + LOOP_CHUNK2(o1, o0, 2f)
    2179     FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
    2180     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2181     + LOOP_CHUNK3(o1, o0, 3f)
    2182     ba,pt %xcc, 1b+4
    2183     faligndata %f10, %f12, %f48
    2184     1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
    2185     @@ -366,11 +479,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2186     STORE_JUMP(o0, f48, 61f)
    2187    
    2188     1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
    2189     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2190     + LOOP_CHUNK1(o1, o0, 1f)
    2191     FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
    2192     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2193     + LOOP_CHUNK2(o1, o0, 2f)
    2194     FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
    2195     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2196     + LOOP_CHUNK3(o1, o0, 3f)
    2197     ba,pt %xcc, 1b+4
    2198     faligndata %f12, %f14, %f48
    2199     1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
    2200     @@ -387,11 +500,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2201     STORE_JUMP(o0, f48, 62f)
    2202    
    2203     1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
    2204     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2205     + LOOP_CHUNK1(o1, o0, 1f)
    2206     FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2207     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2208     + LOOP_CHUNK2(o1, o0, 2f)
    2209     FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
    2210     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2211     + LOOP_CHUNK3(o1, o0, 3f)
    2212     ba,pt %xcc, 1b+4
    2213     faligndata %f14, %f16, %f48
    2214     1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2215     @@ -407,53 +520,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2216     FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2217     STORE_JUMP(o0, f48, 63f)
    2218    
    2219     -40: FINISH_VISCHUNK(o0, f0, f2, g3)
    2220     -41: FINISH_VISCHUNK(o0, f2, f4, g3)
    2221     -42: FINISH_VISCHUNK(o0, f4, f6, g3)
    2222     -43: FINISH_VISCHUNK(o0, f6, f8, g3)
    2223     -44: FINISH_VISCHUNK(o0, f8, f10, g3)
    2224     -45: FINISH_VISCHUNK(o0, f10, f12, g3)
    2225     -46: FINISH_VISCHUNK(o0, f12, f14, g3)
    2226     -47: UNEVEN_VISCHUNK(o0, f14, f0, g3)
    2227     -48: FINISH_VISCHUNK(o0, f16, f18, g3)
    2228     -49: FINISH_VISCHUNK(o0, f18, f20, g3)
    2229     -50: FINISH_VISCHUNK(o0, f20, f22, g3)
    2230     -51: FINISH_VISCHUNK(o0, f22, f24, g3)
    2231     -52: FINISH_VISCHUNK(o0, f24, f26, g3)
    2232     -53: FINISH_VISCHUNK(o0, f26, f28, g3)
    2233     -54: FINISH_VISCHUNK(o0, f28, f30, g3)
    2234     -55: UNEVEN_VISCHUNK(o0, f30, f0, g3)
    2235     -56: FINISH_VISCHUNK(o0, f32, f34, g3)
    2236     -57: FINISH_VISCHUNK(o0, f34, f36, g3)
    2237     -58: FINISH_VISCHUNK(o0, f36, f38, g3)
    2238     -59: FINISH_VISCHUNK(o0, f38, f40, g3)
    2239     -60: FINISH_VISCHUNK(o0, f40, f42, g3)
    2240     -61: FINISH_VISCHUNK(o0, f42, f44, g3)
    2241     -62: FINISH_VISCHUNK(o0, f44, f46, g3)
    2242     -63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
    2243     -
    2244     -93: EX_LD_FP(LOAD(ldd, %o1, %f2))
    2245     +40: FINISH_VISCHUNK(o0, f0, f2)
    2246     +41: FINISH_VISCHUNK(o0, f2, f4)
    2247     +42: FINISH_VISCHUNK(o0, f4, f6)
    2248     +43: FINISH_VISCHUNK(o0, f6, f8)
    2249     +44: FINISH_VISCHUNK(o0, f8, f10)
    2250     +45: FINISH_VISCHUNK(o0, f10, f12)
    2251     +46: FINISH_VISCHUNK(o0, f12, f14)
    2252     +47: UNEVEN_VISCHUNK(o0, f14, f0)
    2253     +48: FINISH_VISCHUNK(o0, f16, f18)
    2254     +49: FINISH_VISCHUNK(o0, f18, f20)
    2255     +50: FINISH_VISCHUNK(o0, f20, f22)
    2256     +51: FINISH_VISCHUNK(o0, f22, f24)
    2257     +52: FINISH_VISCHUNK(o0, f24, f26)
    2258     +53: FINISH_VISCHUNK(o0, f26, f28)
    2259     +54: FINISH_VISCHUNK(o0, f28, f30)
    2260     +55: UNEVEN_VISCHUNK(o0, f30, f0)
    2261     +56: FINISH_VISCHUNK(o0, f32, f34)
    2262     +57: FINISH_VISCHUNK(o0, f34, f36)
    2263     +58: FINISH_VISCHUNK(o0, f36, f38)
    2264     +59: FINISH_VISCHUNK(o0, f38, f40)
    2265     +60: FINISH_VISCHUNK(o0, f40, f42)
    2266     +61: FINISH_VISCHUNK(o0, f42, f44)
    2267     +62: FINISH_VISCHUNK(o0, f44, f46)
    2268     +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0)
    2269     +
    2270     +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp)
    2271     add %o1, 8, %o1
    2272     subcc %g3, 8, %g3
    2273     faligndata %f0, %f2, %f8
    2274     - EX_ST_FP(STORE(std, %f8, %o0))
    2275     + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
    2276     bl,pn %xcc, 95f
    2277     add %o0, 8, %o0
    2278     - EX_LD_FP(LOAD(ldd, %o1, %f0))
    2279     + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp)
    2280     add %o1, 8, %o1
    2281     subcc %g3, 8, %g3
    2282     faligndata %f2, %f0, %f8
    2283     - EX_ST_FP(STORE(std, %f8, %o0))
    2284     + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
    2285     bge,pt %xcc, 93b
    2286     add %o0, 8, %o0
    2287    
    2288     95: brz,pt %o2, 2f
    2289     mov %g1, %o1
    2290    
    2291     -1: EX_LD_FP(LOAD(ldub, %o1, %o3))
    2292     +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp)
    2293     add %o1, 1, %o1
    2294     subcc %o2, 1, %o2
    2295     - EX_ST_FP(STORE(stb, %o3, %o0))
    2296     + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp)
    2297     bne,pt %xcc, 1b
    2298     add %o0, 1, %o0
    2299    
    2300     @@ -469,27 +582,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2301    
    2302     72: andn %o2, 0xf, %GLOBAL_SPARE
    2303     and %o2, 0xf, %o2
    2304     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
    2305     - EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
    2306     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0)
    2307     + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0)
    2308     subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
    2309     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2310     + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10)
    2311     add %o1, 0x8, %o1
    2312     - EX_ST(STORE(stx, %g1, %o1 + %o3))
    2313     + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8)
    2314     bgu,pt %XCC, 1b
    2315     add %o1, 0x8, %o1
    2316     73: andcc %o2, 0x8, %g0
    2317     be,pt %XCC, 1f
    2318     nop
    2319     - EX_LD(LOAD(ldx, %o1, %o5))
    2320     + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0)
    2321     sub %o2, 0x8, %o2
    2322     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2323     + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8)
    2324     add %o1, 0x8, %o1
    2325     1: andcc %o2, 0x4, %g0
    2326     be,pt %XCC, 1f
    2327     nop
    2328     - EX_LD(LOAD(lduw, %o1, %o5))
    2329     + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0)
    2330     sub %o2, 0x4, %o2
    2331     - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2332     + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4)
    2333     add %o1, 0x4, %o1
    2334     1: cmp %o2, 0
    2335     be,pt %XCC, 85f
    2336     @@ -503,9 +616,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2337     sub %g0, %g1, %g1
    2338     sub %o2, %g1, %o2
    2339    
    2340     -1: EX_LD(LOAD(ldub, %o1, %o5))
    2341     +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0)
    2342     subcc %g1, 1, %g1
    2343     - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2344     + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1)
    2345     bgu,pt %icc, 1b
    2346     add %o1, 1, %o1
    2347    
    2348     @@ -521,16 +634,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2349    
    2350     8: mov 64, %o3
    2351     andn %o1, 0x7, %o1
    2352     - EX_LD(LOAD(ldx, %o1, %g2))
    2353     + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0)
    2354     sub %o3, %g1, %o3
    2355     andn %o2, 0x7, %GLOBAL_SPARE
    2356     sllx %g2, %g1, %g2
    2357     -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
    2358     +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj)
    2359     subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
    2360     add %o1, 0x8, %o1
    2361     srlx %g3, %o3, %o5
    2362     or %o5, %g2, %o5
    2363     - EX_ST(STORE(stx, %o5, %o0))
    2364     + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj)
    2365     add %o0, 0x8, %o0
    2366     bgu,pt %icc, 1b
    2367     sllx %g3, %g1, %g2
    2368     @@ -548,9 +661,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2369     bne,pn %XCC, 90f
    2370     sub %o0, %o1, %o3
    2371    
    2372     -1: EX_LD(LOAD(lduw, %o1, %g1))
    2373     +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0)
    2374     subcc %o2, 4, %o2
    2375     - EX_ST(STORE(stw, %g1, %o1 + %o3))
    2376     + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4)
    2377     bgu,pt %XCC, 1b
    2378     add %o1, 4, %o1
    2379    
    2380     @@ -558,9 +671,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2381     mov EX_RETVAL(%o4), %o0
    2382    
    2383     .align 32
    2384     -90: EX_LD(LOAD(ldub, %o1, %g1))
    2385     +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0)
    2386     subcc %o2, 1, %o2
    2387     - EX_ST(STORE(stb, %g1, %o1 + %o3))
    2388     + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1)
    2389     bgu,pt %XCC, 90b
    2390     add %o1, 1, %o1
    2391     retl
    2392     diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S
    2393     index 88ad73d86fe4..db73010a1af8 100644
    2394     --- a/arch/sparc/lib/U3copy_from_user.S
    2395     +++ b/arch/sparc/lib/U3copy_from_user.S
    2396     @@ -3,19 +3,19 @@
    2397     * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    2398     */
    2399    
    2400     -#define EX_LD(x) \
    2401     +#define EX_LD(x,y) \
    2402     98: x; \
    2403     .section __ex_table,"a";\
    2404     .align 4; \
    2405     - .word 98b, __retl_one; \
    2406     + .word 98b, y; \
    2407     .text; \
    2408     .align 4;
    2409    
    2410     -#define EX_LD_FP(x) \
    2411     +#define EX_LD_FP(x,y) \
    2412     98: x; \
    2413     .section __ex_table,"a";\
    2414     .align 4; \
    2415     - .word 98b, __retl_one_fp;\
    2416     + .word 98b, y##_fp; \
    2417     .text; \
    2418     .align 4;
    2419    
    2420     diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
    2421     index 845139d75537..c4ee858e352a 100644
    2422     --- a/arch/sparc/lib/U3copy_to_user.S
    2423     +++ b/arch/sparc/lib/U3copy_to_user.S
    2424     @@ -3,19 +3,19 @@
    2425     * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    2426     */
    2427    
    2428     -#define EX_ST(x) \
    2429     +#define EX_ST(x,y) \
    2430     98: x; \
    2431     .section __ex_table,"a";\
    2432     .align 4; \
    2433     - .word 98b, __retl_one; \
    2434     + .word 98b, y; \
    2435     .text; \
    2436     .align 4;
    2437    
    2438     -#define EX_ST_FP(x) \
    2439     +#define EX_ST_FP(x,y) \
    2440     98: x; \
    2441     .section __ex_table,"a";\
    2442     .align 4; \
    2443     - .word 98b, __retl_one_fp;\
    2444     + .word 98b, y##_fp; \
    2445     .text; \
    2446     .align 4;
    2447    
    2448     diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S
    2449     index 491ee69e4995..54f98706b03b 100644
    2450     --- a/arch/sparc/lib/U3memcpy.S
    2451     +++ b/arch/sparc/lib/U3memcpy.S
    2452     @@ -4,6 +4,7 @@
    2453     */
    2454    
    2455     #ifdef __KERNEL__
    2456     +#include <linux/linkage.h>
    2457     #include <asm/visasm.h>
    2458     #include <asm/asi.h>
    2459     #define GLOBAL_SPARE %g7
    2460     @@ -22,21 +23,17 @@
    2461     #endif
    2462    
    2463     #ifndef EX_LD
    2464     -#define EX_LD(x) x
    2465     +#define EX_LD(x,y) x
    2466     #endif
    2467     #ifndef EX_LD_FP
    2468     -#define EX_LD_FP(x) x
    2469     +#define EX_LD_FP(x,y) x
    2470     #endif
    2471    
    2472     #ifndef EX_ST
    2473     -#define EX_ST(x) x
    2474     +#define EX_ST(x,y) x
    2475     #endif
    2476     #ifndef EX_ST_FP
    2477     -#define EX_ST_FP(x) x
    2478     -#endif
    2479     -
    2480     -#ifndef EX_RETVAL
    2481     -#define EX_RETVAL(x) x
    2482     +#define EX_ST_FP(x,y) x
    2483     #endif
    2484    
    2485     #ifndef LOAD
    2486     @@ -77,6 +74,87 @@
    2487     */
    2488    
    2489     .text
    2490     +#ifndef EX_RETVAL
    2491     +#define EX_RETVAL(x) x
    2492     +__restore_fp:
    2493     + VISExitHalf
    2494     + retl
    2495     + nop
    2496     +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
    2497     + add %g1, 1, %g1
    2498     + add %g2, %g1, %g2
    2499     + ba,pt %xcc, __restore_fp
    2500     + add %o2, %g2, %o0
    2501     +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
    2502     +ENTRY(U3_retl_o2_plus_g2_fp)
    2503     + ba,pt %xcc, __restore_fp
    2504     + add %o2, %g2, %o0
    2505     +ENDPROC(U3_retl_o2_plus_g2_fp)
    2506     +ENTRY(U3_retl_o2_plus_g2_plus_8_fp)
    2507     + add %g2, 8, %g2
    2508     + ba,pt %xcc, __restore_fp
    2509     + add %o2, %g2, %o0
    2510     +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp)
    2511     +ENTRY(U3_retl_o2)
    2512     + retl
    2513     + mov %o2, %o0
    2514     +ENDPROC(U3_retl_o2)
    2515     +ENTRY(U3_retl_o2_plus_1)
    2516     + retl
    2517     + add %o2, 1, %o0
    2518     +ENDPROC(U3_retl_o2_plus_1)
    2519     +ENTRY(U3_retl_o2_plus_4)
    2520     + retl
    2521     + add %o2, 4, %o0
    2522     +ENDPROC(U3_retl_o2_plus_4)
    2523     +ENTRY(U3_retl_o2_plus_8)
    2524     + retl
    2525     + add %o2, 8, %o0
    2526     +ENDPROC(U3_retl_o2_plus_8)
    2527     +ENTRY(U3_retl_o2_plus_g1_plus_1)
    2528     + add %g1, 1, %g1
    2529     + retl
    2530     + add %o2, %g1, %o0
    2531     +ENDPROC(U3_retl_o2_plus_g1_plus_1)
    2532     +ENTRY(U3_retl_o2_fp)
    2533     + ba,pt %xcc, __restore_fp
    2534     + mov %o2, %o0
    2535     +ENDPROC(U3_retl_o2_fp)
    2536     +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
    2537     + sll %o3, 6, %o3
    2538     + add %o3, 0x80, %o3
    2539     + ba,pt %xcc, __restore_fp
    2540     + add %o2, %o3, %o0
    2541     +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
    2542     +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
    2543     + sll %o3, 6, %o3
    2544     + add %o3, 0x40, %o3
    2545     + ba,pt %xcc, __restore_fp
    2546     + add %o2, %o3, %o0
    2547     +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
    2548     +ENTRY(U3_retl_o2_plus_GS_plus_0x10)
    2549     + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE
    2550     + retl
    2551     + add %o2, GLOBAL_SPARE, %o0
    2552     +ENDPROC(U3_retl_o2_plus_GS_plus_0x10)
    2553     +ENTRY(U3_retl_o2_plus_GS_plus_0x08)
    2554     + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE
    2555     + retl
    2556     + add %o2, GLOBAL_SPARE, %o0
    2557     +ENDPROC(U3_retl_o2_plus_GS_plus_0x08)
    2558     +ENTRY(U3_retl_o2_and_7_plus_GS)
    2559     + and %o2, 7, %o2
    2560     + retl
    2561     + add %o2, GLOBAL_SPARE, %o2
    2562     +ENDPROC(U3_retl_o2_and_7_plus_GS)
    2563     +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8)
    2564     + add GLOBAL_SPARE, 8, GLOBAL_SPARE
    2565     + and %o2, 7, %o2
    2566     + retl
    2567     + add %o2, GLOBAL_SPARE, %o2
    2568     +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8)
    2569     +#endif
    2570     +
    2571     .align 64
    2572    
    2573     /* The cheetah's flexible spine, oversized liver, enlarged heart,
    2574     @@ -126,8 +204,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2575     and %g2, 0x38, %g2
    2576    
    2577     1: subcc %g1, 0x1, %g1
    2578     - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
    2579     - EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
    2580     + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1)
    2581     + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1)
    2582     bgu,pt %XCC, 1b
    2583     add %o1, 0x1, %o1
    2584    
    2585     @@ -138,20 +216,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2586     be,pt %icc, 3f
    2587     alignaddr %o1, %g0, %o1
    2588    
    2589     - EX_LD_FP(LOAD(ldd, %o1, %f4))
    2590     -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
    2591     + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2)
    2592     +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2)
    2593     add %o1, 0x8, %o1
    2594     subcc %g2, 0x8, %g2
    2595     faligndata %f4, %f6, %f0
    2596     - EX_ST_FP(STORE(std, %f0, %o0))
    2597     + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8)
    2598     be,pn %icc, 3f
    2599     add %o0, 0x8, %o0
    2600    
    2601     - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
    2602     + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2)
    2603     add %o1, 0x8, %o1
    2604     subcc %g2, 0x8, %g2
    2605     faligndata %f6, %f4, %f2
    2606     - EX_ST_FP(STORE(std, %f2, %o0))
    2607     + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8)
    2608     bne,pt %icc, 1b
    2609     add %o0, 0x8, %o0
    2610    
    2611     @@ -161,25 +239,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2612     LOAD(prefetch, %o1 + 0x080, #one_read)
    2613     LOAD(prefetch, %o1 + 0x0c0, #one_read)
    2614     LOAD(prefetch, %o1 + 0x100, #one_read)
    2615     - EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0))
    2616     + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2)
    2617     LOAD(prefetch, %o1 + 0x140, #one_read)
    2618     - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2619     + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2)
    2620     LOAD(prefetch, %o1 + 0x180, #one_read)
    2621     - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2622     + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2)
    2623     LOAD(prefetch, %o1 + 0x1c0, #one_read)
    2624     faligndata %f0, %f2, %f16
    2625     - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2626     + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2)
    2627     faligndata %f2, %f4, %f18
    2628     - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2629     + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2)
    2630     faligndata %f4, %f6, %f20
    2631     - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2632     + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2)
    2633     faligndata %f6, %f8, %f22
    2634    
    2635     - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2636     + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2)
    2637     faligndata %f8, %f10, %f24
    2638     - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2639     + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2)
    2640     faligndata %f10, %f12, %f26
    2641     - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2642     + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2)
    2643    
    2644     subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE
    2645     add %o1, 0x40, %o1
    2646     @@ -190,26 +268,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2647    
    2648     .align 64
    2649     1:
    2650     - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2651     + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2652     faligndata %f12, %f14, %f28
    2653     - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2654     + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2655     faligndata %f14, %f0, %f30
    2656     - EX_ST_FP(STORE_BLK(%f16, %o0))
    2657     - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2658     + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2659     + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2660     faligndata %f0, %f2, %f16
    2661     add %o0, 0x40, %o0
    2662    
    2663     - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2664     + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2665     faligndata %f2, %f4, %f18
    2666     - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2667     + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2668     faligndata %f4, %f6, %f20
    2669     - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2670     + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2671     subcc %o3, 0x01, %o3
    2672     faligndata %f6, %f8, %f22
    2673     - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2674     + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2675    
    2676     faligndata %f8, %f10, %f24
    2677     - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2678     + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2679     LOAD(prefetch, %o1 + 0x1c0, #one_read)
    2680     faligndata %f10, %f12, %f26
    2681     bg,pt %XCC, 1b
    2682     @@ -217,29 +295,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2683    
    2684     /* Finally we copy the last full 64-byte block. */
    2685     2:
    2686     - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2687     + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2688     faligndata %f12, %f14, %f28
    2689     - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2690     + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2691     faligndata %f14, %f0, %f30
    2692     - EX_ST_FP(STORE_BLK(%f16, %o0))
    2693     - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2694     + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2695     + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2696     faligndata %f0, %f2, %f16
    2697     - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2698     + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2699     faligndata %f2, %f4, %f18
    2700     - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2701     + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2702     faligndata %f4, %f6, %f20
    2703     - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2704     + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2705     faligndata %f6, %f8, %f22
    2706     - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2707     + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2708     faligndata %f8, %f10, %f24
    2709     cmp %g1, 0
    2710     be,pt %XCC, 1f
    2711     add %o0, 0x40, %o0
    2712     - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2713     + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2714     1: faligndata %f10, %f12, %f26
    2715     faligndata %f12, %f14, %f28
    2716     faligndata %f14, %f0, %f30
    2717     - EX_ST_FP(STORE_BLK(%f16, %o0))
    2718     + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2719     add %o0, 0x40, %o0
    2720     add %o1, 0x40, %o1
    2721     membar #Sync
    2722     @@ -259,20 +337,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2723    
    2724     sub %o2, %g2, %o2
    2725     be,a,pt %XCC, 1f
    2726     - EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0))
    2727     + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2)
    2728    
    2729     -1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2))
    2730     +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2)
    2731     add %o1, 0x8, %o1
    2732     subcc %g2, 0x8, %g2
    2733     faligndata %f0, %f2, %f8
    2734     - EX_ST_FP(STORE(std, %f8, %o0))
    2735     + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
    2736     be,pn %XCC, 2f
    2737     add %o0, 0x8, %o0
    2738     - EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0))
    2739     + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2)
    2740     add %o1, 0x8, %o1
    2741     subcc %g2, 0x8, %g2
    2742     faligndata %f2, %f0, %f8
    2743     - EX_ST_FP(STORE(std, %f8, %o0))
    2744     + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
    2745     bne,pn %XCC, 1b
    2746     add %o0, 0x8, %o0
    2747    
    2748     @@ -292,30 +370,33 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2749     andcc %o2, 0x8, %g0
    2750     be,pt %icc, 1f
    2751     nop
    2752     - EX_LD(LOAD(ldx, %o1, %o5))
    2753     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2754     + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2)
    2755     + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2)
    2756     add %o1, 0x8, %o1
    2757     + sub %o2, 8, %o2
    2758    
    2759     1: andcc %o2, 0x4, %g0
    2760     be,pt %icc, 1f
    2761     nop
    2762     - EX_LD(LOAD(lduw, %o1, %o5))
    2763     - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2764     + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2)
    2765     + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2)
    2766     add %o1, 0x4, %o1
    2767     + sub %o2, 4, %o2
    2768    
    2769     1: andcc %o2, 0x2, %g0
    2770     be,pt %icc, 1f
    2771     nop
    2772     - EX_LD(LOAD(lduh, %o1, %o5))
    2773     - EX_ST(STORE(sth, %o5, %o1 + %o3))
    2774     + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2)
    2775     + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2)
    2776     add %o1, 0x2, %o1
    2777     + sub %o2, 2, %o2
    2778    
    2779     1: andcc %o2, 0x1, %g0
    2780     be,pt %icc, 85f
    2781     nop
    2782     - EX_LD(LOAD(ldub, %o1, %o5))
    2783     + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2)
    2784     ba,pt %xcc, 85f
    2785     - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2786     + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2)
    2787    
    2788     .align 64
    2789     70: /* 16 < len <= 64 */
    2790     @@ -326,26 +407,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2791     andn %o2, 0xf, GLOBAL_SPARE
    2792     and %o2, 0xf, %o2
    2793     1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE
    2794     - EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
    2795     - EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
    2796     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2797     + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10)
    2798     + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10)
    2799     + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10)
    2800     add %o1, 0x8, %o1
    2801     - EX_ST(STORE(stx, %g1, %o1 + %o3))
    2802     + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08)
    2803     bgu,pt %XCC, 1b
    2804     add %o1, 0x8, %o1
    2805     73: andcc %o2, 0x8, %g0
    2806     be,pt %XCC, 1f
    2807     nop
    2808     sub %o2, 0x8, %o2
    2809     - EX_LD(LOAD(ldx, %o1, %o5))
    2810     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2811     + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8)
    2812     + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8)
    2813     add %o1, 0x8, %o1
    2814     1: andcc %o2, 0x4, %g0
    2815     be,pt %XCC, 1f
    2816     nop
    2817     sub %o2, 0x4, %o2
    2818     - EX_LD(LOAD(lduw, %o1, %o5))
    2819     - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2820     + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4)
    2821     + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4)
    2822     add %o1, 0x4, %o1
    2823     1: cmp %o2, 0
    2824     be,pt %XCC, 85f
    2825     @@ -361,8 +442,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2826     sub %o2, %g1, %o2
    2827    
    2828     1: subcc %g1, 1, %g1
    2829     - EX_LD(LOAD(ldub, %o1, %o5))
    2830     - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2831     + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1)
    2832     + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1)
    2833     bgu,pt %icc, 1b
    2834     add %o1, 1, %o1
    2835    
    2836     @@ -378,16 +459,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2837    
    2838     8: mov 64, %o3
    2839     andn %o1, 0x7, %o1
    2840     - EX_LD(LOAD(ldx, %o1, %g2))
    2841     + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2)
    2842     sub %o3, %g1, %o3
    2843     andn %o2, 0x7, GLOBAL_SPARE
    2844     sllx %g2, %g1, %g2
    2845     -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
    2846     +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS)
    2847     subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE
    2848     add %o1, 0x8, %o1
    2849     srlx %g3, %o3, %o5
    2850     or %o5, %g2, %o5
    2851     - EX_ST(STORE(stx, %o5, %o0))
    2852     + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8)
    2853     add %o0, 0x8, %o0
    2854     bgu,pt %icc, 1b
    2855     sllx %g3, %g1, %g2
    2856     @@ -407,8 +488,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2857    
    2858     1:
    2859     subcc %o2, 4, %o2
    2860     - EX_LD(LOAD(lduw, %o1, %g1))
    2861     - EX_ST(STORE(stw, %g1, %o1 + %o3))
    2862     + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4)
    2863     + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4)
    2864     bgu,pt %XCC, 1b
    2865     add %o1, 4, %o1
    2866    
    2867     @@ -418,8 +499,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2868     .align 32
    2869     90:
    2870     subcc %o2, 1, %o2
    2871     - EX_LD(LOAD(ldub, %o1, %g1))
    2872     - EX_ST(STORE(stb, %g1, %o1 + %o3))
    2873     + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1)
    2874     + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1)
    2875     bgu,pt %XCC, 90b
    2876     add %o1, 1, %o1
    2877     retl
    2878     diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
    2879     index 302c0e60dc2c..4c89b486fa0d 100644
    2880     --- a/arch/sparc/lib/copy_in_user.S
    2881     +++ b/arch/sparc/lib/copy_in_user.S
    2882     @@ -8,18 +8,33 @@
    2883    
    2884     #define XCC xcc
    2885    
    2886     -#define EX(x,y) \
    2887     +#define EX(x,y,z) \
    2888     98: x,y; \
    2889     .section __ex_table,"a";\
    2890     .align 4; \
    2891     - .word 98b, __retl_one; \
    2892     + .word 98b, z; \
    2893     .text; \
    2894     .align 4;
    2895    
    2896     +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8)
    2897     +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4)
    2898     +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1)
    2899     +
    2900     .register %g2,#scratch
    2901     .register %g3,#scratch
    2902    
    2903     .text
    2904     +__retl_o4_plus_8:
    2905     + add %o4, %o2, %o4
    2906     + retl
    2907     + add %o4, 8, %o0
    2908     +__retl_o2_plus_4:
    2909     + retl
    2910     + add %o2, 4, %o0
    2911     +__retl_o2_plus_1:
    2912     + retl
    2913     + add %o2, 1, %o0
    2914     +
    2915     .align 32
    2916    
    2917     /* Don't try to get too fancy here, just nice and
    2918     @@ -44,8 +59,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    2919     andn %o2, 0x7, %o4
    2920     and %o2, 0x7, %o2
    2921     1: subcc %o4, 0x8, %o4
    2922     - EX(ldxa [%o1] %asi, %o5)
    2923     - EX(stxa %o5, [%o0] %asi)
    2924     + EX_O4(ldxa [%o1] %asi, %o5)
    2925     + EX_O4(stxa %o5, [%o0] %asi)
    2926     add %o1, 0x8, %o1
    2927     bgu,pt %XCC, 1b
    2928     add %o0, 0x8, %o0
    2929     @@ -53,8 +68,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    2930     be,pt %XCC, 1f
    2931     nop
    2932     sub %o2, 0x4, %o2
    2933     - EX(lduwa [%o1] %asi, %o5)
    2934     - EX(stwa %o5, [%o0] %asi)
    2935     + EX_O2_4(lduwa [%o1] %asi, %o5)
    2936     + EX_O2_4(stwa %o5, [%o0] %asi)
    2937     add %o1, 0x4, %o1
    2938     add %o0, 0x4, %o0
    2939     1: cmp %o2, 0
    2940     @@ -70,8 +85,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    2941    
    2942     82:
    2943     subcc %o2, 4, %o2
    2944     - EX(lduwa [%o1] %asi, %g1)
    2945     - EX(stwa %g1, [%o0] %asi)
    2946     + EX_O2_4(lduwa [%o1] %asi, %g1)
    2947     + EX_O2_4(stwa %g1, [%o0] %asi)
    2948     add %o1, 4, %o1
    2949     bgu,pt %XCC, 82b
    2950     add %o0, 4, %o0
    2951     @@ -82,8 +97,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    2952     .align 32
    2953     90:
    2954     subcc %o2, 1, %o2
    2955     - EX(lduba [%o1] %asi, %g1)
    2956     - EX(stba %g1, [%o0] %asi)
    2957     + EX_O2_1(lduba [%o1] %asi, %g1)
    2958     + EX_O2_1(stba %g1, [%o0] %asi)
    2959     add %o1, 1, %o1
    2960     bgu,pt %XCC, 90b
    2961     add %o0, 1, %o0
    2962     diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c
    2963     deleted file mode 100644
    2964     index ac96ae236709..000000000000
    2965     --- a/arch/sparc/lib/user_fixup.c
    2966     +++ /dev/null
    2967     @@ -1,71 +0,0 @@
    2968     -/* user_fixup.c: Fix up user copy faults.
    2969     - *
    2970     - * Copyright (C) 2004 David S. Miller <davem@redhat.com>
    2971     - */
    2972     -
    2973     -#include <linux/compiler.h>
    2974     -#include <linux/kernel.h>
    2975     -#include <linux/string.h>
    2976     -#include <linux/errno.h>
    2977     -#include <linux/module.h>
    2978     -
    2979     -#include <asm/uaccess.h>
    2980     -
    2981     -/* Calculating the exact fault address when using
    2982     - * block loads and stores can be very complicated.
    2983     - *
    2984     - * Instead of trying to be clever and handling all
    2985     - * of the cases, just fix things up simply here.
    2986     - */
    2987     -
    2988     -static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset)
    2989     -{
    2990     - unsigned long fault_addr = current_thread_info()->fault_address;
    2991     - unsigned long end = start + size;
    2992     -
    2993     - if (fault_addr < start || fault_addr >= end) {
    2994     - *offset = 0;
    2995     - } else {
    2996     - *offset = fault_addr - start;
    2997     - size = end - fault_addr;
    2998     - }
    2999     - return size;
    3000     -}
    3001     -
    3002     -unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size)
    3003     -{
    3004     - unsigned long offset;
    3005     -
    3006     - size = compute_size((unsigned long) from, size, &offset);
    3007     - if (likely(size))
    3008     - memset(to + offset, 0, size);
    3009     -
    3010     - return size;
    3011     -}
    3012     -EXPORT_SYMBOL(copy_from_user_fixup);
    3013     -
    3014     -unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size)
    3015     -{
    3016     - unsigned long offset;
    3017     -
    3018     - return compute_size((unsigned long) to, size, &offset);
    3019     -}
    3020     -EXPORT_SYMBOL(copy_to_user_fixup);
    3021     -
    3022     -unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size)
    3023     -{
    3024     - unsigned long fault_addr = current_thread_info()->fault_address;
    3025     - unsigned long start = (unsigned long) to;
    3026     - unsigned long end = start + size;
    3027     -
    3028     - if (fault_addr >= start && fault_addr < end)
    3029     - return end - fault_addr;
    3030     -
    3031     - start = (unsigned long) from;
    3032     - end = start + size;
    3033     - if (fault_addr >= start && fault_addr < end)
    3034     - return end - fault_addr;
    3035     -
    3036     - return size;
    3037     -}
    3038     -EXPORT_SYMBOL(copy_in_user_fixup);
    3039     diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
    3040     index f2b77112e9d8..e20fbbafb0b0 100644
    3041     --- a/arch/sparc/mm/tsb.c
    3042     +++ b/arch/sparc/mm/tsb.c
    3043     @@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr)
    3044     return (tag == (vaddr >> 22));
    3045     }
    3046    
    3047     +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end)
    3048     +{
    3049     + unsigned long idx;
    3050     +
    3051     + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) {
    3052     + struct tsb *ent = &swapper_tsb[idx];
    3053     + unsigned long match = idx << 13;
    3054     +
    3055     + match |= (ent->tag << 22);
    3056     + if (match >= start && match < end)
    3057     + ent->tag = (1UL << TSB_TAG_INVALID_BIT);
    3058     + }
    3059     +}
    3060     +
    3061     /* TSB flushes need only occur on the processor initiating the address
    3062     * space modification, not on each cpu the address space has run on.
    3063     * Only the TLB flush needs that treatment.
    3064     @@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
    3065     {
    3066     unsigned long v;
    3067    
    3068     + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES)
    3069     + return flush_tsb_kernel_range_scan(start, end);
    3070     +
    3071     for (v = start; v < end; v += PAGE_SIZE) {
    3072     unsigned long hash = tsb_hash(v, PAGE_SHIFT,
    3073     KERNEL_TSB_NENTRIES);
    3074     diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
    3075     index b4f4733abc6e..5d2fd6cd3189 100644
    3076     --- a/arch/sparc/mm/ultra.S
    3077     +++ b/arch/sparc/mm/ultra.S
    3078     @@ -30,7 +30,7 @@
    3079     .text
    3080     .align 32
    3081     .globl __flush_tlb_mm
    3082     -__flush_tlb_mm: /* 18 insns */
    3083     +__flush_tlb_mm: /* 19 insns */
    3084     /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
    3085     ldxa [%o1] ASI_DMMU, %g2
    3086     cmp %g2, %o0
    3087     @@ -81,7 +81,7 @@ __flush_tlb_page: /* 22 insns */
    3088    
    3089     .align 32
    3090     .globl __flush_tlb_pending
    3091     -__flush_tlb_pending: /* 26 insns */
    3092     +__flush_tlb_pending: /* 27 insns */
    3093     /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
    3094     rdpr %pstate, %g7
    3095     sllx %o1, 3, %o1
    3096     @@ -113,12 +113,14 @@ __flush_tlb_pending: /* 26 insns */
    3097    
    3098     .align 32
    3099     .globl __flush_tlb_kernel_range
    3100     -__flush_tlb_kernel_range: /* 16 insns */
    3101     +__flush_tlb_kernel_range: /* 31 insns */
    3102     /* %o0=start, %o1=end */
    3103     cmp %o0, %o1
    3104     be,pn %xcc, 2f
    3105     + sub %o1, %o0, %o3
    3106     + srlx %o3, 18, %o4
    3107     + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow
    3108     sethi %hi(PAGE_SIZE), %o4
    3109     - sub %o1, %o0, %o3
    3110     sub %o3, %o4, %o3
    3111     or %o0, 0x20, %o0 ! Nucleus
    3112     1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
    3113     @@ -131,6 +133,41 @@ __flush_tlb_kernel_range: /* 16 insns */
    3114     retl
    3115     nop
    3116     nop
    3117     + nop
    3118     + nop
    3119     + nop
    3120     + nop
    3121     + nop
    3122     + nop
    3123     + nop
    3124     + nop
    3125     + nop
    3126     + nop
    3127     + nop
    3128     + nop
    3129     + nop
    3130     +
    3131     +__spitfire_flush_tlb_kernel_range_slow:
    3132     + mov 63 * 8, %o4
    3133     +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3
    3134     + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */
    3135     + bne,pn %xcc, 2f
    3136     + mov TLB_TAG_ACCESS, %o3
    3137     + stxa %g0, [%o3] ASI_IMMU
    3138     + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS
    3139     + membar #Sync
    3140     +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3
    3141     + andcc %o3, 0x40, %g0
    3142     + bne,pn %xcc, 2f
    3143     + mov TLB_TAG_ACCESS, %o3
    3144     + stxa %g0, [%o3] ASI_DMMU
    3145     + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS
    3146     + membar #Sync
    3147     +2: sub %o4, 8, %o4
    3148     + brgez,pt %o4, 1b
    3149     + nop
    3150     + retl
    3151     + nop
    3152    
    3153     __spitfire_flush_tlb_mm_slow:
    3154     rdpr %pstate, %g1
    3155     @@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */
    3156     retl
    3157     wrpr %g7, 0x0, %pstate
    3158    
    3159     +__cheetah_flush_tlb_kernel_range: /* 31 insns */
    3160     + /* %o0=start, %o1=end */
    3161     + cmp %o0, %o1
    3162     + be,pn %xcc, 2f
    3163     + sub %o1, %o0, %o3
    3164     + srlx %o3, 18, %o4
    3165     + brnz,pn %o4, 3f
    3166     + sethi %hi(PAGE_SIZE), %o4
    3167     + sub %o3, %o4, %o3
    3168     + or %o0, 0x20, %o0 ! Nucleus
    3169     +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
    3170     + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
    3171     + membar #Sync
    3172     + brnz,pt %o3, 1b
    3173     + sub %o3, %o4, %o3
    3174     +2: sethi %hi(KERNBASE), %o3
    3175     + flush %o3
    3176     + retl
    3177     + nop
    3178     +3: mov 0x80, %o4
    3179     + stxa %g0, [%o4] ASI_DMMU_DEMAP
    3180     + membar #Sync
    3181     + stxa %g0, [%o4] ASI_IMMU_DEMAP
    3182     + membar #Sync
    3183     + retl
    3184     + nop
    3185     + nop
    3186     + nop
    3187     + nop
    3188     + nop
    3189     + nop
    3190     + nop
    3191     + nop
    3192     +
    3193     #ifdef DCACHE_ALIASING_POSSIBLE
    3194     __cheetah_flush_dcache_page: /* 11 insns */
    3195     sethi %hi(PAGE_OFFSET), %g1
    3196     @@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error:
    3197     ret
    3198     restore
    3199    
    3200     -__hypervisor_flush_tlb_mm: /* 10 insns */
    3201     +__hypervisor_flush_tlb_mm: /* 19 insns */
    3202     mov %o0, %o2 /* ARG2: mmu context */
    3203     mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3204     mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3205     mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3206     mov HV_FAST_MMU_DEMAP_CTX, %o5
    3207     ta HV_FAST_TRAP
    3208     - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3209     + brnz,pn %o0, 1f
    3210     mov HV_FAST_MMU_DEMAP_CTX, %o1
    3211     retl
    3212     nop
    3213     +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5
    3214     + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0
    3215     + nop
    3216     + nop
    3217     + nop
    3218     + nop
    3219     + nop
    3220     + nop
    3221     + nop
    3222    
    3223     -__hypervisor_flush_tlb_page: /* 11 insns */
    3224     +__hypervisor_flush_tlb_page: /* 22 insns */
    3225     /* %o0 = context, %o1 = vaddr */
    3226     mov %o0, %g2
    3227     mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */
    3228     @@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */
    3229     srlx %o0, PAGE_SHIFT, %o0
    3230     sllx %o0, PAGE_SHIFT, %o0
    3231     ta HV_MMU_UNMAP_ADDR_TRAP
    3232     - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3233     + brnz,pn %o0, 1f
    3234     mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3235     retl
    3236     nop
    3237     +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3238     + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3239     + nop
    3240     + nop
    3241     + nop
    3242     + nop
    3243     + nop
    3244     + nop
    3245     + nop
    3246     + nop
    3247     + nop
    3248    
    3249     -__hypervisor_flush_tlb_pending: /* 16 insns */
    3250     +__hypervisor_flush_tlb_pending: /* 27 insns */
    3251     /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
    3252     sllx %o1, 3, %g1
    3253     mov %o2, %g2
    3254     @@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */
    3255     srlx %o0, PAGE_SHIFT, %o0
    3256     sllx %o0, PAGE_SHIFT, %o0
    3257     ta HV_MMU_UNMAP_ADDR_TRAP
    3258     - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3259     + brnz,pn %o0, 1f
    3260     mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3261     brnz,pt %g1, 1b
    3262     nop
    3263     retl
    3264     nop
    3265     +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3266     + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3267     + nop
    3268     + nop
    3269     + nop
    3270     + nop
    3271     + nop
    3272     + nop
    3273     + nop
    3274     + nop
    3275     + nop
    3276    
    3277     -__hypervisor_flush_tlb_kernel_range: /* 16 insns */
    3278     +__hypervisor_flush_tlb_kernel_range: /* 31 insns */
    3279     /* %o0=start, %o1=end */
    3280     cmp %o0, %o1
    3281     be,pn %xcc, 2f
    3282     - sethi %hi(PAGE_SIZE), %g3
    3283     - mov %o0, %g1
    3284     - sub %o1, %g1, %g2
    3285     + sub %o1, %o0, %g2
    3286     + srlx %g2, 18, %g3
    3287     + brnz,pn %g3, 4f
    3288     + mov %o0, %g1
    3289     + sethi %hi(PAGE_SIZE), %g3
    3290     sub %g2, %g3, %g2
    3291     1: add %g1, %g2, %o0 /* ARG0: virtual address */
    3292     mov 0, %o1 /* ARG1: mmu context */
    3293     mov HV_MMU_ALL, %o2 /* ARG2: flags */
    3294     ta HV_MMU_UNMAP_ADDR_TRAP
    3295     - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3296     + brnz,pn %o0, 3f
    3297     mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3298     brnz,pt %g2, 1b
    3299     sub %g2, %g3, %g2
    3300     2: retl
    3301     nop
    3302     +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3303     + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3304     + nop
    3305     +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3306     + mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3307     + mov 0, %o2 /* ARG2: mmu context == nucleus */
    3308     + mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3309     + mov HV_FAST_MMU_DEMAP_CTX, %o5
    3310     + ta HV_FAST_TRAP
    3311     + brnz,pn %o0, 3b
    3312     + mov HV_FAST_MMU_DEMAP_CTX, %o1
    3313     + retl
    3314     + nop
    3315    
    3316     #ifdef DCACHE_ALIASING_POSSIBLE
    3317     /* XXX Niagara and friends have an 8K cache, so no aliasing is
    3318     @@ -394,43 +511,6 @@ tlb_patch_one:
    3319     retl
    3320     nop
    3321    
    3322     - .globl cheetah_patch_cachetlbops
    3323     -cheetah_patch_cachetlbops:
    3324     - save %sp, -128, %sp
    3325     -
    3326     - sethi %hi(__flush_tlb_mm), %o0
    3327     - or %o0, %lo(__flush_tlb_mm), %o0
    3328     - sethi %hi(__cheetah_flush_tlb_mm), %o1
    3329     - or %o1, %lo(__cheetah_flush_tlb_mm), %o1
    3330     - call tlb_patch_one
    3331     - mov 19, %o2
    3332     -
    3333     - sethi %hi(__flush_tlb_page), %o0
    3334     - or %o0, %lo(__flush_tlb_page), %o0
    3335     - sethi %hi(__cheetah_flush_tlb_page), %o1
    3336     - or %o1, %lo(__cheetah_flush_tlb_page), %o1
    3337     - call tlb_patch_one
    3338     - mov 22, %o2
    3339     -
    3340     - sethi %hi(__flush_tlb_pending), %o0
    3341     - or %o0, %lo(__flush_tlb_pending), %o0
    3342     - sethi %hi(__cheetah_flush_tlb_pending), %o1
    3343     - or %o1, %lo(__cheetah_flush_tlb_pending), %o1
    3344     - call tlb_patch_one
    3345     - mov 27, %o2
    3346     -
    3347     -#ifdef DCACHE_ALIASING_POSSIBLE
    3348     - sethi %hi(__flush_dcache_page), %o0
    3349     - or %o0, %lo(__flush_dcache_page), %o0
    3350     - sethi %hi(__cheetah_flush_dcache_page), %o1
    3351     - or %o1, %lo(__cheetah_flush_dcache_page), %o1
    3352     - call tlb_patch_one
    3353     - mov 11, %o2
    3354     -#endif /* DCACHE_ALIASING_POSSIBLE */
    3355     -
    3356     - ret
    3357     - restore
    3358     -
    3359     #ifdef CONFIG_SMP
    3360     /* These are all called by the slaves of a cross call, at
    3361     * trap level 1, with interrupts fully disabled.
    3362     @@ -447,7 +527,7 @@ cheetah_patch_cachetlbops:
    3363     */
    3364     .align 32
    3365     .globl xcall_flush_tlb_mm
    3366     -xcall_flush_tlb_mm: /* 21 insns */
    3367     +xcall_flush_tlb_mm: /* 24 insns */
    3368     mov PRIMARY_CONTEXT, %g2
    3369     ldxa [%g2] ASI_DMMU, %g3
    3370     srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
    3371     @@ -469,9 +549,12 @@ xcall_flush_tlb_mm: /* 21 insns */
    3372     nop
    3373     nop
    3374     nop
    3375     + nop
    3376     + nop
    3377     + nop
    3378    
    3379     .globl xcall_flush_tlb_page
    3380     -xcall_flush_tlb_page: /* 17 insns */
    3381     +xcall_flush_tlb_page: /* 20 insns */
    3382     /* %g5=context, %g1=vaddr */
    3383     mov PRIMARY_CONTEXT, %g4
    3384     ldxa [%g4] ASI_DMMU, %g2
    3385     @@ -490,15 +573,20 @@ xcall_flush_tlb_page: /* 17 insns */
    3386     retry
    3387     nop
    3388     nop
    3389     + nop
    3390     + nop
    3391     + nop
    3392    
    3393     .globl xcall_flush_tlb_kernel_range
    3394     -xcall_flush_tlb_kernel_range: /* 25 insns */
    3395     +xcall_flush_tlb_kernel_range: /* 44 insns */
    3396     sethi %hi(PAGE_SIZE - 1), %g2
    3397     or %g2, %lo(PAGE_SIZE - 1), %g2
    3398     andn %g1, %g2, %g1
    3399     andn %g7, %g2, %g7
    3400     sub %g7, %g1, %g3
    3401     - add %g2, 1, %g2
    3402     + srlx %g3, 18, %g2
    3403     + brnz,pn %g2, 2f
    3404     + add %g2, 1, %g2
    3405     sub %g3, %g2, %g3
    3406     or %g1, 0x20, %g1 ! Nucleus
    3407     1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
    3408     @@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */
    3409     brnz,pt %g3, 1b
    3410     sub %g3, %g2, %g3
    3411     retry
    3412     - nop
    3413     - nop
    3414     +2: mov 63 * 8, %g1
    3415     +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2
    3416     + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */
    3417     + bne,pn %xcc, 2f
    3418     + mov TLB_TAG_ACCESS, %g2
    3419     + stxa %g0, [%g2] ASI_IMMU
    3420     + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS
    3421     + membar #Sync
    3422     +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2
    3423     + andcc %g2, 0x40, %g0
    3424     + bne,pn %xcc, 2f
    3425     + mov TLB_TAG_ACCESS, %g2
    3426     + stxa %g0, [%g2] ASI_DMMU
    3427     + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS
    3428     + membar #Sync
    3429     +2: sub %g1, 8, %g1
    3430     + brgez,pt %g1, 1b
    3431     + nop
    3432     + retry
    3433     nop
    3434     nop
    3435     nop
    3436     @@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4:
    3437    
    3438     retry
    3439    
    3440     +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */
    3441     + sethi %hi(PAGE_SIZE - 1), %g2
    3442     + or %g2, %lo(PAGE_SIZE - 1), %g2
    3443     + andn %g1, %g2, %g1
    3444     + andn %g7, %g2, %g7
    3445     + sub %g7, %g1, %g3
    3446     + srlx %g3, 18, %g2
    3447     + brnz,pn %g2, 2f
    3448     + add %g2, 1, %g2
    3449     + sub %g3, %g2, %g3
    3450     + or %g1, 0x20, %g1 ! Nucleus
    3451     +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
    3452     + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
    3453     + membar #Sync
    3454     + brnz,pt %g3, 1b
    3455     + sub %g3, %g2, %g3
    3456     + retry
    3457     +2: mov 0x80, %g2
    3458     + stxa %g0, [%g2] ASI_DMMU_DEMAP
    3459     + membar #Sync
    3460     + stxa %g0, [%g2] ASI_IMMU_DEMAP
    3461     + membar #Sync
    3462     + retry
    3463     + nop
    3464     + nop
    3465     + nop
    3466     + nop
    3467     + nop
    3468     + nop
    3469     + nop
    3470     + nop
    3471     + nop
    3472     + nop
    3473     + nop
    3474     + nop
    3475     + nop
    3476     + nop
    3477     + nop
    3478     + nop
    3479     + nop
    3480     + nop
    3481     + nop
    3482     + nop
    3483     + nop
    3484     + nop
    3485     +
    3486     #ifdef DCACHE_ALIASING_POSSIBLE
    3487     .align 32
    3488     .globl xcall_flush_dcache_page_cheetah
    3489     @@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error:
    3490     ba,a,pt %xcc, rtrap
    3491    
    3492     .globl __hypervisor_xcall_flush_tlb_mm
    3493     -__hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3494     +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
    3495     /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
    3496     mov %o0, %g2
    3497     mov %o1, %g3
    3498     @@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3499     mov HV_FAST_MMU_DEMAP_CTX, %o5
    3500     ta HV_FAST_TRAP
    3501     mov HV_FAST_MMU_DEMAP_CTX, %g6
    3502     - brnz,pn %o0, __hypervisor_tlb_xcall_error
    3503     + brnz,pn %o0, 1f
    3504     mov %o0, %g5
    3505     mov %g2, %o0
    3506     mov %g3, %o1
    3507     @@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3508     mov %g7, %o5
    3509     membar #Sync
    3510     retry
    3511     +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3512     + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3513     + nop
    3514    
    3515     .globl __hypervisor_xcall_flush_tlb_page
    3516     -__hypervisor_xcall_flush_tlb_page: /* 17 insns */
    3517     +__hypervisor_xcall_flush_tlb_page: /* 20 insns */
    3518     /* %g5=ctx, %g1=vaddr */
    3519     mov %o0, %g2
    3520     mov %o1, %g3
    3521     @@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */
    3522     sllx %o0, PAGE_SHIFT, %o0
    3523     ta HV_MMU_UNMAP_ADDR_TRAP
    3524     mov HV_MMU_UNMAP_ADDR_TRAP, %g6
    3525     - brnz,a,pn %o0, __hypervisor_tlb_xcall_error
    3526     + brnz,a,pn %o0, 1f
    3527     mov %o0, %g5
    3528     mov %g2, %o0
    3529     mov %g3, %o1
    3530     mov %g4, %o2
    3531     membar #Sync
    3532     retry
    3533     +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3534     + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3535     + nop
    3536    
    3537     .globl __hypervisor_xcall_flush_tlb_kernel_range
    3538     -__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
    3539     +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
    3540     /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
    3541     sethi %hi(PAGE_SIZE - 1), %g2
    3542     or %g2, %lo(PAGE_SIZE - 1), %g2
    3543     andn %g1, %g2, %g1
    3544     andn %g7, %g2, %g7
    3545     sub %g7, %g1, %g3
    3546     + srlx %g3, 18, %g7
    3547     add %g2, 1, %g2
    3548     sub %g3, %g2, %g3
    3549     mov %o0, %g2
    3550     mov %o1, %g4
    3551     - mov %o2, %g7
    3552     + brnz,pn %g7, 2f
    3553     + mov %o2, %g7
    3554     1: add %g1, %g3, %o0 /* ARG0: virtual address */
    3555     mov 0, %o1 /* ARG1: mmu context */
    3556     mov HV_MMU_ALL, %o2 /* ARG2: flags */
    3557     ta HV_MMU_UNMAP_ADDR_TRAP
    3558     mov HV_MMU_UNMAP_ADDR_TRAP, %g6
    3559     - brnz,pn %o0, __hypervisor_tlb_xcall_error
    3560     + brnz,pn %o0, 1f
    3561     mov %o0, %g5
    3562     sethi %hi(PAGE_SIZE), %o2
    3563     brnz,pt %g3, 1b
    3564     sub %g3, %o2, %g3
    3565     - mov %g2, %o0
    3566     +5: mov %g2, %o0
    3567     mov %g4, %o1
    3568     mov %g7, %o2
    3569     membar #Sync
    3570     retry
    3571     +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3572     + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3573     + nop
    3574     +2: mov %o3, %g1
    3575     + mov %o5, %g3
    3576     + mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3577     + mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3578     + mov 0, %o2 /* ARG2: mmu context == nucleus */
    3579     + mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3580     + mov HV_FAST_MMU_DEMAP_CTX, %o5
    3581     + ta HV_FAST_TRAP
    3582     + mov %g1, %o3
    3583     + brz,pt %o0, 5b
    3584     + mov %g3, %o5
    3585     + mov HV_FAST_MMU_DEMAP_CTX, %g6
    3586     + ba,pt %xcc, 1b
    3587     + clr %g5
    3588    
    3589     /* These just get rescheduled to PIL vectors. */
    3590     .globl xcall_call_function
    3591     @@ -809,6 +985,58 @@ xcall_kgdb_capture:
    3592    
    3593     #endif /* CONFIG_SMP */
    3594    
    3595     + .globl cheetah_patch_cachetlbops
    3596     +cheetah_patch_cachetlbops:
    3597     + save %sp, -128, %sp
    3598     +
    3599     + sethi %hi(__flush_tlb_mm), %o0
    3600     + or %o0, %lo(__flush_tlb_mm), %o0
    3601     + sethi %hi(__cheetah_flush_tlb_mm), %o1
    3602     + or %o1, %lo(__cheetah_flush_tlb_mm), %o1
    3603     + call tlb_patch_one
    3604     + mov 19, %o2
    3605     +
    3606     + sethi %hi(__flush_tlb_page), %o0
    3607     + or %o0, %lo(__flush_tlb_page), %o0
    3608     + sethi %hi(__cheetah_flush_tlb_page), %o1
    3609     + or %o1, %lo(__cheetah_flush_tlb_page), %o1
    3610     + call tlb_patch_one
    3611     + mov 22, %o2
    3612     +
    3613     + sethi %hi(__flush_tlb_pending), %o0
    3614     + or %o0, %lo(__flush_tlb_pending), %o0
    3615     + sethi %hi(__cheetah_flush_tlb_pending), %o1
    3616     + or %o1, %lo(__cheetah_flush_tlb_pending), %o1
    3617     + call tlb_patch_one
    3618     + mov 27, %o2
    3619     +
    3620     + sethi %hi(__flush_tlb_kernel_range), %o0
    3621     + or %o0, %lo(__flush_tlb_kernel_range), %o0
    3622     + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1
    3623     + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
    3624     + call tlb_patch_one
    3625     + mov 31, %o2
    3626     +
    3627     +#ifdef DCACHE_ALIASING_POSSIBLE
    3628     + sethi %hi(__flush_dcache_page), %o0
    3629     + or %o0, %lo(__flush_dcache_page), %o0
    3630     + sethi %hi(__cheetah_flush_dcache_page), %o1
    3631     + or %o1, %lo(__cheetah_flush_dcache_page), %o1
    3632     + call tlb_patch_one
    3633     + mov 11, %o2
    3634     +#endif /* DCACHE_ALIASING_POSSIBLE */
    3635     +
    3636     +#ifdef CONFIG_SMP
    3637     + sethi %hi(xcall_flush_tlb_kernel_range), %o0
    3638     + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
    3639     + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
    3640     + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
    3641     + call tlb_patch_one
    3642     + mov 44, %o2
    3643     +#endif /* CONFIG_SMP */
    3644     +
    3645     + ret
    3646     + restore
    3647    
    3648     .globl hypervisor_patch_cachetlbops
    3649     hypervisor_patch_cachetlbops:
    3650     @@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops:
    3651     sethi %hi(__hypervisor_flush_tlb_mm), %o1
    3652     or %o1, %lo(__hypervisor_flush_tlb_mm), %o1
    3653     call tlb_patch_one
    3654     - mov 10, %o2
    3655     + mov 19, %o2
    3656    
    3657     sethi %hi(__flush_tlb_page), %o0
    3658     or %o0, %lo(__flush_tlb_page), %o0
    3659     sethi %hi(__hypervisor_flush_tlb_page), %o1
    3660     or %o1, %lo(__hypervisor_flush_tlb_page), %o1
    3661     call tlb_patch_one
    3662     - mov 11, %o2
    3663     + mov 22, %o2
    3664    
    3665     sethi %hi(__flush_tlb_pending), %o0
    3666     or %o0, %lo(__flush_tlb_pending), %o0
    3667     sethi %hi(__hypervisor_flush_tlb_pending), %o1
    3668     or %o1, %lo(__hypervisor_flush_tlb_pending), %o1
    3669     call tlb_patch_one
    3670     - mov 16, %o2
    3671     + mov 27, %o2
    3672    
    3673     sethi %hi(__flush_tlb_kernel_range), %o0
    3674     or %o0, %lo(__flush_tlb_kernel_range), %o0
    3675     sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
    3676     or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
    3677     call tlb_patch_one
    3678     - mov 16, %o2
    3679     + mov 31, %o2
    3680    
    3681     #ifdef DCACHE_ALIASING_POSSIBLE
    3682     sethi %hi(__flush_dcache_page), %o0
    3683     @@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops:
    3684     sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1
    3685     or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
    3686     call tlb_patch_one
    3687     - mov 21, %o2
    3688     + mov 24, %o2
    3689    
    3690     sethi %hi(xcall_flush_tlb_page), %o0
    3691     or %o0, %lo(xcall_flush_tlb_page), %o0
    3692     sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1
    3693     or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
    3694     call tlb_patch_one
    3695     - mov 17, %o2
    3696     + mov 20, %o2
    3697    
    3698     sethi %hi(xcall_flush_tlb_kernel_range), %o0
    3699     or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
    3700     sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
    3701     or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
    3702     call tlb_patch_one
    3703     - mov 25, %o2
    3704     + mov 44, %o2
    3705     #endif /* CONFIG_SMP */
    3706    
    3707     ret
    3708     diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
    3709     index c4751ece76f6..45e87c9cc828 100644
    3710     --- a/drivers/net/ethernet/broadcom/bgmac.c
    3711     +++ b/drivers/net/ethernet/broadcom/bgmac.c
    3712     @@ -307,6 +307,10 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
    3713     u32 ctl;
    3714    
    3715     ctl = bgmac_read(bgmac, ring->mmio_base + BGMAC_DMA_RX_CTL);
    3716     +
    3717     + /* preserve ONLY bits 16-17 from current hardware value */
    3718     + ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
    3719     +
    3720     if (bgmac->feature_flags & BGMAC_FEAT_RX_MASK_SETUP) {
    3721     ctl &= ~BGMAC_DMA_RX_BL_MASK;
    3722     ctl |= BGMAC_DMA_RX_BL_128 << BGMAC_DMA_RX_BL_SHIFT;
    3723     @@ -317,7 +321,6 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
    3724     ctl &= ~BGMAC_DMA_RX_PT_MASK;
    3725     ctl |= BGMAC_DMA_RX_PT_1 << BGMAC_DMA_RX_PT_SHIFT;
    3726     }
    3727     - ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
    3728     ctl |= BGMAC_DMA_RX_ENABLE;
    3729     ctl |= BGMAC_DMA_RX_PARITY_DISABLE;
    3730     ctl |= BGMAC_DMA_RX_OVERFLOW_CONT;
    3731     diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
    3732     index 505ceaf451e2..2c850a92ab15 100644
    3733     --- a/drivers/net/ethernet/broadcom/bnx2.c
    3734     +++ b/drivers/net/ethernet/broadcom/bnx2.c
    3735     @@ -49,6 +49,7 @@
    3736     #include <linux/firmware.h>
    3737     #include <linux/log2.h>
    3738     #include <linux/aer.h>
    3739     +#include <linux/crash_dump.h>
    3740    
    3741     #if defined(CONFIG_CNIC) || defined(CONFIG_CNIC_MODULE)
    3742     #define BCM_CNIC 1
    3743     @@ -4759,15 +4760,16 @@ bnx2_setup_msix_tbl(struct bnx2 *bp)
    3744     BNX2_WR(bp, BNX2_PCI_GRC_WINDOW3_ADDR, BNX2_MSIX_PBA_ADDR);
    3745     }
    3746    
    3747     -static int
    3748     -bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
    3749     +static void
    3750     +bnx2_wait_dma_complete(struct bnx2 *bp)
    3751     {
    3752     u32 val;
    3753     - int i, rc = 0;
    3754     - u8 old_port;
    3755     + int i;
    3756    
    3757     - /* Wait for the current PCI transaction to complete before
    3758     - * issuing a reset. */
    3759     + /*
    3760     + * Wait for the current PCI transaction to complete before
    3761     + * issuing a reset.
    3762     + */
    3763     if ((BNX2_CHIP(bp) == BNX2_CHIP_5706) ||
    3764     (BNX2_CHIP(bp) == BNX2_CHIP_5708)) {
    3765     BNX2_WR(bp, BNX2_MISC_ENABLE_CLR_BITS,
    3766     @@ -4791,6 +4793,21 @@ bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
    3767     }
    3768     }
    3769    
    3770     + return;
    3771     +}
    3772     +
    3773     +
    3774     +static int
    3775     +bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
    3776     +{
    3777     + u32 val;
    3778     + int i, rc = 0;
    3779     + u8 old_port;
    3780     +
    3781     + /* Wait for the current PCI transaction to complete before
    3782     + * issuing a reset. */
    3783     + bnx2_wait_dma_complete(bp);
    3784     +
    3785     /* Wait for the firmware to tell us it is ok to issue a reset. */
    3786     bnx2_fw_sync(bp, BNX2_DRV_MSG_DATA_WAIT0 | reset_code, 1, 1);
    3787    
    3788     @@ -6356,6 +6373,10 @@ bnx2_open(struct net_device *dev)
    3789     struct bnx2 *bp = netdev_priv(dev);
    3790     int rc;
    3791    
    3792     + rc = bnx2_request_firmware(bp);
    3793     + if (rc < 0)
    3794     + goto out;
    3795     +
    3796     netif_carrier_off(dev);
    3797    
    3798     bnx2_disable_int(bp);
    3799     @@ -6424,6 +6445,7 @@ open_err:
    3800     bnx2_free_irq(bp);
    3801     bnx2_free_mem(bp);
    3802     bnx2_del_napi(bp);
    3803     + bnx2_release_firmware(bp);
    3804     goto out;
    3805     }
    3806    
    3807     @@ -8570,12 +8592,15 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
    3808    
    3809     pci_set_drvdata(pdev, dev);
    3810    
    3811     - rc = bnx2_request_firmware(bp);
    3812     - if (rc < 0)
    3813     - goto error;
    3814     -
    3815     + /*
    3816     + * In-flight DMA from 1st kernel could continue going in kdump kernel.
    3817     + * New io-page table has been created before bnx2 does reset at open stage.
    3818     + * We have to wait for the in-flight DMA to complete to avoid it look up
    3819     + * into the newly created io-page table.
    3820     + */
    3821     + if (is_kdump_kernel())
    3822     + bnx2_wait_dma_complete(bp);
    3823    
    3824     - bnx2_reset_chip(bp, BNX2_DRV_MSG_CODE_RESET);
    3825     memcpy(dev->dev_addr, bp->mac_addr, ETH_ALEN);
    3826    
    3827     dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG |
    3828     @@ -8608,7 +8633,6 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
    3829     return 0;
    3830    
    3831     error:
    3832     - bnx2_release_firmware(bp);
    3833     pci_iounmap(pdev, bp->regview);
    3834     pci_release_regions(pdev);
    3835     pci_disable_device(pdev);
    3836     diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
    3837     index d48873bcbddf..5cdc96bdd444 100644
    3838     --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
    3839     +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
    3840     @@ -231,7 +231,7 @@ mlxsw_sp_span_entry_create(struct mlxsw_sp_port *port)
    3841    
    3842     span_entry->used = true;
    3843     span_entry->id = index;
    3844     - span_entry->ref_count = 0;
    3845     + span_entry->ref_count = 1;
    3846     span_entry->local_port = local_port;
    3847     return span_entry;
    3848     }
    3849     @@ -268,6 +268,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port)
    3850    
    3851     span_entry = mlxsw_sp_span_entry_find(port);
    3852     if (span_entry) {
    3853     + /* Already exists, just take a reference */
    3854     span_entry->ref_count++;
    3855     return span_entry;
    3856     }
    3857     @@ -278,6 +279,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port)
    3858     static int mlxsw_sp_span_entry_put(struct mlxsw_sp *mlxsw_sp,
    3859     struct mlxsw_sp_span_entry *span_entry)
    3860     {
    3861     + WARN_ON(!span_entry->ref_count);
    3862     if (--span_entry->ref_count == 0)
    3863     mlxsw_sp_span_entry_destroy(mlxsw_sp, span_entry);
    3864     return 0;
    3865     diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
    3866     index 3f5c51da6d3e..62514b9bf988 100644
    3867     --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
    3868     +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
    3869     @@ -777,6 +777,26 @@ static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
    3870     }
    3871     }
    3872    
    3873     +static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
    3874     +{
    3875     + u8 num_rec, last_rec_index, num_entries;
    3876     +
    3877     + num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
    3878     + last_rec_index = num_rec - 1;
    3879     +
    3880     + if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
    3881     + return false;
    3882     + if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
    3883     + MLXSW_REG_RAUHTD_TYPE_IPV6)
    3884     + return true;
    3885     +
    3886     + num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
    3887     + last_rec_index);
    3888     + if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
    3889     + return true;
    3890     + return false;
    3891     +}
    3892     +
    3893     static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
    3894     {
    3895     char *rauhtd_pl;
    3896     @@ -803,7 +823,7 @@ static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
    3897     for (i = 0; i < num_rec; i++)
    3898     mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
    3899     i);
    3900     - } while (num_rec);
    3901     + } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
    3902     rtnl_unlock();
    3903    
    3904     kfree(rauhtd_pl);
    3905     diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
    3906     index 4c8c60af7985..fe9e7b1979b8 100644
    3907     --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
    3908     +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
    3909     @@ -871,6 +871,13 @@ static int stmmac_init_phy(struct net_device *dev)
    3910     return -ENODEV;
    3911     }
    3912    
    3913     + /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
    3914     + * subsequent PHY polling, make sure we force a link transition if
    3915     + * we have a UP/DOWN/UP transition
    3916     + */
    3917     + if (phydev->is_pseudo_fixed_link)
    3918     + phydev->irq = PHY_POLL;
    3919     +
    3920     pr_debug("stmmac_init_phy: %s: attached to PHY (UID 0x%x)"
    3921     " Link = %d\n", dev->name, phydev->phy_id, phydev->link);
    3922    
    3923     diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
    3924     index 5c8429f23a89..3a5530d0511b 100644
    3925     --- a/drivers/usb/gadget/function/f_fs.c
    3926     +++ b/drivers/usb/gadget/function/f_fs.c
    3927     @@ -133,8 +133,60 @@ struct ffs_epfile {
    3928     /*
    3929     * Buffer for holding data from partial reads which may happen since
    3930     * we’re rounding user read requests to a multiple of a max packet size.
    3931     + *
    3932     + * The pointer is initialised with NULL value and may be set by
    3933     + * __ffs_epfile_read_data function to point to a temporary buffer.
    3934     + *
    3935     + * In normal operation, calls to __ffs_epfile_read_buffered will consume
    3936     + * data from said buffer and eventually free it. Importantly, while the
    3937     + * function is using the buffer, it sets the pointer to NULL. This is
    3938     + * all right since __ffs_epfile_read_data and __ffs_epfile_read_buffered
    3939     + * can never run concurrently (they are synchronised by epfile->mutex)
    3940     + * so the latter will not assign a new value to the pointer.
    3941     + *
    3942     + * Meanwhile ffs_func_eps_disable frees the buffer (if the pointer is
    3943     + * valid) and sets the pointer to READ_BUFFER_DROP value. This special
    3944     + * value is crux of the synchronisation between ffs_func_eps_disable and
    3945     + * __ffs_epfile_read_data.
    3946     + *
    3947     + * Once __ffs_epfile_read_data is about to finish it will try to set the
    3948     + * pointer back to its old value (as described above), but seeing as the
    3949     + * pointer is not-NULL (namely READ_BUFFER_DROP) it will instead free
    3950     + * the buffer.
    3951     + *
    3952     + * == State transitions ==
    3953     + *
    3954     + * • ptr == NULL: (initial state)
    3955     + * â—¦ __ffs_epfile_read_buffer_free: go to ptr == DROP
    3956     + * â—¦ __ffs_epfile_read_buffered: nop
    3957     + * â—¦ __ffs_epfile_read_data allocates temp buffer: go to ptr == buf
    3958     + * â—¦ reading finishes: n/a, not in ‘and reading’ state
    3959     + * • ptr == DROP:
    3960     + * â—¦ __ffs_epfile_read_buffer_free: nop
    3961     + * â—¦ __ffs_epfile_read_buffered: go to ptr == NULL
    3962     + * â—¦ __ffs_epfile_read_data allocates temp buffer: free buf, nop
    3963     + * â—¦ reading finishes: n/a, not in ‘and reading’ state
    3964     + * • ptr == buf:
    3965     + * â—¦ __ffs_epfile_read_buffer_free: free buf, go to ptr == DROP
    3966     + * â—¦ __ffs_epfile_read_buffered: go to ptr == NULL and reading
    3967     + * â—¦ __ffs_epfile_read_data: n/a, __ffs_epfile_read_buffered
    3968     + * is always called first
    3969     + * â—¦ reading finishes: n/a, not in ‘and reading’ state
    3970     + * • ptr == NULL and reading:
    3971     + * â—¦ __ffs_epfile_read_buffer_free: go to ptr == DROP and reading
    3972     + * â—¦ __ffs_epfile_read_buffered: n/a, mutex is held
    3973     + * â—¦ __ffs_epfile_read_data: n/a, mutex is held
    3974     + * â—¦ reading finishes and …
    3975     + * … all data read: free buf, go to ptr == NULL
    3976     + * … otherwise: go to ptr == buf and reading
    3977     + * • ptr == DROP and reading:
    3978     + * â—¦ __ffs_epfile_read_buffer_free: nop
    3979     + * â—¦ __ffs_epfile_read_buffered: n/a, mutex is held
    3980     + * â—¦ __ffs_epfile_read_data: n/a, mutex is held
    3981     + * â—¦ reading finishes: free buf, go to ptr == DROP
    3982     */
    3983     - struct ffs_buffer *read_buffer; /* P: epfile->mutex */
    3984     + struct ffs_buffer *read_buffer;
    3985     +#define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN))
    3986    
    3987     char name[5];
    3988    
    3989     @@ -733,25 +785,47 @@ static void ffs_epfile_async_io_complete(struct usb_ep *_ep,
    3990     schedule_work(&io_data->work);
    3991     }
    3992    
    3993     +static void __ffs_epfile_read_buffer_free(struct ffs_epfile *epfile)
    3994     +{
    3995     + /*
    3996     + * See comment in struct ffs_epfile for full read_buffer pointer
    3997     + * synchronisation story.
    3998     + */
    3999     + struct ffs_buffer *buf = xchg(&epfile->read_buffer, READ_BUFFER_DROP);
    4000     + if (buf && buf != READ_BUFFER_DROP)
    4001     + kfree(buf);
    4002     +}
    4003     +
    4004     /* Assumes epfile->mutex is held. */
    4005     static ssize_t __ffs_epfile_read_buffered(struct ffs_epfile *epfile,
    4006     struct iov_iter *iter)
    4007     {
    4008     - struct ffs_buffer *buf = epfile->read_buffer;
    4009     + /*
    4010     + * Null out epfile->read_buffer so ffs_func_eps_disable does not free
    4011     + * the buffer while we are using it. See comment in struct ffs_epfile
    4012     + * for full read_buffer pointer synchronisation story.
    4013     + */
    4014     + struct ffs_buffer *buf = xchg(&epfile->read_buffer, NULL);
    4015     ssize_t ret;
    4016     - if (!buf)
    4017     + if (!buf || buf == READ_BUFFER_DROP)
    4018     return 0;
    4019    
    4020     ret = copy_to_iter(buf->data, buf->length, iter);
    4021     if (buf->length == ret) {
    4022     kfree(buf);
    4023     - epfile->read_buffer = NULL;
    4024     - } else if (unlikely(iov_iter_count(iter))) {
    4025     + return ret;
    4026     + }
    4027     +
    4028     + if (unlikely(iov_iter_count(iter))) {
    4029     ret = -EFAULT;
    4030     } else {
    4031     buf->length -= ret;
    4032     buf->data += ret;
    4033     }
    4034     +
    4035     + if (cmpxchg(&epfile->read_buffer, NULL, buf))
    4036     + kfree(buf);
    4037     +
    4038     return ret;
    4039     }
    4040    
    4041     @@ -780,7 +854,15 @@ static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile,
    4042     buf->length = data_len;
    4043     buf->data = buf->storage;
    4044     memcpy(buf->storage, data + ret, data_len);
    4045     - epfile->read_buffer = buf;
    4046     +
    4047     + /*
    4048     + * At this point read_buffer is NULL or READ_BUFFER_DROP (if
    4049     + * ffs_func_eps_disable has been called in the meanwhile). See comment
    4050     + * in struct ffs_epfile for full read_buffer pointer synchronisation
    4051     + * story.
    4052     + */
    4053     + if (unlikely(cmpxchg(&epfile->read_buffer, NULL, buf)))
    4054     + kfree(buf);
    4055    
    4056     return ret;
    4057     }
    4058     @@ -1094,8 +1176,7 @@ ffs_epfile_release(struct inode *inode, struct file *file)
    4059    
    4060     ENTER();
    4061    
    4062     - kfree(epfile->read_buffer);
    4063     - epfile->read_buffer = NULL;
    4064     + __ffs_epfile_read_buffer_free(epfile);
    4065     ffs_data_closed(epfile->ffs);
    4066    
    4067     return 0;
    4068     @@ -1721,24 +1802,20 @@ static void ffs_func_eps_disable(struct ffs_function *func)
    4069     unsigned count = func->ffs->eps_count;
    4070     unsigned long flags;
    4071    
    4072     + spin_lock_irqsave(&func->ffs->eps_lock, flags);
    4073     do {
    4074     - if (epfile)
    4075     - mutex_lock(&epfile->mutex);
    4076     - spin_lock_irqsave(&func->ffs->eps_lock, flags);
    4077     /* pending requests get nuked */
    4078     if (likely(ep->ep))
    4079     usb_ep_disable(ep->ep);
    4080     ++ep;
    4081     - spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
    4082    
    4083     if (epfile) {
    4084     epfile->ep = NULL;
    4085     - kfree(epfile->read_buffer);
    4086     - epfile->read_buffer = NULL;
    4087     - mutex_unlock(&epfile->mutex);
    4088     + __ffs_epfile_read_buffer_free(epfile);
    4089     ++epfile;
    4090     }
    4091     } while (--count);
    4092     + spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
    4093     }
    4094    
    4095     static int ffs_func_eps_enable(struct ffs_function *func)
    4096     diff --git a/include/net/ip.h b/include/net/ip.h
    4097     index 156b0c11b524..0ccf6daf6f56 100644
    4098     --- a/include/net/ip.h
    4099     +++ b/include/net/ip.h
    4100     @@ -47,7 +47,6 @@ struct inet_skb_parm {
    4101     #define IPSKB_REROUTED BIT(4)
    4102     #define IPSKB_DOREDIRECT BIT(5)
    4103     #define IPSKB_FRAG_PMTU BIT(6)
    4104     -#define IPSKB_FRAG_SEGS BIT(7)
    4105    
    4106     u16 frag_max_size;
    4107     };
    4108     diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
    4109     index 43a5a0e4524c..b01d5d1d7439 100644
    4110     --- a/include/net/ip6_tunnel.h
    4111     +++ b/include/net/ip6_tunnel.h
    4112     @@ -145,6 +145,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb,
    4113     {
    4114     int pkt_len, err;
    4115    
    4116     + memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
    4117     pkt_len = skb->len - skb_inner_network_offset(skb);
    4118     err = ip6_local_out(dev_net(skb_dst(skb)->dev), sk, skb);
    4119     if (unlikely(net_xmit_eval(err)))
    4120     diff --git a/include/net/sock.h b/include/net/sock.h
    4121     index 8741988e6880..c26eab962ec7 100644
    4122     --- a/include/net/sock.h
    4123     +++ b/include/net/sock.h
    4124     @@ -1587,11 +1587,11 @@ static inline void sock_put(struct sock *sk)
    4125     void sock_gen_put(struct sock *sk);
    4126    
    4127     int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
    4128     - unsigned int trim_cap);
    4129     + unsigned int trim_cap, bool refcounted);
    4130     static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
    4131     const int nested)
    4132     {
    4133     - return __sk_receive_skb(sk, skb, nested, 1);
    4134     + return __sk_receive_skb(sk, skb, nested, 1, true);
    4135     }
    4136    
    4137     static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
    4138     diff --git a/include/net/tcp.h b/include/net/tcp.h
    4139     index 7717302cab91..0de698940793 100644
    4140     --- a/include/net/tcp.h
    4141     +++ b/include/net/tcp.h
    4142     @@ -1164,6 +1164,7 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp)
    4143     }
    4144    
    4145     bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
    4146     +int tcp_filter(struct sock *sk, struct sk_buff *skb);
    4147    
    4148     #undef STATE_TRACE
    4149    
    4150     diff --git a/include/uapi/linux/atm_zatm.h b/include/uapi/linux/atm_zatm.h
    4151     index 5cd4d4d2dd1d..9c9c6ad55f14 100644
    4152     --- a/include/uapi/linux/atm_zatm.h
    4153     +++ b/include/uapi/linux/atm_zatm.h
    4154     @@ -14,7 +14,6 @@
    4155    
    4156     #include <linux/atmapi.h>
    4157     #include <linux/atmioc.h>
    4158     -#include <linux/time.h>
    4159    
    4160     #define ZATM_GETPOOL _IOW('a',ATMIOC_SARPRV+1,struct atmif_sioc)
    4161     /* get pool statistics */
    4162     diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
    4163     index 570eeca7bdfa..ad1bc67aff1b 100644
    4164     --- a/kernel/bpf/hashtab.c
    4165     +++ b/kernel/bpf/hashtab.c
    4166     @@ -687,7 +687,8 @@ static void delete_all_elements(struct bpf_htab *htab)
    4167    
    4168     hlist_for_each_entry_safe(l, n, head, hash_node) {
    4169     hlist_del_rcu(&l->hash_node);
    4170     - htab_elem_free(htab, l);
    4171     + if (l->state != HTAB_EXTRA_ELEM_USED)
    4172     + htab_elem_free(htab, l);
    4173     }
    4174     }
    4175     }
    4176     diff --git a/net/core/dev.c b/net/core/dev.c
    4177     index 44b3ba462ba1..9ce9d7284ea7 100644
    4178     --- a/net/core/dev.c
    4179     +++ b/net/core/dev.c
    4180     @@ -2484,7 +2484,7 @@ int skb_checksum_help(struct sk_buff *skb)
    4181     goto out;
    4182     }
    4183    
    4184     - *(__sum16 *)(skb->data + offset) = csum_fold(csum);
    4185     + *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
    4186     out_set_summed:
    4187     skb->ip_summed = CHECKSUM_NONE;
    4188     out:
    4189     diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
    4190     index 52742a02814f..5550a86f7264 100644
    4191     --- a/net/core/flow_dissector.c
    4192     +++ b/net/core/flow_dissector.c
    4193     @@ -118,7 +118,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
    4194     struct flow_dissector_key_tags *key_tags;
    4195     struct flow_dissector_key_keyid *key_keyid;
    4196     u8 ip_proto = 0;
    4197     - bool ret = false;
    4198     + bool ret;
    4199    
    4200     if (!data) {
    4201     data = skb->data;
    4202     @@ -481,12 +481,17 @@ ip_proto_again:
    4203     out_good:
    4204     ret = true;
    4205    
    4206     -out_bad:
    4207     + key_control->thoff = (u16)nhoff;
    4208     +out:
    4209     key_basic->n_proto = proto;
    4210     key_basic->ip_proto = ip_proto;
    4211     - key_control->thoff = (u16)nhoff;
    4212    
    4213     return ret;
    4214     +
    4215     +out_bad:
    4216     + ret = false;
    4217     + key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
    4218     + goto out;
    4219     }
    4220     EXPORT_SYMBOL(__skb_flow_dissect);
    4221    
    4222     diff --git a/net/core/sock.c b/net/core/sock.c
    4223     index fd7b41edf1ce..10acaccca5c8 100644
    4224     --- a/net/core/sock.c
    4225     +++ b/net/core/sock.c
    4226     @@ -453,7 +453,7 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
    4227     EXPORT_SYMBOL(sock_queue_rcv_skb);
    4228    
    4229     int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
    4230     - const int nested, unsigned int trim_cap)
    4231     + const int nested, unsigned int trim_cap, bool refcounted)
    4232     {
    4233     int rc = NET_RX_SUCCESS;
    4234    
    4235     @@ -487,7 +487,8 @@ int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
    4236    
    4237     bh_unlock_sock(sk);
    4238     out:
    4239     - sock_put(sk);
    4240     + if (refcounted)
    4241     + sock_put(sk);
    4242     return rc;
    4243     discard_and_relse:
    4244     kfree_skb(skb);
    4245     @@ -1563,6 +1564,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
    4246     RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
    4247    
    4248     newsk->sk_err = 0;
    4249     + newsk->sk_err_soft = 0;
    4250     newsk->sk_priority = 0;
    4251     newsk->sk_incoming_cpu = raw_smp_processor_id();
    4252     atomic64_set(&newsk->sk_cookie, 0);
    4253     diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
    4254     index 345a3aeb8c7e..b567c8725aea 100644
    4255     --- a/net/dccp/ipv4.c
    4256     +++ b/net/dccp/ipv4.c
    4257     @@ -235,7 +235,7 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
    4258     {
    4259     const struct iphdr *iph = (struct iphdr *)skb->data;
    4260     const u8 offset = iph->ihl << 2;
    4261     - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
    4262     + const struct dccp_hdr *dh;
    4263     struct dccp_sock *dp;
    4264     struct inet_sock *inet;
    4265     const int type = icmp_hdr(skb)->type;
    4266     @@ -245,11 +245,13 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
    4267     int err;
    4268     struct net *net = dev_net(skb->dev);
    4269    
    4270     - if (skb->len < offset + sizeof(*dh) ||
    4271     - skb->len < offset + __dccp_basic_hdr_len(dh)) {
    4272     - __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
    4273     - return;
    4274     - }
    4275     + /* Only need dccph_dport & dccph_sport which are the first
    4276     + * 4 bytes in dccp header.
    4277     + * Our caller (icmp_socket_deliver()) already pulled 8 bytes for us.
    4278     + */
    4279     + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
    4280     + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
    4281     + dh = (struct dccp_hdr *)(skb->data + offset);
    4282    
    4283     sk = __inet_lookup_established(net, &dccp_hashinfo,
    4284     iph->daddr, dh->dccph_dport,
    4285     @@ -868,7 +870,7 @@ lookup:
    4286     goto discard_and_relse;
    4287     nf_reset(skb);
    4288    
    4289     - return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4);
    4290     + return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4, refcounted);
    4291    
    4292     no_dccp_socket:
    4293     if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
    4294     diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
    4295     index 3828f94b234c..715e5d1dc107 100644
    4296     --- a/net/dccp/ipv6.c
    4297     +++ b/net/dccp/ipv6.c
    4298     @@ -70,7 +70,7 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
    4299     u8 type, u8 code, int offset, __be32 info)
    4300     {
    4301     const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
    4302     - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
    4303     + const struct dccp_hdr *dh;
    4304     struct dccp_sock *dp;
    4305     struct ipv6_pinfo *np;
    4306     struct sock *sk;
    4307     @@ -78,12 +78,13 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
    4308     __u64 seq;
    4309     struct net *net = dev_net(skb->dev);
    4310    
    4311     - if (skb->len < offset + sizeof(*dh) ||
    4312     - skb->len < offset + __dccp_basic_hdr_len(dh)) {
    4313     - __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
    4314     - ICMP6_MIB_INERRORS);
    4315     - return;
    4316     - }
    4317     + /* Only need dccph_dport & dccph_sport which are the first
    4318     + * 4 bytes in dccp header.
    4319     + * Our caller (icmpv6_notify()) already pulled 8 bytes for us.
    4320     + */
    4321     + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
    4322     + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
    4323     + dh = (struct dccp_hdr *)(skb->data + offset);
    4324    
    4325     sk = __inet6_lookup_established(net, &dccp_hashinfo,
    4326     &hdr->daddr, dh->dccph_dport,
    4327     @@ -738,7 +739,8 @@ lookup:
    4328     if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
    4329     goto discard_and_relse;
    4330    
    4331     - return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4) ? -1 : 0;
    4332     + return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4,
    4333     + refcounted) ? -1 : 0;
    4334    
    4335     no_dccp_socket:
    4336     if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
    4337     @@ -956,6 +958,7 @@ static const struct inet_connection_sock_af_ops dccp_ipv6_mapped = {
    4338     .getsockopt = ipv6_getsockopt,
    4339     .addr2sockaddr = inet6_csk_addr2sockaddr,
    4340     .sockaddr_len = sizeof(struct sockaddr_in6),
    4341     + .bind_conflict = inet6_csk_bind_conflict,
    4342     #ifdef CONFIG_COMPAT
    4343     .compat_setsockopt = compat_ipv6_setsockopt,
    4344     .compat_getsockopt = compat_ipv6_getsockopt,
    4345     diff --git a/net/dccp/proto.c b/net/dccp/proto.c
    4346     index 41e65804ddf5..9fe25bf63296 100644
    4347     --- a/net/dccp/proto.c
    4348     +++ b/net/dccp/proto.c
    4349     @@ -1009,6 +1009,10 @@ void dccp_close(struct sock *sk, long timeout)
    4350     __kfree_skb(skb);
    4351     }
    4352    
    4353     + /* If socket has been already reset kill it. */
    4354     + if (sk->sk_state == DCCP_CLOSED)
    4355     + goto adjudge_to_death;
    4356     +
    4357     if (data_was_unread) {
    4358     /* Unread data was tossed, send an appropriate Reset Code */
    4359     DCCP_WARN("ABORT with %u bytes unread\n", data_was_unread);
    4360     diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
    4361     index e2ffc2a5c7db..7ef703102dca 100644
    4362     --- a/net/ipv4/fib_trie.c
    4363     +++ b/net/ipv4/fib_trie.c
    4364     @@ -2455,22 +2455,19 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
    4365     struct key_vector *l, **tp = &iter->tnode;
    4366     t_key key;
    4367    
    4368     - /* use cache location of next-to-find key */
    4369     + /* use cached location of previously found key */
    4370     if (iter->pos > 0 && pos >= iter->pos) {
    4371     - pos -= iter->pos;
    4372     key = iter->key;
    4373     } else {
    4374     - iter->pos = 0;
    4375     + iter->pos = 1;
    4376     key = 0;
    4377     }
    4378    
    4379     - while ((l = leaf_walk_rcu(tp, key)) != NULL) {
    4380     + pos -= iter->pos;
    4381     +
    4382     + while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) {
    4383     key = l->key + 1;
    4384     iter->pos++;
    4385     -
    4386     - if (--pos <= 0)
    4387     - break;
    4388     -
    4389     l = NULL;
    4390    
    4391     /* handle unlikely case of a key wrap */
    4392     @@ -2479,7 +2476,7 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
    4393     }
    4394    
    4395     if (l)
    4396     - iter->key = key; /* remember it */
    4397     + iter->key = l->key; /* remember it */
    4398     else
    4399     iter->pos = 0; /* forget it */
    4400    
    4401     @@ -2507,7 +2504,7 @@ static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos)
    4402     return fib_route_get_idx(iter, *pos);
    4403    
    4404     iter->pos = 0;
    4405     - iter->key = 0;
    4406     + iter->key = KEY_MAX;
    4407    
    4408     return SEQ_START_TOKEN;
    4409     }
    4410     @@ -2516,7 +2513,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
    4411     {
    4412     struct fib_route_iter *iter = seq->private;
    4413     struct key_vector *l = NULL;
    4414     - t_key key = iter->key;
    4415     + t_key key = iter->key + 1;
    4416    
    4417     ++*pos;
    4418    
    4419     @@ -2525,7 +2522,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
    4420     l = leaf_walk_rcu(&iter->tnode, key);
    4421    
    4422     if (l) {
    4423     - iter->key = l->key + 1;
    4424     + iter->key = l->key;
    4425     iter->pos++;
    4426     } else {
    4427     iter->pos = 0;
    4428     diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c
    4429     index 38abe70e595f..48734ee6293f 100644
    4430     --- a/net/ipv4/icmp.c
    4431     +++ b/net/ipv4/icmp.c
    4432     @@ -477,7 +477,7 @@ static struct rtable *icmp_route_lookup(struct net *net,
    4433     fl4->flowi4_proto = IPPROTO_ICMP;
    4434     fl4->fl4_icmp_type = type;
    4435     fl4->fl4_icmp_code = code;
    4436     - fl4->flowi4_oif = l3mdev_master_ifindex(skb_in->dev);
    4437     + fl4->flowi4_oif = l3mdev_master_ifindex(skb_dst(skb_in)->dev);
    4438    
    4439     security_skb_classify_flow(skb_in, flowi4_to_flowi(fl4));
    4440     rt = __ip_route_output_key_hash(net, fl4,
    4441     @@ -502,7 +502,7 @@ static struct rtable *icmp_route_lookup(struct net *net,
    4442     if (err)
    4443     goto relookup_failed;
    4444    
    4445     - if (inet_addr_type_dev_table(net, skb_in->dev,
    4446     + if (inet_addr_type_dev_table(net, skb_dst(skb_in)->dev,
    4447     fl4_dec.saddr) == RTN_LOCAL) {
    4448     rt2 = __ip_route_output_key(net, &fl4_dec);
    4449     if (IS_ERR(rt2))
    4450     diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c
    4451     index 8b4ffd216839..9f0a7b96646f 100644
    4452     --- a/net/ipv4/ip_forward.c
    4453     +++ b/net/ipv4/ip_forward.c
    4454     @@ -117,7 +117,7 @@ int ip_forward(struct sk_buff *skb)
    4455     if (opt->is_strictroute && rt->rt_uses_gateway)
    4456     goto sr_failed;
    4457    
    4458     - IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS;
    4459     + IPCB(skb)->flags |= IPSKB_FORWARDED;
    4460     mtu = ip_dst_mtu_maybe_forward(&rt->dst, true);
    4461     if (ip_exceeds_mtu(skb, mtu)) {
    4462     IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
    4463     diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
    4464     index dde37fb340bf..307daed9a4b9 100644
    4465     --- a/net/ipv4/ip_output.c
    4466     +++ b/net/ipv4/ip_output.c
    4467     @@ -223,11 +223,9 @@ static int ip_finish_output_gso(struct net *net, struct sock *sk,
    4468     struct sk_buff *segs;
    4469     int ret = 0;
    4470    
    4471     - /* common case: fragmentation of segments is not allowed,
    4472     - * or seglen is <= mtu
    4473     + /* common case: seglen is <= mtu
    4474     */
    4475     - if (((IPCB(skb)->flags & IPSKB_FRAG_SEGS) == 0) ||
    4476     - skb_gso_validate_mtu(skb, mtu))
    4477     + if (skb_gso_validate_mtu(skb, mtu))
    4478     return ip_finish_output2(net, sk, skb);
    4479    
    4480     /* Slowpath - GSO segment length is exceeding the dst MTU.
    4481     diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c
    4482     index 0f227db0e9ac..afd6b5968caf 100644
    4483     --- a/net/ipv4/ip_tunnel_core.c
    4484     +++ b/net/ipv4/ip_tunnel_core.c
    4485     @@ -63,7 +63,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
    4486     int pkt_len = skb->len - skb_inner_network_offset(skb);
    4487     struct net *net = dev_net(rt->dst.dev);
    4488     struct net_device *dev = skb->dev;
    4489     - int skb_iif = skb->skb_iif;
    4490     struct iphdr *iph;
    4491     int err;
    4492    
    4493     @@ -73,16 +72,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
    4494     skb_dst_set(skb, &rt->dst);
    4495     memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
    4496    
    4497     - if (skb_iif && !(df & htons(IP_DF))) {
    4498     - /* Arrived from an ingress interface, got encapsulated, with
    4499     - * fragmentation of encapulating frames allowed.
    4500     - * If skb is gso, the resulting encapsulated network segments
    4501     - * may exceed dst mtu.
    4502     - * Allow IP Fragmentation of segments.
    4503     - */
    4504     - IPCB(skb)->flags |= IPSKB_FRAG_SEGS;
    4505     - }
    4506     -
    4507     /* Push down and install the IP header. */
    4508     skb_push(skb, sizeof(struct iphdr));
    4509     skb_reset_network_header(skb);
    4510     diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
    4511     index 5f006e13de56..27089f5ebbb1 100644
    4512     --- a/net/ipv4/ipmr.c
    4513     +++ b/net/ipv4/ipmr.c
    4514     @@ -1749,7 +1749,7 @@ static void ipmr_queue_xmit(struct net *net, struct mr_table *mrt,
    4515     vif->dev->stats.tx_bytes += skb->len;
    4516     }
    4517    
    4518     - IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS;
    4519     + IPCB(skb)->flags |= IPSKB_FORWARDED;
    4520    
    4521     /* RFC1584 teaches, that DVMRP/PIM router must deliver packets locally
    4522     * not only before forwarding, but after forwarding on all output
    4523     diff --git a/net/ipv4/route.c b/net/ipv4/route.c
    4524     index 62c3ed0b7556..2f23ef1a8486 100644
    4525     --- a/net/ipv4/route.c
    4526     +++ b/net/ipv4/route.c
    4527     @@ -753,7 +753,9 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
    4528     goto reject_redirect;
    4529     }
    4530    
    4531     - n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
    4532     + n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
    4533     + if (!n)
    4534     + n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
    4535     if (!IS_ERR(n)) {
    4536     if (!(n->nud_state & NUD_VALID)) {
    4537     neigh_event_send(n, NULL);
    4538     diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
    4539     index ffbb218de520..c876f5ddc86c 100644
    4540     --- a/net/ipv4/tcp.c
    4541     +++ b/net/ipv4/tcp.c
    4542     @@ -1145,7 +1145,7 @@ restart:
    4543    
    4544     err = -EPIPE;
    4545     if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
    4546     - goto out_err;
    4547     + goto do_error;
    4548    
    4549     sg = !!(sk->sk_route_caps & NETIF_F_SG);
    4550    
    4551     @@ -1219,7 +1219,7 @@ new_segment:
    4552    
    4553     if (!skb_can_coalesce(skb, i, pfrag->page,
    4554     pfrag->offset)) {
    4555     - if (i == sysctl_max_skb_frags || !sg) {
    4556     + if (i >= sysctl_max_skb_frags || !sg) {
    4557     tcp_mark_push(tp, skb);
    4558     goto new_segment;
    4559     }
    4560     diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c
    4561     index 10d728b6804c..ab37c6775630 100644
    4562     --- a/net/ipv4/tcp_dctcp.c
    4563     +++ b/net/ipv4/tcp_dctcp.c
    4564     @@ -56,6 +56,7 @@ struct dctcp {
    4565     u32 next_seq;
    4566     u32 ce_state;
    4567     u32 delayed_ack_reserved;
    4568     + u32 loss_cwnd;
    4569     };
    4570    
    4571     static unsigned int dctcp_shift_g __read_mostly = 4; /* g = 1/2^4 */
    4572     @@ -96,6 +97,7 @@ static void dctcp_init(struct sock *sk)
    4573     ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
    4574    
    4575     ca->delayed_ack_reserved = 0;
    4576     + ca->loss_cwnd = 0;
    4577     ca->ce_state = 0;
    4578    
    4579     dctcp_reset(tp, ca);
    4580     @@ -111,9 +113,10 @@ static void dctcp_init(struct sock *sk)
    4581    
    4582     static u32 dctcp_ssthresh(struct sock *sk)
    4583     {
    4584     - const struct dctcp *ca = inet_csk_ca(sk);
    4585     + struct dctcp *ca = inet_csk_ca(sk);
    4586     struct tcp_sock *tp = tcp_sk(sk);
    4587    
    4588     + ca->loss_cwnd = tp->snd_cwnd;
    4589     return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
    4590     }
    4591    
    4592     @@ -308,12 +311,20 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr,
    4593     return 0;
    4594     }
    4595    
    4596     +static u32 dctcp_cwnd_undo(struct sock *sk)
    4597     +{
    4598     + const struct dctcp *ca = inet_csk_ca(sk);
    4599     +
    4600     + return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
    4601     +}
    4602     +
    4603     static struct tcp_congestion_ops dctcp __read_mostly = {
    4604     .init = dctcp_init,
    4605     .in_ack_event = dctcp_update_alpha,
    4606     .cwnd_event = dctcp_cwnd_event,
    4607     .ssthresh = dctcp_ssthresh,
    4608     .cong_avoid = tcp_reno_cong_avoid,
    4609     + .undo_cwnd = dctcp_cwnd_undo,
    4610     .set_state = dctcp_state,
    4611     .get_info = dctcp_get_info,
    4612     .flags = TCP_CONG_NEEDS_ECN,
    4613     diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
    4614     index 7158d4f8dae4..7b235fa12903 100644
    4615     --- a/net/ipv4/tcp_ipv4.c
    4616     +++ b/net/ipv4/tcp_ipv4.c
    4617     @@ -1537,6 +1537,21 @@ bool tcp_prequeue(struct sock *sk, struct sk_buff *skb)
    4618     }
    4619     EXPORT_SYMBOL(tcp_prequeue);
    4620    
    4621     +int tcp_filter(struct sock *sk, struct sk_buff *skb)
    4622     +{
    4623     + struct tcphdr *th = (struct tcphdr *)skb->data;
    4624     + unsigned int eaten = skb->len;
    4625     + int err;
    4626     +
    4627     + err = sk_filter_trim_cap(sk, skb, th->doff * 4);
    4628     + if (!err) {
    4629     + eaten -= skb->len;
    4630     + TCP_SKB_CB(skb)->end_seq -= eaten;
    4631     + }
    4632     + return err;
    4633     +}
    4634     +EXPORT_SYMBOL(tcp_filter);
    4635     +
    4636     /*
    4637     * From tcp_input.c
    4638     */
    4639     @@ -1648,8 +1663,10 @@ process:
    4640    
    4641     nf_reset(skb);
    4642    
    4643     - if (sk_filter(sk, skb))
    4644     + if (tcp_filter(sk, skb))
    4645     goto discard_and_relse;
    4646     + th = (const struct tcphdr *)skb->data;
    4647     + iph = ip_hdr(skb);
    4648    
    4649     skb->dev = NULL;
    4650    
    4651     diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c
    4652     index bd59c343d35f..7370ad2e693a 100644
    4653     --- a/net/ipv6/icmp.c
    4654     +++ b/net/ipv6/icmp.c
    4655     @@ -448,7 +448,7 @@ static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info,
    4656     if (__ipv6_addr_needs_scope_id(addr_type))
    4657     iif = skb->dev->ifindex;
    4658     else
    4659     - iif = l3mdev_master_ifindex(skb->dev);
    4660     + iif = l3mdev_master_ifindex(skb_dst(skb)->dev);
    4661    
    4662     /*
    4663     * Must not send error if the source does not uniquely
    4664     diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
    4665     index fc67822c42e0..af6a09efad5b 100644
    4666     --- a/net/ipv6/tcp_ipv6.c
    4667     +++ b/net/ipv6/tcp_ipv6.c
    4668     @@ -1228,7 +1228,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
    4669     if (skb->protocol == htons(ETH_P_IP))
    4670     return tcp_v4_do_rcv(sk, skb);
    4671    
    4672     - if (sk_filter(sk, skb))
    4673     + if (tcp_filter(sk, skb))
    4674     goto discard;
    4675    
    4676     /*
    4677     @@ -1455,8 +1455,10 @@ process:
    4678     if (tcp_v6_inbound_md5_hash(sk, skb))
    4679     goto discard_and_relse;
    4680    
    4681     - if (sk_filter(sk, skb))
    4682     + if (tcp_filter(sk, skb))
    4683     goto discard_and_relse;
    4684     + th = (const struct tcphdr *)skb->data;
    4685     + hdr = ipv6_hdr(skb);
    4686    
    4687     skb->dev = NULL;
    4688    
    4689     diff --git a/net/sctp/socket.c b/net/sctp/socket.c
    4690     index baccbf3c1c60..7b0e059bf13b 100644
    4691     --- a/net/sctp/socket.c
    4692     +++ b/net/sctp/socket.c
    4693     @@ -1214,9 +1214,12 @@ static int __sctp_connect(struct sock *sk,
    4694    
    4695     timeo = sock_sndtimeo(sk, f_flags & O_NONBLOCK);
    4696    
    4697     - err = sctp_wait_for_connect(asoc, &timeo);
    4698     - if ((err == 0 || err == -EINPROGRESS) && assoc_id)
    4699     + if (assoc_id)
    4700     *assoc_id = asoc->assoc_id;
    4701     + err = sctp_wait_for_connect(asoc, &timeo);
    4702     + /* Note: the asoc may be freed after the return of
    4703     + * sctp_wait_for_connect.
    4704     + */
    4705    
    4706     /* Don't free association on exit. */
    4707     asoc = NULL;
    4708     @@ -4278,19 +4281,18 @@ static void sctp_shutdown(struct sock *sk, int how)
    4709     {
    4710     struct net *net = sock_net(sk);
    4711     struct sctp_endpoint *ep;
    4712     - struct sctp_association *asoc;
    4713    
    4714     if (!sctp_style(sk, TCP))
    4715     return;
    4716    
    4717     - if (how & SEND_SHUTDOWN) {
    4718     + ep = sctp_sk(sk)->ep;
    4719     + if (how & SEND_SHUTDOWN && !list_empty(&ep->asocs)) {
    4720     + struct sctp_association *asoc;
    4721     +
    4722     sk->sk_state = SCTP_SS_CLOSING;
    4723     - ep = sctp_sk(sk)->ep;
    4724     - if (!list_empty(&ep->asocs)) {
    4725     - asoc = list_entry(ep->asocs.next,
    4726     - struct sctp_association, asocs);
    4727     - sctp_primitive_SHUTDOWN(net, asoc, NULL);
    4728     - }
    4729     + asoc = list_entry(ep->asocs.next,
    4730     + struct sctp_association, asocs);
    4731     + sctp_primitive_SHUTDOWN(net, asoc, NULL);
    4732     }
    4733     }
    4734    
    4735     diff --git a/net/socket.c b/net/socket.c
    4736     index a1bd16106625..03bc2c289c94 100644
    4737     --- a/net/socket.c
    4738     +++ b/net/socket.c
    4739     @@ -2041,6 +2041,8 @@ int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen,
    4740     if (err)
    4741     break;
    4742     ++datagrams;
    4743     + if (msg_data_left(&msg_sys))
    4744     + break;
    4745     cond_resched();
    4746     }
    4747    
    4748     diff --git a/tools/spi/spidev_test.c b/tools/spi/spidev_test.c
    4749     index f3825b676e38..f046b77cfefe 100644
    4750     --- a/tools/spi/spidev_test.c
    4751     +++ b/tools/spi/spidev_test.c
    4752     @@ -19,6 +19,7 @@
    4753     #include <getopt.h>
    4754     #include <fcntl.h>
    4755     #include <sys/ioctl.h>
    4756     +#include <linux/ioctl.h>
    4757     #include <sys/stat.h>
    4758     #include <linux/types.h>
    4759     #include <linux/spi/spidev.h>