Magellan Linux

  • Back to homepage
    • /[pkg-src]/trunk/kernel-alx/patches-4.4/0133-4.4.34-all-fixes.patch

    Annotation of /trunk/kernel-alx/patches-4.4/0133-4.4.34-all-fixes.patch

    Parent Directory Parent Directory | Revision Log Revision Log

    Revision 2869 - (hide annotations) (download)
    Mon Mar 27 13:49:12 2017 UTC (7 years, 1 month ago) by niro
    File size: 133458 byte(s) 
    linux-4.4.34
    1 niro 2869 diff --git a/Makefile b/Makefile
    2     index a513c045c8de..30924aabf1b4 100644
    3     --- a/Makefile
    4     +++ b/Makefile
    5     @@ -1,6 +1,6 @@
    6     VERSION = 4
    7     PATCHLEVEL = 4
    8     -SUBLEVEL = 33
    9     +SUBLEVEL = 34
    10     EXTRAVERSION =
    11     NAME = Blurry Fish Butt
    12    
    13     diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h
    14     index 70067ce184b1..f7de0dbc38af 100644
    15     --- a/arch/sparc/include/asm/mmu_64.h
    16     +++ b/arch/sparc/include/asm/mmu_64.h
    17     @@ -92,7 +92,8 @@ struct tsb_config {
    18     typedef struct {
    19     spinlock_t lock;
    20     unsigned long sparc64_ctx_val;
    21     - unsigned long huge_pte_count;
    22     + unsigned long hugetlb_pte_count;
    23     + unsigned long thp_pte_count;
    24     struct tsb_config tsb_block[MM_NUM_TSBS];
    25     struct hv_tsb_descr tsb_descr[MM_NUM_TSBS];
    26     } mm_context_t;
    27     diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
    28     index ea6e9a20f3ff..f428512481f9 100644
    29     --- a/arch/sparc/include/asm/uaccess_64.h
    30     +++ b/arch/sparc/include/asm/uaccess_64.h
    31     @@ -98,7 +98,6 @@ struct exception_table_entry {
    32     unsigned int insn, fixup;
    33     };
    34    
    35     -void __ret_efault(void);
    36     void __retl_efault(void);
    37    
    38     /* Uh, these should become the main single-value transfer routines..
    39     @@ -179,20 +178,6 @@ int __put_user_bad(void);
    40     __gu_ret; \
    41     })
    42    
    43     -#define __get_user_nocheck_ret(data, addr, size, type, retval) ({ \
    44     - register unsigned long __gu_val __asm__ ("l1"); \
    45     - switch (size) { \
    46     - case 1: __get_user_asm_ret(__gu_val, ub, addr, retval); break; \
    47     - case 2: __get_user_asm_ret(__gu_val, uh, addr, retval); break; \
    48     - case 4: __get_user_asm_ret(__gu_val, uw, addr, retval); break; \
    49     - case 8: __get_user_asm_ret(__gu_val, x, addr, retval); break; \
    50     - default: \
    51     - if (__get_user_bad()) \
    52     - return retval; \
    53     - } \
    54     - data = (__force type) __gu_val; \
    55     -})
    56     -
    57     #define __get_user_asm(x, size, addr, ret) \
    58     __asm__ __volatile__( \
    59     "/* Get user asm, inline. */\n" \
    60     @@ -214,80 +199,35 @@ __asm__ __volatile__( \
    61     : "=r" (ret), "=r" (x) : "r" (__m(addr)), \
    62     "i" (-EFAULT))
    63    
    64     -#define __get_user_asm_ret(x, size, addr, retval) \
    65     -if (__builtin_constant_p(retval) && retval == -EFAULT) \
    66     - __asm__ __volatile__( \
    67     - "/* Get user asm ret, inline. */\n" \
    68     - "1:\t" "ld"#size "a [%1] %%asi, %0\n\n\t" \
    69     - ".section __ex_table,\"a\"\n\t" \
    70     - ".align 4\n\t" \
    71     - ".word 1b,__ret_efault\n\n\t" \
    72     - ".previous\n\t" \
    73     - : "=r" (x) : "r" (__m(addr))); \
    74     -else \
    75     - __asm__ __volatile__( \
    76     - "/* Get user asm ret, inline. */\n" \
    77     - "1:\t" "ld"#size "a [%1] %%asi, %0\n\n\t" \
    78     - ".section .fixup,#alloc,#execinstr\n\t" \
    79     - ".align 4\n" \
    80     - "3:\n\t" \
    81     - "ret\n\t" \
    82     - " restore %%g0, %2, %%o0\n\n\t" \
    83     - ".previous\n\t" \
    84     - ".section __ex_table,\"a\"\n\t" \
    85     - ".align 4\n\t" \
    86     - ".word 1b, 3b\n\n\t" \
    87     - ".previous\n\t" \
    88     - : "=r" (x) : "r" (__m(addr)), "i" (retval))
    89     -
    90     int __get_user_bad(void);
    91    
    92     unsigned long __must_check ___copy_from_user(void *to,
    93     const void __user *from,
    94     unsigned long size);
    95     -unsigned long copy_from_user_fixup(void *to, const void __user *from,
    96     - unsigned long size);
    97     static inline unsigned long __must_check
    98     copy_from_user(void *to, const void __user *from, unsigned long size)
    99     {
    100     - unsigned long ret = ___copy_from_user(to, from, size);
    101     -
    102     - if (unlikely(ret))
    103     - ret = copy_from_user_fixup(to, from, size);
    104     -
    105     - return ret;
    106     + return ___copy_from_user(to, from, size);
    107     }
    108     #define __copy_from_user copy_from_user
    109    
    110     unsigned long __must_check ___copy_to_user(void __user *to,
    111     const void *from,
    112     unsigned long size);
    113     -unsigned long copy_to_user_fixup(void __user *to, const void *from,
    114     - unsigned long size);
    115     static inline unsigned long __must_check
    116     copy_to_user(void __user *to, const void *from, unsigned long size)
    117     {
    118     - unsigned long ret = ___copy_to_user(to, from, size);
    119     -
    120     - if (unlikely(ret))
    121     - ret = copy_to_user_fixup(to, from, size);
    122     - return ret;
    123     + return ___copy_to_user(to, from, size);
    124     }
    125     #define __copy_to_user copy_to_user
    126    
    127     unsigned long __must_check ___copy_in_user(void __user *to,
    128     const void __user *from,
    129     unsigned long size);
    130     -unsigned long copy_in_user_fixup(void __user *to, void __user *from,
    131     - unsigned long size);
    132     static inline unsigned long __must_check
    133     copy_in_user(void __user *to, void __user *from, unsigned long size)
    134     {
    135     - unsigned long ret = ___copy_in_user(to, from, size);
    136     -
    137     - if (unlikely(ret))
    138     - ret = copy_in_user_fixup(to, from, size);
    139     - return ret;
    140     + return ___copy_in_user(to, from, size);
    141     }
    142     #define __copy_in_user copy_in_user
    143    
    144     diff --git a/arch/sparc/kernel/dtlb_prot.S b/arch/sparc/kernel/dtlb_prot.S
    145     index d668ca149e64..4087a62f96b0 100644
    146     --- a/arch/sparc/kernel/dtlb_prot.S
    147     +++ b/arch/sparc/kernel/dtlb_prot.S
    148     @@ -25,13 +25,13 @@
    149    
    150     /* PROT ** ICACHE line 2: More real fault processing */
    151     ldxa [%g4] ASI_DMMU, %g5 ! Put tagaccess in %g5
    152     + srlx %g5, PAGE_SHIFT, %g5
    153     + sllx %g5, PAGE_SHIFT, %g5 ! Clear context ID bits
    154     bgu,pn %xcc, winfix_trampoline ! Yes, perform winfixup
    155     mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
    156     ba,pt %xcc, sparc64_realfault_common ! Nope, normal fault
    157     nop
    158     nop
    159     - nop
    160     - nop
    161    
    162     /* PROT ** ICACHE line 3: Unused... */
    163     nop
    164     diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
    165     index 51faf92ace00..7eeeb1d5a410 100644
    166     --- a/arch/sparc/kernel/head_64.S
    167     +++ b/arch/sparc/kernel/head_64.S
    168     @@ -922,47 +922,11 @@ prom_tba: .xword 0
    169     tlb_type: .word 0 /* Must NOT end up in BSS */
    170     .section ".fixup",#alloc,#execinstr
    171    
    172     - .globl __ret_efault, __retl_efault, __ret_one, __retl_one
    173     -ENTRY(__ret_efault)
    174     - ret
    175     - restore %g0, -EFAULT, %o0
    176     -ENDPROC(__ret_efault)
    177     -
    178     ENTRY(__retl_efault)
    179     retl
    180     mov -EFAULT, %o0
    181     ENDPROC(__retl_efault)
    182    
    183     -ENTRY(__retl_one)
    184     - retl
    185     - mov 1, %o0
    186     -ENDPROC(__retl_one)
    187     -
    188     -ENTRY(__retl_one_fp)
    189     - VISExitHalf
    190     - retl
    191     - mov 1, %o0
    192     -ENDPROC(__retl_one_fp)
    193     -
    194     -ENTRY(__ret_one_asi)
    195     - wr %g0, ASI_AIUS, %asi
    196     - ret
    197     - restore %g0, 1, %o0
    198     -ENDPROC(__ret_one_asi)
    199     -
    200     -ENTRY(__retl_one_asi)
    201     - wr %g0, ASI_AIUS, %asi
    202     - retl
    203     - mov 1, %o0
    204     -ENDPROC(__retl_one_asi)
    205     -
    206     -ENTRY(__retl_one_asi_fp)
    207     - wr %g0, ASI_AIUS, %asi
    208     - VISExitHalf
    209     - retl
    210     - mov 1, %o0
    211     -ENDPROC(__retl_one_asi_fp)
    212     -
    213     ENTRY(__retl_o1)
    214     retl
    215     mov %o1, %o0
    216     diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
    217     index 59bbeff55024..07933b9e9ce0 100644
    218     --- a/arch/sparc/kernel/jump_label.c
    219     +++ b/arch/sparc/kernel/jump_label.c
    220     @@ -13,19 +13,30 @@
    221     void arch_jump_label_transform(struct jump_entry *entry,
    222     enum jump_label_type type)
    223     {
    224     - u32 val;
    225     u32 *insn = (u32 *) (unsigned long) entry->code;
    226     + u32 val;
    227    
    228     if (type == JUMP_LABEL_JMP) {
    229     s32 off = (s32)entry->target - (s32)entry->code;
    230     + bool use_v9_branch = false;
    231     +
    232     + BUG_ON(off & 3);
    233    
    234     #ifdef CONFIG_SPARC64
    235     - /* ba,pt %xcc, . + (off << 2) */
    236     - val = 0x10680000 | ((u32) off >> 2);
    237     -#else
    238     - /* ba . + (off << 2) */
    239     - val = 0x10800000 | ((u32) off >> 2);
    240     + if (off <= 0xfffff && off >= -0x100000)
    241     + use_v9_branch = true;
    242     #endif
    243     + if (use_v9_branch) {
    244     + /* WDISP19 - target is . + immed << 2 */
    245     + /* ba,pt %xcc, . + off */
    246     + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff);
    247     + } else {
    248     + /* WDISP22 - target is . + immed << 2 */
    249     + BUG_ON(off > 0x7fffff);
    250     + BUG_ON(off < -0x800000);
    251     + /* ba . + off */
    252     + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff);
    253     + }
    254     } else {
    255     val = 0x01000000;
    256     }
    257     diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S
    258     index ef0d8e9e1210..f22bec0db645 100644
    259     --- a/arch/sparc/kernel/ktlb.S
    260     +++ b/arch/sparc/kernel/ktlb.S
    261     @@ -20,6 +20,10 @@ kvmap_itlb:
    262     mov TLB_TAG_ACCESS, %g4
    263     ldxa [%g4] ASI_IMMU, %g4
    264    
    265     + /* The kernel executes in context zero, therefore we do not
    266     + * need to clear the context ID bits out of %g4 here.
    267     + */
    268     +
    269     /* sun4v_itlb_miss branches here with the missing virtual
    270     * address already loaded into %g4
    271     */
    272     @@ -128,6 +132,10 @@ kvmap_dtlb:
    273     mov TLB_TAG_ACCESS, %g4
    274     ldxa [%g4] ASI_DMMU, %g4
    275    
    276     + /* The kernel executes in context zero, therefore we do not
    277     + * need to clear the context ID bits out of %g4 here.
    278     + */
    279     +
    280     /* sun4v_dtlb_miss branches here with the missing virtual
    281     * address already loaded into %g4
    282     */
    283     @@ -251,6 +259,10 @@ kvmap_dtlb_longpath:
    284     nop
    285     .previous
    286    
    287     + /* The kernel executes in context zero, therefore we do not
    288     + * need to clear the context ID bits out of %g5 here.
    289     + */
    290     +
    291     be,pt %xcc, sparc64_realfault_common
    292     mov FAULT_CODE_DTLB, %g4
    293     ba,pt %xcc, winfix_trampoline
    294     diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c
    295     index a92d5d2c46a3..51b25325a961 100644
    296     --- a/arch/sparc/kernel/sparc_ksyms_64.c
    297     +++ b/arch/sparc/kernel/sparc_ksyms_64.c
    298     @@ -27,7 +27,6 @@ EXPORT_SYMBOL(__flushw_user);
    299     EXPORT_SYMBOL_GPL(real_hard_smp_processor_id);
    300    
    301     /* from head_64.S */
    302     -EXPORT_SYMBOL(__ret_efault);
    303     EXPORT_SYMBOL(tlb_type);
    304     EXPORT_SYMBOL(sun4v_chip_type);
    305     EXPORT_SYMBOL(prom_root_node);
    306     diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S
    307     index be98685c14c6..d568c8207af7 100644
    308     --- a/arch/sparc/kernel/tsb.S
    309     +++ b/arch/sparc/kernel/tsb.S
    310     @@ -29,13 +29,17 @@
    311     */
    312     tsb_miss_dtlb:
    313     mov TLB_TAG_ACCESS, %g4
    314     + ldxa [%g4] ASI_DMMU, %g4
    315     + srlx %g4, PAGE_SHIFT, %g4
    316     ba,pt %xcc, tsb_miss_page_table_walk
    317     - ldxa [%g4] ASI_DMMU, %g4
    318     + sllx %g4, PAGE_SHIFT, %g4
    319    
    320     tsb_miss_itlb:
    321     mov TLB_TAG_ACCESS, %g4
    322     + ldxa [%g4] ASI_IMMU, %g4
    323     + srlx %g4, PAGE_SHIFT, %g4
    324     ba,pt %xcc, tsb_miss_page_table_walk
    325     - ldxa [%g4] ASI_IMMU, %g4
    326     + sllx %g4, PAGE_SHIFT, %g4
    327    
    328     /* At this point we have:
    329     * %g1 -- PAGE_SIZE TSB entry address
    330     @@ -284,6 +288,10 @@ tsb_do_dtlb_fault:
    331     nop
    332     .previous
    333    
    334     + /* Clear context ID bits. */
    335     + srlx %g5, PAGE_SHIFT, %g5
    336     + sllx %g5, PAGE_SHIFT, %g5
    337     +
    338     be,pt %xcc, sparc64_realfault_common
    339     mov FAULT_CODE_DTLB, %g4
    340     ba,pt %xcc, winfix_trampoline
    341     diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
    342     index b7d0bd6b1406..69a439fa2fc1 100644
    343     --- a/arch/sparc/lib/GENcopy_from_user.S
    344     +++ b/arch/sparc/lib/GENcopy_from_user.S
    345     @@ -3,11 +3,11 @@
    346     * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    347     */
    348    
    349     -#define EX_LD(x) \
    350     +#define EX_LD(x,y) \
    351     98: x; \
    352     .section __ex_table,"a";\
    353     .align 4; \
    354     - .word 98b, __retl_one; \
    355     + .word 98b, y; \
    356     .text; \
    357     .align 4;
    358    
    359     diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
    360     index 780550e1afc7..9947427ce354 100644
    361     --- a/arch/sparc/lib/GENcopy_to_user.S
    362     +++ b/arch/sparc/lib/GENcopy_to_user.S
    363     @@ -3,11 +3,11 @@
    364     * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    365     */
    366    
    367     -#define EX_ST(x) \
    368     +#define EX_ST(x,y) \
    369     98: x; \
    370     .section __ex_table,"a";\
    371     .align 4; \
    372     - .word 98b, __retl_one; \
    373     + .word 98b, y; \
    374     .text; \
    375     .align 4;
    376    
    377     diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S
    378     index 89358ee94851..059ea24ad73d 100644
    379     --- a/arch/sparc/lib/GENmemcpy.S
    380     +++ b/arch/sparc/lib/GENmemcpy.S
    381     @@ -4,21 +4,18 @@
    382     */
    383    
    384     #ifdef __KERNEL__
    385     +#include <linux/linkage.h>
    386     #define GLOBAL_SPARE %g7
    387     #else
    388     #define GLOBAL_SPARE %g5
    389     #endif
    390    
    391     #ifndef EX_LD
    392     -#define EX_LD(x) x
    393     +#define EX_LD(x,y) x
    394     #endif
    395    
    396     #ifndef EX_ST
    397     -#define EX_ST(x) x
    398     -#endif
    399     -
    400     -#ifndef EX_RETVAL
    401     -#define EX_RETVAL(x) x
    402     +#define EX_ST(x,y) x
    403     #endif
    404    
    405     #ifndef LOAD
    406     @@ -45,6 +42,29 @@
    407     .register %g3,#scratch
    408    
    409     .text
    410     +
    411     +#ifndef EX_RETVAL
    412     +#define EX_RETVAL(x) x
    413     +ENTRY(GEN_retl_o4_1)
    414     + add %o4, %o2, %o4
    415     + retl
    416     + add %o4, 1, %o0
    417     +ENDPROC(GEN_retl_o4_1)
    418     +ENTRY(GEN_retl_g1_8)
    419     + add %g1, %o2, %g1
    420     + retl
    421     + add %g1, 8, %o0
    422     +ENDPROC(GEN_retl_g1_8)
    423     +ENTRY(GEN_retl_o2_4)
    424     + retl
    425     + add %o2, 4, %o0
    426     +ENDPROC(GEN_retl_o2_4)
    427     +ENTRY(GEN_retl_o2_1)
    428     + retl
    429     + add %o2, 1, %o0
    430     +ENDPROC(GEN_retl_o2_1)
    431     +#endif
    432     +
    433     .align 64
    434    
    435     .globl FUNC_NAME
    436     @@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    437     sub %g0, %o4, %o4
    438     sub %o2, %o4, %o2
    439     1: subcc %o4, 1, %o4
    440     - EX_LD(LOAD(ldub, %o1, %g1))
    441     - EX_ST(STORE(stb, %g1, %o0))
    442     + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1)
    443     + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1)
    444     add %o1, 1, %o1
    445     bne,pt %XCC, 1b
    446     add %o0, 1, %o0
    447     @@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    448     andn %o2, 0x7, %g1
    449     sub %o2, %g1, %o2
    450     1: subcc %g1, 0x8, %g1
    451     - EX_LD(LOAD(ldx, %o1, %g2))
    452     - EX_ST(STORE(stx, %g2, %o0))
    453     + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8)
    454     + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8)
    455     add %o1, 0x8, %o1
    456     bne,pt %XCC, 1b
    457     add %o0, 0x8, %o0
    458     @@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    459    
    460     1:
    461     subcc %o2, 4, %o2
    462     - EX_LD(LOAD(lduw, %o1, %g1))
    463     - EX_ST(STORE(stw, %g1, %o1 + %o3))
    464     + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4)
    465     + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4)
    466     bgu,pt %XCC, 1b
    467     add %o1, 4, %o1
    468    
    469     @@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    470     .align 32
    471     90:
    472     subcc %o2, 1, %o2
    473     - EX_LD(LOAD(ldub, %o1, %g1))
    474     - EX_ST(STORE(stb, %g1, %o1 + %o3))
    475     + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1)
    476     + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1)
    477     bgu,pt %XCC, 90b
    478     add %o1, 1, %o1
    479     retl
    480     diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
    481     index 3269b0234093..4f2384a4286a 100644
    482     --- a/arch/sparc/lib/Makefile
    483     +++ b/arch/sparc/lib/Makefile
    484     @@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o
    485     lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
    486     lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
    487    
    488     -lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o
    489     +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
    490     lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
    491    
    492     obj-$(CONFIG_SPARC64) += iomap.o
    493     diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
    494     index d5242b8c4f94..b79a6998d87c 100644
    495     --- a/arch/sparc/lib/NG2copy_from_user.S
    496     +++ b/arch/sparc/lib/NG2copy_from_user.S
    497     @@ -3,19 +3,19 @@
    498     * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    499     */
    500    
    501     -#define EX_LD(x) \
    502     +#define EX_LD(x,y) \
    503     98: x; \
    504     .section __ex_table,"a";\
    505     .align 4; \
    506     - .word 98b, __retl_one_asi;\
    507     + .word 98b, y; \
    508     .text; \
    509     .align 4;
    510    
    511     -#define EX_LD_FP(x) \
    512     +#define EX_LD_FP(x,y) \
    513     98: x; \
    514     .section __ex_table,"a";\
    515     .align 4; \
    516     - .word 98b, __retl_one_asi_fp;\
    517     + .word 98b, y##_fp; \
    518     .text; \
    519     .align 4;
    520    
    521     diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
    522     index 4e962d993b10..dcec55f254ab 100644
    523     --- a/arch/sparc/lib/NG2copy_to_user.S
    524     +++ b/arch/sparc/lib/NG2copy_to_user.S
    525     @@ -3,19 +3,19 @@
    526     * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    527     */
    528    
    529     -#define EX_ST(x) \
    530     +#define EX_ST(x,y) \
    531     98: x; \
    532     .section __ex_table,"a";\
    533     .align 4; \
    534     - .word 98b, __retl_one_asi;\
    535     + .word 98b, y; \
    536     .text; \
    537     .align 4;
    538    
    539     -#define EX_ST_FP(x) \
    540     +#define EX_ST_FP(x,y) \
    541     98: x; \
    542     .section __ex_table,"a";\
    543     .align 4; \
    544     - .word 98b, __retl_one_asi_fp;\
    545     + .word 98b, y##_fp; \
    546     .text; \
    547     .align 4;
    548    
    549     diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
    550     index d5f585df2f3f..c629dbd121b6 100644
    551     --- a/arch/sparc/lib/NG2memcpy.S
    552     +++ b/arch/sparc/lib/NG2memcpy.S
    553     @@ -4,6 +4,7 @@
    554     */
    555    
    556     #ifdef __KERNEL__
    557     +#include <linux/linkage.h>
    558     #include <asm/visasm.h>
    559     #include <asm/asi.h>
    560     #define GLOBAL_SPARE %g7
    561     @@ -32,21 +33,17 @@
    562     #endif
    563    
    564     #ifndef EX_LD
    565     -#define EX_LD(x) x
    566     +#define EX_LD(x,y) x
    567     #endif
    568     #ifndef EX_LD_FP
    569     -#define EX_LD_FP(x) x
    570     +#define EX_LD_FP(x,y) x
    571     #endif
    572    
    573     #ifndef EX_ST
    574     -#define EX_ST(x) x
    575     +#define EX_ST(x,y) x
    576     #endif
    577     #ifndef EX_ST_FP
    578     -#define EX_ST_FP(x) x
    579     -#endif
    580     -
    581     -#ifndef EX_RETVAL
    582     -#define EX_RETVAL(x) x
    583     +#define EX_ST_FP(x,y) x
    584     #endif
    585    
    586     #ifndef LOAD
    587     @@ -140,45 +137,110 @@
    588     fsrc2 %x6, %f12; \
    589     fsrc2 %x7, %f14;
    590     #define FREG_LOAD_1(base, x0) \
    591     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0))
    592     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1)
    593     #define FREG_LOAD_2(base, x0, x1) \
    594     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    595     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1));
    596     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    597     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1);
    598     #define FREG_LOAD_3(base, x0, x1, x2) \
    599     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    600     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    601     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2));
    602     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    603     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    604     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1);
    605     #define FREG_LOAD_4(base, x0, x1, x2, x3) \
    606     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    607     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    608     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    609     - EX_LD_FP(LOAD(ldd, base + 0x18, %x3));
    610     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    611     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    612     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    613     + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1);
    614     #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
    615     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    616     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    617     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    618     - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    619     - EX_LD_FP(LOAD(ldd, base + 0x20, %x4));
    620     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    621     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    622     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    623     + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    624     + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1);
    625     #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
    626     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    627     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    628     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    629     - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    630     - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
    631     - EX_LD_FP(LOAD(ldd, base + 0x28, %x5));
    632     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    633     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    634     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    635     + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    636     + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
    637     + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1);
    638     #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
    639     - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    640     - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    641     - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    642     - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    643     - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
    644     - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \
    645     - EX_LD_FP(LOAD(ldd, base + 0x30, %x6));
    646     + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    647     + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    648     + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    649     + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    650     + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
    651     + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \
    652     + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1);
    653    
    654     .register %g2,#scratch
    655     .register %g3,#scratch
    656    
    657     .text
    658     +#ifndef EX_RETVAL
    659     +#define EX_RETVAL(x) x
    660     +__restore_fp:
    661     + VISExitHalf
    662     +__restore_asi:
    663     + retl
    664     + wr %g0, ASI_AIUS, %asi
    665     +ENTRY(NG2_retl_o2)
    666     + ba,pt %xcc, __restore_asi
    667     + mov %o2, %o0
    668     +ENDPROC(NG2_retl_o2)
    669     +ENTRY(NG2_retl_o2_plus_1)
    670     + ba,pt %xcc, __restore_asi
    671     + add %o2, 1, %o0
    672     +ENDPROC(NG2_retl_o2_plus_1)
    673     +ENTRY(NG2_retl_o2_plus_4)
    674     + ba,pt %xcc, __restore_asi
    675     + add %o2, 4, %o0
    676     +ENDPROC(NG2_retl_o2_plus_4)
    677     +ENTRY(NG2_retl_o2_plus_8)
    678     + ba,pt %xcc, __restore_asi
    679     + add %o2, 8, %o0
    680     +ENDPROC(NG2_retl_o2_plus_8)
    681     +ENTRY(NG2_retl_o2_plus_o4_plus_1)
    682     + add %o4, 1, %o4
    683     + ba,pt %xcc, __restore_asi
    684     + add %o2, %o4, %o0
    685     +ENDPROC(NG2_retl_o2_plus_o4_plus_1)
    686     +ENTRY(NG2_retl_o2_plus_o4_plus_8)
    687     + add %o4, 8, %o4
    688     + ba,pt %xcc, __restore_asi
    689     + add %o2, %o4, %o0
    690     +ENDPROC(NG2_retl_o2_plus_o4_plus_8)
    691     +ENTRY(NG2_retl_o2_plus_o4_plus_16)
    692     + add %o4, 16, %o4
    693     + ba,pt %xcc, __restore_asi
    694     + add %o2, %o4, %o0
    695     +ENDPROC(NG2_retl_o2_plus_o4_plus_16)
    696     +ENTRY(NG2_retl_o2_plus_g1_fp)
    697     + ba,pt %xcc, __restore_fp
    698     + add %o2, %g1, %o0
    699     +ENDPROC(NG2_retl_o2_plus_g1_fp)
    700     +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp)
    701     + add %g1, 64, %g1
    702     + ba,pt %xcc, __restore_fp
    703     + add %o2, %g1, %o0
    704     +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp)
    705     +ENTRY(NG2_retl_o2_plus_g1_plus_1)
    706     + add %g1, 1, %g1
    707     + ba,pt %xcc, __restore_asi
    708     + add %o2, %g1, %o0
    709     +ENDPROC(NG2_retl_o2_plus_g1_plus_1)
    710     +ENTRY(NG2_retl_o2_and_7_plus_o4)
    711     + and %o2, 7, %o2
    712     + ba,pt %xcc, __restore_asi
    713     + add %o2, %o4, %o0
    714     +ENDPROC(NG2_retl_o2_and_7_plus_o4)
    715     +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8)
    716     + and %o2, 7, %o2
    717     + add %o4, 8, %o4
    718     + ba,pt %xcc, __restore_asi
    719     + add %o2, %o4, %o0
    720     +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8)
    721     +#endif
    722     +
    723     .align 64
    724    
    725     .globl FUNC_NAME
    726     @@ -230,8 +292,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    727     sub %g0, %o4, %o4 ! bytes to align dst
    728     sub %o2, %o4, %o2
    729     1: subcc %o4, 1, %o4
    730     - EX_LD(LOAD(ldub, %o1, %g1))
    731     - EX_ST(STORE(stb, %g1, %o0))
    732     + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1)
    733     + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1)
    734     add %o1, 1, %o1
    735     bne,pt %XCC, 1b
    736     add %o0, 1, %o0
    737     @@ -281,11 +343,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    738     nop
    739     /* fall through for 0 < low bits < 8 */
    740     110: sub %o4, 64, %g2
    741     - EX_LD_FP(LOAD_BLK(%g2, %f0))
    742     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    743     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    744     + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1)
    745     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    746     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    747     FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
    748     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    749     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    750     FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
    751     subcc %g1, 64, %g1
    752     add %o4, 64, %o4
    753     @@ -296,10 +358,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    754    
    755     120: sub %o4, 56, %g2
    756     FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
    757     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    758     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    759     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    760     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    761     FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
    762     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    763     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    764     FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
    765     subcc %g1, 64, %g1
    766     add %o4, 64, %o4
    767     @@ -310,10 +372,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    768    
    769     130: sub %o4, 48, %g2
    770     FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
    771     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    772     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    773     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    774     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    775     FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
    776     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    777     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    778     FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
    779     subcc %g1, 64, %g1
    780     add %o4, 64, %o4
    781     @@ -324,10 +386,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    782    
    783     140: sub %o4, 40, %g2
    784     FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
    785     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    786     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    787     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    788     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    789     FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
    790     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    791     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    792     FREG_MOVE_5(f22, f24, f26, f28, f30)
    793     subcc %g1, 64, %g1
    794     add %o4, 64, %o4
    795     @@ -338,10 +400,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    796    
    797     150: sub %o4, 32, %g2
    798     FREG_LOAD_4(%g2, f0, f2, f4, f6)
    799     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    800     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    801     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    802     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    803     FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
    804     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    805     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    806     FREG_MOVE_4(f24, f26, f28, f30)
    807     subcc %g1, 64, %g1
    808     add %o4, 64, %o4
    809     @@ -352,10 +414,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    810    
    811     160: sub %o4, 24, %g2
    812     FREG_LOAD_3(%g2, f0, f2, f4)
    813     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    814     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    815     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    816     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    817     FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
    818     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    819     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    820     FREG_MOVE_3(f26, f28, f30)
    821     subcc %g1, 64, %g1
    822     add %o4, 64, %o4
    823     @@ -366,10 +428,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    824    
    825     170: sub %o4, 16, %g2
    826     FREG_LOAD_2(%g2, f0, f2)
    827     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    828     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    829     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    830     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    831     FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
    832     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    833     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    834     FREG_MOVE_2(f28, f30)
    835     subcc %g1, 64, %g1
    836     add %o4, 64, %o4
    837     @@ -380,10 +442,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    838    
    839     180: sub %o4, 8, %g2
    840     FREG_LOAD_1(%g2, f0)
    841     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    842     - EX_LD_FP(LOAD_BLK(%o4, %f16))
    843     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    844     + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    845     FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
    846     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    847     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    848     FREG_MOVE_1(f30)
    849     subcc %g1, 64, %g1
    850     add %o4, 64, %o4
    851     @@ -393,10 +455,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    852     nop
    853    
    854     190:
    855     -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    856     +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    857     subcc %g1, 64, %g1
    858     - EX_LD_FP(LOAD_BLK(%o4, %f0))
    859     - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    860     + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64)
    861     + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64)
    862     add %o4, 64, %o4
    863     bne,pt %xcc, 1b
    864     LOAD(prefetch, %o4 + 64, #one_read)
    865     @@ -423,28 +485,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    866     andn %o2, 0xf, %o4
    867     and %o2, 0xf, %o2
    868     1: subcc %o4, 0x10, %o4
    869     - EX_LD(LOAD(ldx, %o1, %o5))
    870     + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16)
    871     add %o1, 0x08, %o1
    872     - EX_LD(LOAD(ldx, %o1, %g1))
    873     + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16)
    874     sub %o1, 0x08, %o1
    875     - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
    876     + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16)
    877     add %o1, 0x8, %o1
    878     - EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE))
    879     + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8)
    880     bgu,pt %XCC, 1b
    881     add %o1, 0x8, %o1
    882     73: andcc %o2, 0x8, %g0
    883     be,pt %XCC, 1f
    884     nop
    885     sub %o2, 0x8, %o2
    886     - EX_LD(LOAD(ldx, %o1, %o5))
    887     - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
    888     + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8)
    889     + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8)
    890     add %o1, 0x8, %o1
    891     1: andcc %o2, 0x4, %g0
    892     be,pt %XCC, 1f
    893     nop
    894     sub %o2, 0x4, %o2
    895     - EX_LD(LOAD(lduw, %o1, %o5))
    896     - EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE))
    897     + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4)
    898     + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
    899     add %o1, 0x4, %o1
    900     1: cmp %o2, 0
    901     be,pt %XCC, 85f
    902     @@ -460,8 +522,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    903     sub %o2, %g1, %o2
    904    
    905     1: subcc %g1, 1, %g1
    906     - EX_LD(LOAD(ldub, %o1, %o5))
    907     - EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE))
    908     + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1)
    909     + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1)
    910     bgu,pt %icc, 1b
    911     add %o1, 1, %o1
    912    
    913     @@ -477,16 +539,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    914    
    915     8: mov 64, GLOBAL_SPARE
    916     andn %o1, 0x7, %o1
    917     - EX_LD(LOAD(ldx, %o1, %g2))
    918     + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2)
    919     sub GLOBAL_SPARE, %g1, GLOBAL_SPARE
    920     andn %o2, 0x7, %o4
    921     sllx %g2, %g1, %g2
    922     1: add %o1, 0x8, %o1
    923     - EX_LD(LOAD(ldx, %o1, %g3))
    924     + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4)
    925     subcc %o4, 0x8, %o4
    926     srlx %g3, GLOBAL_SPARE, %o5
    927     or %o5, %g2, %o5
    928     - EX_ST(STORE(stx, %o5, %o0))
    929     + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8)
    930     add %o0, 0x8, %o0
    931     bgu,pt %icc, 1b
    932     sllx %g3, %g1, %g2
    933     @@ -506,8 +568,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    934    
    935     1:
    936     subcc %o2, 4, %o2
    937     - EX_LD(LOAD(lduw, %o1, %g1))
    938     - EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE))
    939     + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4)
    940     + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
    941     bgu,pt %XCC, 1b
    942     add %o1, 4, %o1
    943    
    944     @@ -517,8 +579,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    945     .align 32
    946     90:
    947     subcc %o2, 1, %o2
    948     - EX_LD(LOAD(ldub, %o1, %g1))
    949     - EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE))
    950     + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1)
    951     + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1)
    952     bgu,pt %XCC, 90b
    953     add %o1, 1, %o1
    954     retl
    955     diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S
    956     index 2e8ee7ad07a9..16a286c1a528 100644
    957     --- a/arch/sparc/lib/NG4copy_from_user.S
    958     +++ b/arch/sparc/lib/NG4copy_from_user.S
    959     @@ -3,19 +3,19 @@
    960     * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
    961     */
    962    
    963     -#define EX_LD(x) \
    964     +#define EX_LD(x, y) \
    965     98: x; \
    966     .section __ex_table,"a";\
    967     .align 4; \
    968     - .word 98b, __retl_one_asi;\
    969     + .word 98b, y; \
    970     .text; \
    971     .align 4;
    972    
    973     -#define EX_LD_FP(x) \
    974     +#define EX_LD_FP(x,y) \
    975     98: x; \
    976     .section __ex_table,"a";\
    977     .align 4; \
    978     - .word 98b, __retl_one_asi_fp;\
    979     + .word 98b, y##_fp; \
    980     .text; \
    981     .align 4;
    982    
    983     diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S
    984     index be0bf4590df8..6b0276ffc858 100644
    985     --- a/arch/sparc/lib/NG4copy_to_user.S
    986     +++ b/arch/sparc/lib/NG4copy_to_user.S
    987     @@ -3,19 +3,19 @@
    988     * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
    989     */
    990    
    991     -#define EX_ST(x) \
    992     +#define EX_ST(x,y) \
    993     98: x; \
    994     .section __ex_table,"a";\
    995     .align 4; \
    996     - .word 98b, __retl_one_asi;\
    997     + .word 98b, y; \
    998     .text; \
    999     .align 4;
    1000    
    1001     -#define EX_ST_FP(x) \
    1002     +#define EX_ST_FP(x,y) \
    1003     98: x; \
    1004     .section __ex_table,"a";\
    1005     .align 4; \
    1006     - .word 98b, __retl_one_asi_fp;\
    1007     + .word 98b, y##_fp; \
    1008     .text; \
    1009     .align 4;
    1010    
    1011     diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S
    1012     index 8e13ee1f4454..75bb93b1437f 100644
    1013     --- a/arch/sparc/lib/NG4memcpy.S
    1014     +++ b/arch/sparc/lib/NG4memcpy.S
    1015     @@ -4,6 +4,7 @@
    1016     */
    1017    
    1018     #ifdef __KERNEL__
    1019     +#include <linux/linkage.h>
    1020     #include <asm/visasm.h>
    1021     #include <asm/asi.h>
    1022     #define GLOBAL_SPARE %g7
    1023     @@ -46,22 +47,19 @@
    1024     #endif
    1025    
    1026     #ifndef EX_LD
    1027     -#define EX_LD(x) x
    1028     +#define EX_LD(x,y) x
    1029     #endif
    1030     #ifndef EX_LD_FP
    1031     -#define EX_LD_FP(x) x
    1032     +#define EX_LD_FP(x,y) x
    1033     #endif
    1034    
    1035     #ifndef EX_ST
    1036     -#define EX_ST(x) x
    1037     +#define EX_ST(x,y) x
    1038     #endif
    1039     #ifndef EX_ST_FP
    1040     -#define EX_ST_FP(x) x
    1041     +#define EX_ST_FP(x,y) x
    1042     #endif
    1043    
    1044     -#ifndef EX_RETVAL
    1045     -#define EX_RETVAL(x) x
    1046     -#endif
    1047    
    1048     #ifndef LOAD
    1049     #define LOAD(type,addr,dest) type [addr], dest
    1050     @@ -94,6 +92,158 @@
    1051     .register %g3,#scratch
    1052    
    1053     .text
    1054     +#ifndef EX_RETVAL
    1055     +#define EX_RETVAL(x) x
    1056     +__restore_asi_fp:
    1057     + VISExitHalf
    1058     +__restore_asi:
    1059     + retl
    1060     + wr %g0, ASI_AIUS, %asi
    1061     +
    1062     +ENTRY(NG4_retl_o2)
    1063     + ba,pt %xcc, __restore_asi
    1064     + mov %o2, %o0
    1065     +ENDPROC(NG4_retl_o2)
    1066     +ENTRY(NG4_retl_o2_plus_1)
    1067     + ba,pt %xcc, __restore_asi
    1068     + add %o2, 1, %o0
    1069     +ENDPROC(NG4_retl_o2_plus_1)
    1070     +ENTRY(NG4_retl_o2_plus_4)
    1071     + ba,pt %xcc, __restore_asi
    1072     + add %o2, 4, %o0
    1073     +ENDPROC(NG4_retl_o2_plus_4)
    1074     +ENTRY(NG4_retl_o2_plus_o5)
    1075     + ba,pt %xcc, __restore_asi
    1076     + add %o2, %o5, %o0
    1077     +ENDPROC(NG4_retl_o2_plus_o5)
    1078     +ENTRY(NG4_retl_o2_plus_o5_plus_4)
    1079     + add %o5, 4, %o5
    1080     + ba,pt %xcc, __restore_asi
    1081     + add %o2, %o5, %o0
    1082     +ENDPROC(NG4_retl_o2_plus_o5_plus_4)
    1083     +ENTRY(NG4_retl_o2_plus_o5_plus_8)
    1084     + add %o5, 8, %o5
    1085     + ba,pt %xcc, __restore_asi
    1086     + add %o2, %o5, %o0
    1087     +ENDPROC(NG4_retl_o2_plus_o5_plus_8)
    1088     +ENTRY(NG4_retl_o2_plus_o5_plus_16)
    1089     + add %o5, 16, %o5
    1090     + ba,pt %xcc, __restore_asi
    1091     + add %o2, %o5, %o0
    1092     +ENDPROC(NG4_retl_o2_plus_o5_plus_16)
    1093     +ENTRY(NG4_retl_o2_plus_o5_plus_24)
    1094     + add %o5, 24, %o5
    1095     + ba,pt %xcc, __restore_asi
    1096     + add %o2, %o5, %o0
    1097     +ENDPROC(NG4_retl_o2_plus_o5_plus_24)
    1098     +ENTRY(NG4_retl_o2_plus_o5_plus_32)
    1099     + add %o5, 32, %o5
    1100     + ba,pt %xcc, __restore_asi
    1101     + add %o2, %o5, %o0
    1102     +ENDPROC(NG4_retl_o2_plus_o5_plus_32)
    1103     +ENTRY(NG4_retl_o2_plus_g1)
    1104     + ba,pt %xcc, __restore_asi
    1105     + add %o2, %g1, %o0
    1106     +ENDPROC(NG4_retl_o2_plus_g1)
    1107     +ENTRY(NG4_retl_o2_plus_g1_plus_1)
    1108     + add %g1, 1, %g1
    1109     + ba,pt %xcc, __restore_asi
    1110     + add %o2, %g1, %o0
    1111     +ENDPROC(NG4_retl_o2_plus_g1_plus_1)
    1112     +ENTRY(NG4_retl_o2_plus_g1_plus_8)
    1113     + add %g1, 8, %g1
    1114     + ba,pt %xcc, __restore_asi
    1115     + add %o2, %g1, %o0
    1116     +ENDPROC(NG4_retl_o2_plus_g1_plus_8)
    1117     +ENTRY(NG4_retl_o2_plus_o4)
    1118     + ba,pt %xcc, __restore_asi
    1119     + add %o2, %o4, %o0
    1120     +ENDPROC(NG4_retl_o2_plus_o4)
    1121     +ENTRY(NG4_retl_o2_plus_o4_plus_8)
    1122     + add %o4, 8, %o4
    1123     + ba,pt %xcc, __restore_asi
    1124     + add %o2, %o4, %o0
    1125     +ENDPROC(NG4_retl_o2_plus_o4_plus_8)
    1126     +ENTRY(NG4_retl_o2_plus_o4_plus_16)
    1127     + add %o4, 16, %o4
    1128     + ba,pt %xcc, __restore_asi
    1129     + add %o2, %o4, %o0
    1130     +ENDPROC(NG4_retl_o2_plus_o4_plus_16)
    1131     +ENTRY(NG4_retl_o2_plus_o4_plus_24)
    1132     + add %o4, 24, %o4
    1133     + ba,pt %xcc, __restore_asi
    1134     + add %o2, %o4, %o0
    1135     +ENDPROC(NG4_retl_o2_plus_o4_plus_24)
    1136     +ENTRY(NG4_retl_o2_plus_o4_plus_32)
    1137     + add %o4, 32, %o4
    1138     + ba,pt %xcc, __restore_asi
    1139     + add %o2, %o4, %o0
    1140     +ENDPROC(NG4_retl_o2_plus_o4_plus_32)
    1141     +ENTRY(NG4_retl_o2_plus_o4_plus_40)
    1142     + add %o4, 40, %o4
    1143     + ba,pt %xcc, __restore_asi
    1144     + add %o2, %o4, %o0
    1145     +ENDPROC(NG4_retl_o2_plus_o4_plus_40)
    1146     +ENTRY(NG4_retl_o2_plus_o4_plus_48)
    1147     + add %o4, 48, %o4
    1148     + ba,pt %xcc, __restore_asi
    1149     + add %o2, %o4, %o0
    1150     +ENDPROC(NG4_retl_o2_plus_o4_plus_48)
    1151     +ENTRY(NG4_retl_o2_plus_o4_plus_56)
    1152     + add %o4, 56, %o4
    1153     + ba,pt %xcc, __restore_asi
    1154     + add %o2, %o4, %o0
    1155     +ENDPROC(NG4_retl_o2_plus_o4_plus_56)
    1156     +ENTRY(NG4_retl_o2_plus_o4_plus_64)
    1157     + add %o4, 64, %o4
    1158     + ba,pt %xcc, __restore_asi
    1159     + add %o2, %o4, %o0
    1160     +ENDPROC(NG4_retl_o2_plus_o4_plus_64)
    1161     +ENTRY(NG4_retl_o2_plus_o4_fp)
    1162     + ba,pt %xcc, __restore_asi_fp
    1163     + add %o2, %o4, %o0
    1164     +ENDPROC(NG4_retl_o2_plus_o4_fp)
    1165     +ENTRY(NG4_retl_o2_plus_o4_plus_8_fp)
    1166     + add %o4, 8, %o4
    1167     + ba,pt %xcc, __restore_asi_fp
    1168     + add %o2, %o4, %o0
    1169     +ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp)
    1170     +ENTRY(NG4_retl_o2_plus_o4_plus_16_fp)
    1171     + add %o4, 16, %o4
    1172     + ba,pt %xcc, __restore_asi_fp
    1173     + add %o2, %o4, %o0
    1174     +ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp)
    1175     +ENTRY(NG4_retl_o2_plus_o4_plus_24_fp)
    1176     + add %o4, 24, %o4
    1177     + ba,pt %xcc, __restore_asi_fp
    1178     + add %o2, %o4, %o0
    1179     +ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp)
    1180     +ENTRY(NG4_retl_o2_plus_o4_plus_32_fp)
    1181     + add %o4, 32, %o4
    1182     + ba,pt %xcc, __restore_asi_fp
    1183     + add %o2, %o4, %o0
    1184     +ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp)
    1185     +ENTRY(NG4_retl_o2_plus_o4_plus_40_fp)
    1186     + add %o4, 40, %o4
    1187     + ba,pt %xcc, __restore_asi_fp
    1188     + add %o2, %o4, %o0
    1189     +ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp)
    1190     +ENTRY(NG4_retl_o2_plus_o4_plus_48_fp)
    1191     + add %o4, 48, %o4
    1192     + ba,pt %xcc, __restore_asi_fp
    1193     + add %o2, %o4, %o0
    1194     +ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp)
    1195     +ENTRY(NG4_retl_o2_plus_o4_plus_56_fp)
    1196     + add %o4, 56, %o4
    1197     + ba,pt %xcc, __restore_asi_fp
    1198     + add %o2, %o4, %o0
    1199     +ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp)
    1200     +ENTRY(NG4_retl_o2_plus_o4_plus_64_fp)
    1201     + add %o4, 64, %o4
    1202     + ba,pt %xcc, __restore_asi_fp
    1203     + add %o2, %o4, %o0
    1204     +ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp)
    1205     +#endif
    1206     .align 64
    1207    
    1208     .globl FUNC_NAME
    1209     @@ -124,12 +274,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1210     brz,pt %g1, 51f
    1211     sub %o2, %g1, %o2
    1212    
    1213     -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
    1214     +
    1215     +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1216     add %o1, 1, %o1
    1217     subcc %g1, 1, %g1
    1218     add %o0, 1, %o0
    1219     bne,pt %icc, 1b
    1220     - EX_ST(STORE(stb, %g2, %o0 - 0x01))
    1221     + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
    1222    
    1223     51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong)
    1224     LOAD(prefetch, %o1 + 0x080, #n_reads_strong)
    1225     @@ -154,43 +305,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1226     brz,pt %g1, .Llarge_aligned
    1227     sub %o2, %g1, %o2
    1228    
    1229     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2))
    1230     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1231     add %o1, 8, %o1
    1232     subcc %g1, 8, %g1
    1233     add %o0, 8, %o0
    1234     bne,pt %icc, 1b
    1235     - EX_ST(STORE(stx, %g2, %o0 - 0x08))
    1236     + EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8)
    1237    
    1238     .Llarge_aligned:
    1239     /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */
    1240     andn %o2, 0x3f, %o4
    1241     sub %o2, %o4, %o2
    1242    
    1243     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1244     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4)
    1245     add %o1, 0x40, %o1
    1246     - EX_LD(LOAD(ldx, %o1 - 0x38, %g2))
    1247     + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4)
    1248     subcc %o4, 0x40, %o4
    1249     - EX_LD(LOAD(ldx, %o1 - 0x30, %g3))
    1250     - EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE))
    1251     - EX_LD(LOAD(ldx, %o1 - 0x20, %o5))
    1252     - EX_ST(STORE_INIT(%g1, %o0))
    1253     + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64)
    1254     + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64)
    1255     + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64)
    1256     + EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64)
    1257     add %o0, 0x08, %o0
    1258     - EX_ST(STORE_INIT(%g2, %o0))
    1259     + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56)
    1260     add %o0, 0x08, %o0
    1261     - EX_LD(LOAD(ldx, %o1 - 0x18, %g2))
    1262     - EX_ST(STORE_INIT(%g3, %o0))
    1263     + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48)
    1264     + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48)
    1265     add %o0, 0x08, %o0
    1266     - EX_LD(LOAD(ldx, %o1 - 0x10, %g3))
    1267     - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
    1268     + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40)
    1269     + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40)
    1270     add %o0, 0x08, %o0
    1271     - EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE))
    1272     - EX_ST(STORE_INIT(%o5, %o0))
    1273     + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32)
    1274     + EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32)
    1275     add %o0, 0x08, %o0
    1276     - EX_ST(STORE_INIT(%g2, %o0))
    1277     + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24)
    1278     add %o0, 0x08, %o0
    1279     - EX_ST(STORE_INIT(%g3, %o0))
    1280     + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16)
    1281     add %o0, 0x08, %o0
    1282     - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
    1283     + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8)
    1284     add %o0, 0x08, %o0
    1285     bne,pt %icc, 1b
    1286     LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
    1287     @@ -216,17 +367,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1288     sub %o2, %o4, %o2
    1289     alignaddr %o1, %g0, %g1
    1290     add %o1, %o4, %o1
    1291     - EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0))
    1292     -1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2))
    1293     + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4)
    1294     +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4)
    1295     subcc %o4, 0x40, %o4
    1296     - EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4))
    1297     - EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6))
    1298     - EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8))
    1299     - EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10))
    1300     - EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12))
    1301     - EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14))
    1302     + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64)
    1303     + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64)
    1304     + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64)
    1305     + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64)
    1306     + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64)
    1307     + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64)
    1308     faligndata %f0, %f2, %f16
    1309     - EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0))
    1310     + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64)
    1311     faligndata %f2, %f4, %f18
    1312     add %g1, 0x40, %g1
    1313     faligndata %f4, %f6, %f20
    1314     @@ -235,14 +386,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1315     faligndata %f10, %f12, %f26
    1316     faligndata %f12, %f14, %f28
    1317     faligndata %f14, %f0, %f30
    1318     - EX_ST_FP(STORE(std, %f16, %o0 + 0x00))
    1319     - EX_ST_FP(STORE(std, %f18, %o0 + 0x08))
    1320     - EX_ST_FP(STORE(std, %f20, %o0 + 0x10))
    1321     - EX_ST_FP(STORE(std, %f22, %o0 + 0x18))
    1322     - EX_ST_FP(STORE(std, %f24, %o0 + 0x20))
    1323     - EX_ST_FP(STORE(std, %f26, %o0 + 0x28))
    1324     - EX_ST_FP(STORE(std, %f28, %o0 + 0x30))
    1325     - EX_ST_FP(STORE(std, %f30, %o0 + 0x38))
    1326     + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64)
    1327     + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56)
    1328     + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48)
    1329     + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40)
    1330     + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32)
    1331     + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24)
    1332     + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16)
    1333     + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8)
    1334     add %o0, 0x40, %o0
    1335     bne,pt %icc, 1b
    1336     LOAD(prefetch, %g1 + 0x200, #n_reads_strong)
    1337     @@ -270,37 +421,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1338     andncc %o2, 0x20 - 1, %o5
    1339     be,pn %icc, 2f
    1340     sub %o2, %o5, %o2
    1341     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1342     - EX_LD(LOAD(ldx, %o1 + 0x08, %g2))
    1343     - EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE))
    1344     - EX_LD(LOAD(ldx, %o1 + 0x18, %o4))
    1345     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1346     + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5)
    1347     + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5)
    1348     + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5)
    1349     add %o1, 0x20, %o1
    1350     subcc %o5, 0x20, %o5
    1351     - EX_ST(STORE(stx, %g1, %o0 + 0x00))
    1352     - EX_ST(STORE(stx, %g2, %o0 + 0x08))
    1353     - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10))
    1354     - EX_ST(STORE(stx, %o4, %o0 + 0x18))
    1355     + EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32)
    1356     + EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24)
    1357     + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24)
    1358     + EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8)
    1359     bne,pt %icc, 1b
    1360     add %o0, 0x20, %o0
    1361     2: andcc %o2, 0x18, %o5
    1362     be,pt %icc, 3f
    1363     sub %o2, %o5, %o2
    1364     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1365     +
    1366     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1367     add %o1, 0x08, %o1
    1368     add %o0, 0x08, %o0
    1369     subcc %o5, 0x08, %o5
    1370     bne,pt %icc, 1b
    1371     - EX_ST(STORE(stx, %g1, %o0 - 0x08))
    1372     + EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8)
    1373     3: brz,pt %o2, .Lexit
    1374     cmp %o2, 0x04
    1375     bl,pn %icc, .Ltiny
    1376     nop
    1377     - EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
    1378     + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2)
    1379     add %o1, 0x04, %o1
    1380     add %o0, 0x04, %o0
    1381     subcc %o2, 0x04, %o2
    1382     bne,pn %icc, .Ltiny
    1383     - EX_ST(STORE(stw, %g1, %o0 - 0x04))
    1384     + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4)
    1385     ba,a,pt %icc, .Lexit
    1386     .Lmedium_unaligned:
    1387     /* First get dest 8 byte aligned. */
    1388     @@ -309,12 +461,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1389     brz,pt %g1, 2f
    1390     sub %o2, %g1, %o2
    1391    
    1392     -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
    1393     +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1394     add %o1, 1, %o1
    1395     subcc %g1, 1, %g1
    1396     add %o0, 1, %o0
    1397     bne,pt %icc, 1b
    1398     - EX_ST(STORE(stb, %g2, %o0 - 0x01))
    1399     + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
    1400     2:
    1401     and %o1, 0x7, %g1
    1402     brz,pn %g1, .Lmedium_noprefetch
    1403     @@ -322,16 +474,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1404     mov 64, %g2
    1405     sub %g2, %g1, %g2
    1406     andn %o1, 0x7, %o1
    1407     - EX_LD(LOAD(ldx, %o1 + 0x00, %o4))
    1408     + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2)
    1409     sllx %o4, %g1, %o4
    1410     andn %o2, 0x08 - 1, %o5
    1411     sub %o2, %o5, %o2
    1412     -1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3))
    1413     +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5)
    1414     add %o1, 0x08, %o1
    1415     subcc %o5, 0x08, %o5
    1416     srlx %g3, %g2, GLOBAL_SPARE
    1417     or GLOBAL_SPARE, %o4, GLOBAL_SPARE
    1418     - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00))
    1419     + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8)
    1420     add %o0, 0x08, %o0
    1421     bne,pt %icc, 1b
    1422     sllx %g3, %g1, %o4
    1423     @@ -342,17 +494,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1424     ba,pt %icc, .Lsmall_unaligned
    1425    
    1426     .Ltiny:
    1427     - EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
    1428     + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
    1429     subcc %o2, 1, %o2
    1430     be,pn %icc, .Lexit
    1431     - EX_ST(STORE(stb, %g1, %o0 + 0x00))
    1432     - EX_LD(LOAD(ldub, %o1 + 0x01, %g1))
    1433     + EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1)
    1434     + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2)
    1435     subcc %o2, 1, %o2
    1436     be,pn %icc, .Lexit
    1437     - EX_ST(STORE(stb, %g1, %o0 + 0x01))
    1438     - EX_LD(LOAD(ldub, %o1 + 0x02, %g1))
    1439     + EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1)
    1440     + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2)
    1441     ba,pt %icc, .Lexit
    1442     - EX_ST(STORE(stb, %g1, %o0 + 0x02))
    1443     + EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2)
    1444    
    1445     .Lsmall:
    1446     andcc %g2, 0x3, %g0
    1447     @@ -360,22 +512,22 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1448     andn %o2, 0x4 - 1, %o5
    1449     sub %o2, %o5, %o2
    1450     1:
    1451     - EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
    1452     + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1453     add %o1, 0x04, %o1
    1454     subcc %o5, 0x04, %o5
    1455     add %o0, 0x04, %o0
    1456     bne,pt %icc, 1b
    1457     - EX_ST(STORE(stw, %g1, %o0 - 0x04))
    1458     + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4)
    1459     brz,pt %o2, .Lexit
    1460     nop
    1461     ba,a,pt %icc, .Ltiny
    1462    
    1463     .Lsmall_unaligned:
    1464     -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
    1465     +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
    1466     add %o1, 1, %o1
    1467     add %o0, 1, %o0
    1468     subcc %o2, 1, %o2
    1469     bne,pt %icc, 1b
    1470     - EX_ST(STORE(stb, %g1, %o0 - 0x01))
    1471     + EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1)
    1472     ba,a,pt %icc, .Lexit
    1473     .size FUNC_NAME, .-FUNC_NAME
    1474     diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
    1475     index 5d1e4d1ac21e..9cd42fcbc781 100644
    1476     --- a/arch/sparc/lib/NGcopy_from_user.S
    1477     +++ b/arch/sparc/lib/NGcopy_from_user.S
    1478     @@ -3,11 +3,11 @@
    1479     * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
    1480     */
    1481    
    1482     -#define EX_LD(x) \
    1483     +#define EX_LD(x,y) \
    1484     98: x; \
    1485     .section __ex_table,"a";\
    1486     .align 4; \
    1487     - .word 98b, __ret_one_asi;\
    1488     + .word 98b, y; \
    1489     .text; \
    1490     .align 4;
    1491    
    1492     diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
    1493     index ff630dcb273c..5c358afd464e 100644
    1494     --- a/arch/sparc/lib/NGcopy_to_user.S
    1495     +++ b/arch/sparc/lib/NGcopy_to_user.S
    1496     @@ -3,11 +3,11 @@
    1497     * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
    1498     */
    1499    
    1500     -#define EX_ST(x) \
    1501     +#define EX_ST(x,y) \
    1502     98: x; \
    1503     .section __ex_table,"a";\
    1504     .align 4; \
    1505     - .word 98b, __ret_one_asi;\
    1506     + .word 98b, y; \
    1507     .text; \
    1508     .align 4;
    1509    
    1510     diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S
    1511     index 96a14caf6966..d88c4ed50a00 100644
    1512     --- a/arch/sparc/lib/NGmemcpy.S
    1513     +++ b/arch/sparc/lib/NGmemcpy.S
    1514     @@ -4,6 +4,7 @@
    1515     */
    1516    
    1517     #ifdef __KERNEL__
    1518     +#include <linux/linkage.h>
    1519     #include <asm/asi.h>
    1520     #include <asm/thread_info.h>
    1521     #define GLOBAL_SPARE %g7
    1522     @@ -27,15 +28,11 @@
    1523     #endif
    1524    
    1525     #ifndef EX_LD
    1526     -#define EX_LD(x) x
    1527     +#define EX_LD(x,y) x
    1528     #endif
    1529    
    1530     #ifndef EX_ST
    1531     -#define EX_ST(x) x
    1532     -#endif
    1533     -
    1534     -#ifndef EX_RETVAL
    1535     -#define EX_RETVAL(x) x
    1536     +#define EX_ST(x,y) x
    1537     #endif
    1538    
    1539     #ifndef LOAD
    1540     @@ -79,6 +76,92 @@
    1541     .register %g3,#scratch
    1542    
    1543     .text
    1544     +#ifndef EX_RETVAL
    1545     +#define EX_RETVAL(x) x
    1546     +__restore_asi:
    1547     + ret
    1548     + wr %g0, ASI_AIUS, %asi
    1549     + restore
    1550     +ENTRY(NG_ret_i2_plus_i4_plus_1)
    1551     + ba,pt %xcc, __restore_asi
    1552     + add %i2, %i5, %i0
    1553     +ENDPROC(NG_ret_i2_plus_i4_plus_1)
    1554     +ENTRY(NG_ret_i2_plus_g1)
    1555     + ba,pt %xcc, __restore_asi
    1556     + add %i2, %g1, %i0
    1557     +ENDPROC(NG_ret_i2_plus_g1)
    1558     +ENTRY(NG_ret_i2_plus_g1_minus_8)
    1559     + sub %g1, 8, %g1
    1560     + ba,pt %xcc, __restore_asi
    1561     + add %i2, %g1, %i0
    1562     +ENDPROC(NG_ret_i2_plus_g1_minus_8)
    1563     +ENTRY(NG_ret_i2_plus_g1_minus_16)
    1564     + sub %g1, 16, %g1
    1565     + ba,pt %xcc, __restore_asi
    1566     + add %i2, %g1, %i0
    1567     +ENDPROC(NG_ret_i2_plus_g1_minus_16)
    1568     +ENTRY(NG_ret_i2_plus_g1_minus_24)
    1569     + sub %g1, 24, %g1
    1570     + ba,pt %xcc, __restore_asi
    1571     + add %i2, %g1, %i0
    1572     +ENDPROC(NG_ret_i2_plus_g1_minus_24)
    1573     +ENTRY(NG_ret_i2_plus_g1_minus_32)
    1574     + sub %g1, 32, %g1
    1575     + ba,pt %xcc, __restore_asi
    1576     + add %i2, %g1, %i0
    1577     +ENDPROC(NG_ret_i2_plus_g1_minus_32)
    1578     +ENTRY(NG_ret_i2_plus_g1_minus_40)
    1579     + sub %g1, 40, %g1
    1580     + ba,pt %xcc, __restore_asi
    1581     + add %i2, %g1, %i0
    1582     +ENDPROC(NG_ret_i2_plus_g1_minus_40)
    1583     +ENTRY(NG_ret_i2_plus_g1_minus_48)
    1584     + sub %g1, 48, %g1
    1585     + ba,pt %xcc, __restore_asi
    1586     + add %i2, %g1, %i0
    1587     +ENDPROC(NG_ret_i2_plus_g1_minus_48)
    1588     +ENTRY(NG_ret_i2_plus_g1_minus_56)
    1589     + sub %g1, 56, %g1
    1590     + ba,pt %xcc, __restore_asi
    1591     + add %i2, %g1, %i0
    1592     +ENDPROC(NG_ret_i2_plus_g1_minus_56)
    1593     +ENTRY(NG_ret_i2_plus_i4)
    1594     + ba,pt %xcc, __restore_asi
    1595     + add %i2, %i4, %i0
    1596     +ENDPROC(NG_ret_i2_plus_i4)
    1597     +ENTRY(NG_ret_i2_plus_i4_minus_8)
    1598     + sub %i4, 8, %i4
    1599     + ba,pt %xcc, __restore_asi
    1600     + add %i2, %i4, %i0
    1601     +ENDPROC(NG_ret_i2_plus_i4_minus_8)
    1602     +ENTRY(NG_ret_i2_plus_8)
    1603     + ba,pt %xcc, __restore_asi
    1604     + add %i2, 8, %i0
    1605     +ENDPROC(NG_ret_i2_plus_8)
    1606     +ENTRY(NG_ret_i2_plus_4)
    1607     + ba,pt %xcc, __restore_asi
    1608     + add %i2, 4, %i0
    1609     +ENDPROC(NG_ret_i2_plus_4)
    1610     +ENTRY(NG_ret_i2_plus_1)
    1611     + ba,pt %xcc, __restore_asi
    1612     + add %i2, 1, %i0
    1613     +ENDPROC(NG_ret_i2_plus_1)
    1614     +ENTRY(NG_ret_i2_plus_g1_plus_1)
    1615     + add %g1, 1, %g1
    1616     + ba,pt %xcc, __restore_asi
    1617     + add %i2, %g1, %i0
    1618     +ENDPROC(NG_ret_i2_plus_g1_plus_1)
    1619     +ENTRY(NG_ret_i2)
    1620     + ba,pt %xcc, __restore_asi
    1621     + mov %i2, %i0
    1622     +ENDPROC(NG_ret_i2)
    1623     +ENTRY(NG_ret_i2_and_7_plus_i4)
    1624     + and %i2, 7, %i2
    1625     + ba,pt %xcc, __restore_asi
    1626     + add %i2, %i4, %i0
    1627     +ENDPROC(NG_ret_i2_and_7_plus_i4)
    1628     +#endif
    1629     +
    1630     .align 64
    1631    
    1632     .globl FUNC_NAME
    1633     @@ -126,8 +209,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1634     sub %g0, %i4, %i4 ! bytes to align dst
    1635     sub %i2, %i4, %i2
    1636     1: subcc %i4, 1, %i4
    1637     - EX_LD(LOAD(ldub, %i1, %g1))
    1638     - EX_ST(STORE(stb, %g1, %o0))
    1639     + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1)
    1640     + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1)
    1641     add %i1, 1, %i1
    1642     bne,pt %XCC, 1b
    1643     add %o0, 1, %o0
    1644     @@ -160,7 +243,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1645     and %i4, 0x7, GLOBAL_SPARE
    1646     sll GLOBAL_SPARE, 3, GLOBAL_SPARE
    1647     mov 64, %i5
    1648     - EX_LD(LOAD_TWIN(%i1, %g2, %g3))
    1649     + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1)
    1650     sub %i5, GLOBAL_SPARE, %i5
    1651     mov 16, %o4
    1652     mov 32, %o5
    1653     @@ -178,31 +261,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1654     srlx WORD3, PRE_SHIFT, TMP; \
    1655     or WORD2, TMP, WORD2;
    1656    
    1657     -8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
    1658     +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
    1659     MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
    1660     LOAD(prefetch, %i1 + %i3, #one_read)
    1661    
    1662     - EX_ST(STORE_INIT(%g2, %o0 + 0x00))
    1663     - EX_ST(STORE_INIT(%g3, %o0 + 0x08))
    1664     + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1)
    1665     + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1666    
    1667     - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
    1668     + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
    1669     MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
    1670    
    1671     - EX_ST(STORE_INIT(%o2, %o0 + 0x10))
    1672     - EX_ST(STORE_INIT(%o3, %o0 + 0x18))
    1673     + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1674     + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1675    
    1676     - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1677     + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1678     MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
    1679    
    1680     - EX_ST(STORE_INIT(%g2, %o0 + 0x20))
    1681     - EX_ST(STORE_INIT(%g3, %o0 + 0x28))
    1682     + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1683     + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1684    
    1685     - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
    1686     + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
    1687     add %i1, 64, %i1
    1688     MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
    1689    
    1690     - EX_ST(STORE_INIT(%o2, %o0 + 0x30))
    1691     - EX_ST(STORE_INIT(%o3, %o0 + 0x38))
    1692     + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1693     + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1694    
    1695     subcc %g1, 64, %g1
    1696     bne,pt %XCC, 8b
    1697     @@ -211,31 +294,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1698     ba,pt %XCC, 60f
    1699     add %i1, %i4, %i1
    1700    
    1701     -9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
    1702     +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
    1703     MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
    1704     LOAD(prefetch, %i1 + %i3, #one_read)
    1705    
    1706     - EX_ST(STORE_INIT(%g3, %o0 + 0x00))
    1707     - EX_ST(STORE_INIT(%o2, %o0 + 0x08))
    1708     + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1)
    1709     + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1710    
    1711     - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
    1712     + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
    1713     MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
    1714    
    1715     - EX_ST(STORE_INIT(%o3, %o0 + 0x10))
    1716     - EX_ST(STORE_INIT(%g2, %o0 + 0x18))
    1717     + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1718     + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1719    
    1720     - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1721     + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1722     MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
    1723    
    1724     - EX_ST(STORE_INIT(%g3, %o0 + 0x20))
    1725     - EX_ST(STORE_INIT(%o2, %o0 + 0x28))
    1726     + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1727     + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1728    
    1729     - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
    1730     + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
    1731     add %i1, 64, %i1
    1732     MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
    1733    
    1734     - EX_ST(STORE_INIT(%o3, %o0 + 0x30))
    1735     - EX_ST(STORE_INIT(%g2, %o0 + 0x38))
    1736     + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1737     + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1738    
    1739     subcc %g1, 64, %g1
    1740     bne,pt %XCC, 9b
    1741     @@ -249,25 +332,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1742     * one twin load ahead, then add 8 back into source when
    1743     * we finish the loop.
    1744     */
    1745     - EX_LD(LOAD_TWIN(%i1, %o4, %o5))
    1746     + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1)
    1747     mov 16, %o7
    1748     mov 32, %g2
    1749     mov 48, %g3
    1750     mov 64, %o1
    1751     -1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1752     +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
    1753     LOAD(prefetch, %i1 + %o1, #one_read)
    1754     - EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line
    1755     - EX_ST(STORE_INIT(%o2, %o0 + 0x08))
    1756     - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
    1757     - EX_ST(STORE_INIT(%o3, %o0 + 0x10))
    1758     - EX_ST(STORE_INIT(%o4, %o0 + 0x18))
    1759     - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
    1760     - EX_ST(STORE_INIT(%o5, %o0 + 0x20))
    1761     - EX_ST(STORE_INIT(%o2, %o0 + 0x28))
    1762     - EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5))
    1763     + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line
    1764     + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1765     + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
    1766     + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1767     + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1768     + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1769     + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1770     + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1771     + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48)
    1772     add %i1, 64, %i1
    1773     - EX_ST(STORE_INIT(%o3, %o0 + 0x30))
    1774     - EX_ST(STORE_INIT(%o4, %o0 + 0x38))
    1775     + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1776     + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1777     subcc %g1, 64, %g1
    1778     bne,pt %XCC, 1b
    1779     add %o0, 64, %o0
    1780     @@ -282,20 +365,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1781     mov 32, %g2
    1782     mov 48, %g3
    1783     mov 64, %o1
    1784     -1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5))
    1785     - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1786     +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1)
    1787     + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
    1788     LOAD(prefetch, %i1 + %o1, #one_read)
    1789     - EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line
    1790     - EX_ST(STORE_INIT(%o5, %o0 + 0x08))
    1791     - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
    1792     - EX_ST(STORE_INIT(%o2, %o0 + 0x10))
    1793     - EX_ST(STORE_INIT(%o3, %o0 + 0x18))
    1794     - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
    1795     + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line
    1796     + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1797     + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
    1798     + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1799     + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1800     + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1801     add %i1, 64, %i1
    1802     - EX_ST(STORE_INIT(%o4, %o0 + 0x20))
    1803     - EX_ST(STORE_INIT(%o5, %o0 + 0x28))
    1804     - EX_ST(STORE_INIT(%o2, %o0 + 0x30))
    1805     - EX_ST(STORE_INIT(%o3, %o0 + 0x38))
    1806     + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1807     + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1808     + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1809     + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1810     subcc %g1, 64, %g1
    1811     bne,pt %XCC, 1b
    1812     add %o0, 64, %o0
    1813     @@ -321,28 +404,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1814     andn %i2, 0xf, %i4
    1815     and %i2, 0xf, %i2
    1816     1: subcc %i4, 0x10, %i4
    1817     - EX_LD(LOAD(ldx, %i1, %o4))
    1818     + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4)
    1819     add %i1, 0x08, %i1
    1820     - EX_LD(LOAD(ldx, %i1, %g1))
    1821     + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4)
    1822     sub %i1, 0x08, %i1
    1823     - EX_ST(STORE(stx, %o4, %i1 + %i3))
    1824     + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4)
    1825     add %i1, 0x8, %i1
    1826     - EX_ST(STORE(stx, %g1, %i1 + %i3))
    1827     + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8)
    1828     bgu,pt %XCC, 1b
    1829     add %i1, 0x8, %i1
    1830     73: andcc %i2, 0x8, %g0
    1831     be,pt %XCC, 1f
    1832     nop
    1833     sub %i2, 0x8, %i2
    1834     - EX_LD(LOAD(ldx, %i1, %o4))
    1835     - EX_ST(STORE(stx, %o4, %i1 + %i3))
    1836     + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8)
    1837     + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8)
    1838     add %i1, 0x8, %i1
    1839     1: andcc %i2, 0x4, %g0
    1840     be,pt %XCC, 1f
    1841     nop
    1842     sub %i2, 0x4, %i2
    1843     - EX_LD(LOAD(lduw, %i1, %i5))
    1844     - EX_ST(STORE(stw, %i5, %i1 + %i3))
    1845     + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4)
    1846     + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4)
    1847     add %i1, 0x4, %i1
    1848     1: cmp %i2, 0
    1849     be,pt %XCC, 85f
    1850     @@ -358,8 +441,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1851     sub %i2, %g1, %i2
    1852    
    1853     1: subcc %g1, 1, %g1
    1854     - EX_LD(LOAD(ldub, %i1, %i5))
    1855     - EX_ST(STORE(stb, %i5, %i1 + %i3))
    1856     + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1)
    1857     + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1)
    1858     bgu,pt %icc, 1b
    1859     add %i1, 1, %i1
    1860    
    1861     @@ -375,16 +458,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1862    
    1863     8: mov 64, %i3
    1864     andn %i1, 0x7, %i1
    1865     - EX_LD(LOAD(ldx, %i1, %g2))
    1866     + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2)
    1867     sub %i3, %g1, %i3
    1868     andn %i2, 0x7, %i4
    1869     sllx %g2, %g1, %g2
    1870     1: add %i1, 0x8, %i1
    1871     - EX_LD(LOAD(ldx, %i1, %g3))
    1872     + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4)
    1873     subcc %i4, 0x8, %i4
    1874     srlx %g3, %i3, %i5
    1875     or %i5, %g2, %i5
    1876     - EX_ST(STORE(stx, %i5, %o0))
    1877     + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4)
    1878     add %o0, 0x8, %o0
    1879     bgu,pt %icc, 1b
    1880     sllx %g3, %g1, %g2
    1881     @@ -404,8 +487,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1882    
    1883     1:
    1884     subcc %i2, 4, %i2
    1885     - EX_LD(LOAD(lduw, %i1, %g1))
    1886     - EX_ST(STORE(stw, %g1, %i1 + %i3))
    1887     + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4)
    1888     + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4)
    1889     bgu,pt %XCC, 1b
    1890     add %i1, 4, %i1
    1891    
    1892     @@ -415,8 +498,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1893     .align 32
    1894     90:
    1895     subcc %i2, 1, %i2
    1896     - EX_LD(LOAD(ldub, %i1, %g1))
    1897     - EX_ST(STORE(stb, %g1, %i1 + %i3))
    1898     + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1)
    1899     + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1)
    1900     bgu,pt %XCC, 90b
    1901     add %i1, 1, %i1
    1902     ret
    1903     diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
    1904     index ecc5692fa2b4..bb6ff73229e3 100644
    1905     --- a/arch/sparc/lib/U1copy_from_user.S
    1906     +++ b/arch/sparc/lib/U1copy_from_user.S
    1907     @@ -3,19 +3,19 @@
    1908     * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    1909     */
    1910    
    1911     -#define EX_LD(x) \
    1912     +#define EX_LD(x,y) \
    1913     98: x; \
    1914     .section __ex_table,"a";\
    1915     .align 4; \
    1916     - .word 98b, __retl_one; \
    1917     + .word 98b, y; \
    1918     .text; \
    1919     .align 4;
    1920    
    1921     -#define EX_LD_FP(x) \
    1922     +#define EX_LD_FP(x,y) \
    1923     98: x; \
    1924     .section __ex_table,"a";\
    1925     .align 4; \
    1926     - .word 98b, __retl_one_fp;\
    1927     + .word 98b, y; \
    1928     .text; \
    1929     .align 4;
    1930    
    1931     diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
    1932     index 9eea392e44d4..ed92ce739558 100644
    1933     --- a/arch/sparc/lib/U1copy_to_user.S
    1934     +++ b/arch/sparc/lib/U1copy_to_user.S
    1935     @@ -3,19 +3,19 @@
    1936     * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    1937     */
    1938    
    1939     -#define EX_ST(x) \
    1940     +#define EX_ST(x,y) \
    1941     98: x; \
    1942     .section __ex_table,"a";\
    1943     .align 4; \
    1944     - .word 98b, __retl_one; \
    1945     + .word 98b, y; \
    1946     .text; \
    1947     .align 4;
    1948    
    1949     -#define EX_ST_FP(x) \
    1950     +#define EX_ST_FP(x,y) \
    1951     98: x; \
    1952     .section __ex_table,"a";\
    1953     .align 4; \
    1954     - .word 98b, __retl_one_fp;\
    1955     + .word 98b, y; \
    1956     .text; \
    1957     .align 4;
    1958    
    1959     diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S
    1960     index 3e6209ebb7d7..f30d2ab2c371 100644
    1961     --- a/arch/sparc/lib/U1memcpy.S
    1962     +++ b/arch/sparc/lib/U1memcpy.S
    1963     @@ -5,6 +5,7 @@
    1964     */
    1965    
    1966     #ifdef __KERNEL__
    1967     +#include <linux/linkage.h>
    1968     #include <asm/visasm.h>
    1969     #include <asm/asi.h>
    1970     #define GLOBAL_SPARE g7
    1971     @@ -23,21 +24,17 @@
    1972     #endif
    1973    
    1974     #ifndef EX_LD
    1975     -#define EX_LD(x) x
    1976     +#define EX_LD(x,y) x
    1977     #endif
    1978     #ifndef EX_LD_FP
    1979     -#define EX_LD_FP(x) x
    1980     +#define EX_LD_FP(x,y) x
    1981     #endif
    1982    
    1983     #ifndef EX_ST
    1984     -#define EX_ST(x) x
    1985     +#define EX_ST(x,y) x
    1986     #endif
    1987     #ifndef EX_ST_FP
    1988     -#define EX_ST_FP(x) x
    1989     -#endif
    1990     -
    1991     -#ifndef EX_RETVAL
    1992     -#define EX_RETVAL(x) x
    1993     +#define EX_ST_FP(x,y) x
    1994     #endif
    1995    
    1996     #ifndef LOAD
    1997     @@ -78,53 +75,169 @@
    1998     faligndata %f7, %f8, %f60; \
    1999     faligndata %f8, %f9, %f62;
    2000    
    2001     -#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
    2002     - EX_LD_FP(LOAD_BLK(%src, %fdest)); \
    2003     - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    2004     - add %src, 0x40, %src; \
    2005     - subcc %len, 0x40, %len; \
    2006     - be,pn %xcc, jmptgt; \
    2007     - add %dest, 0x40, %dest; \
    2008     -
    2009     -#define LOOP_CHUNK1(src, dest, len, branch_dest) \
    2010     - MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
    2011     -#define LOOP_CHUNK2(src, dest, len, branch_dest) \
    2012     - MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
    2013     -#define LOOP_CHUNK3(src, dest, len, branch_dest) \
    2014     - MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
    2015     +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \
    2016     + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \
    2017     + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
    2018     + add %src, 0x40, %src; \
    2019     + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \
    2020     + be,pn %xcc, jmptgt; \
    2021     + add %dest, 0x40, %dest; \
    2022     +
    2023     +#define LOOP_CHUNK1(src, dest, branch_dest) \
    2024     + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest)
    2025     +#define LOOP_CHUNK2(src, dest, branch_dest) \
    2026     + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest)
    2027     +#define LOOP_CHUNK3(src, dest, branch_dest) \
    2028     + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest)
    2029    
    2030     #define DO_SYNC membar #Sync;
    2031     #define STORE_SYNC(dest, fsrc) \
    2032     - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    2033     + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
    2034     add %dest, 0x40, %dest; \
    2035     DO_SYNC
    2036    
    2037     #define STORE_JUMP(dest, fsrc, target) \
    2038     - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    2039     + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \
    2040     add %dest, 0x40, %dest; \
    2041     ba,pt %xcc, target; \
    2042     nop;
    2043    
    2044     -#define FINISH_VISCHUNK(dest, f0, f1, left) \
    2045     - subcc %left, 8, %left;\
    2046     - bl,pn %xcc, 95f; \
    2047     - faligndata %f0, %f1, %f48; \
    2048     - EX_ST_FP(STORE(std, %f48, %dest)); \
    2049     +#define FINISH_VISCHUNK(dest, f0, f1) \
    2050     + subcc %g3, 8, %g3; \
    2051     + bl,pn %xcc, 95f; \
    2052     + faligndata %f0, %f1, %f48; \
    2053     + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \
    2054     add %dest, 8, %dest;
    2055    
    2056     -#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
    2057     - subcc %left, 8, %left; \
    2058     - bl,pn %xcc, 95f; \
    2059     +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
    2060     + subcc %g3, 8, %g3; \
    2061     + bl,pn %xcc, 95f; \
    2062     fsrc2 %f0, %f1;
    2063    
    2064     -#define UNEVEN_VISCHUNK(dest, f0, f1, left) \
    2065     - UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
    2066     +#define UNEVEN_VISCHUNK(dest, f0, f1) \
    2067     + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
    2068     ba,a,pt %xcc, 93f;
    2069    
    2070     .register %g2,#scratch
    2071     .register %g3,#scratch
    2072    
    2073     .text
    2074     +#ifndef EX_RETVAL
    2075     +#define EX_RETVAL(x) x
    2076     +ENTRY(U1_g1_1_fp)
    2077     + VISExitHalf
    2078     + add %g1, 1, %g1
    2079     + add %g1, %g2, %g1
    2080     + retl
    2081     + add %g1, %o2, %o0
    2082     +ENDPROC(U1_g1_1_fp)
    2083     +ENTRY(U1_g2_0_fp)
    2084     + VISExitHalf
    2085     + retl
    2086     + add %g2, %o2, %o0
    2087     +ENDPROC(U1_g2_0_fp)
    2088     +ENTRY(U1_g2_8_fp)
    2089     + VISExitHalf
    2090     + add %g2, 8, %g2
    2091     + retl
    2092     + add %g2, %o2, %o0
    2093     +ENDPROC(U1_g2_8_fp)
    2094     +ENTRY(U1_gs_0_fp)
    2095     + VISExitHalf
    2096     + add %GLOBAL_SPARE, %g3, %o0
    2097     + retl
    2098     + add %o0, %o2, %o0
    2099     +ENDPROC(U1_gs_0_fp)
    2100     +ENTRY(U1_gs_80_fp)
    2101     + VISExitHalf
    2102     + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
    2103     + add %GLOBAL_SPARE, %g3, %o0
    2104     + retl
    2105     + add %o0, %o2, %o0
    2106     +ENDPROC(U1_gs_80_fp)
    2107     +ENTRY(U1_gs_40_fp)
    2108     + VISExitHalf
    2109     + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE
    2110     + add %GLOBAL_SPARE, %g3, %o0
    2111     + retl
    2112     + add %o0, %o2, %o0
    2113     +ENDPROC(U1_gs_40_fp)
    2114     +ENTRY(U1_g3_0_fp)
    2115     + VISExitHalf
    2116     + retl
    2117     + add %g3, %o2, %o0
    2118     +ENDPROC(U1_g3_0_fp)
    2119     +ENTRY(U1_g3_8_fp)
    2120     + VISExitHalf
    2121     + add %g3, 8, %g3
    2122     + retl
    2123     + add %g3, %o2, %o0
    2124     +ENDPROC(U1_g3_8_fp)
    2125     +ENTRY(U1_o2_0_fp)
    2126     + VISExitHalf
    2127     + retl
    2128     + mov %o2, %o0
    2129     +ENDPROC(U1_o2_0_fp)
    2130     +ENTRY(U1_o2_1_fp)
    2131     + VISExitHalf
    2132     + retl
    2133     + add %o2, 1, %o0
    2134     +ENDPROC(U1_o2_1_fp)
    2135     +ENTRY(U1_gs_0)
    2136     + VISExitHalf
    2137     + retl
    2138     + add %GLOBAL_SPARE, %o2, %o0
    2139     +ENDPROC(U1_gs_0)
    2140     +ENTRY(U1_gs_8)
    2141     + VISExitHalf
    2142     + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
    2143     + retl
    2144     + add %GLOBAL_SPARE, 0x8, %o0
    2145     +ENDPROC(U1_gs_8)
    2146     +ENTRY(U1_gs_10)
    2147     + VISExitHalf
    2148     + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
    2149     + retl
    2150     + add %GLOBAL_SPARE, 0x10, %o0
    2151     +ENDPROC(U1_gs_10)
    2152     +ENTRY(U1_o2_0)
    2153     + retl
    2154     + mov %o2, %o0
    2155     +ENDPROC(U1_o2_0)
    2156     +ENTRY(U1_o2_8)
    2157     + retl
    2158     + add %o2, 8, %o0
    2159     +ENDPROC(U1_o2_8)
    2160     +ENTRY(U1_o2_4)
    2161     + retl
    2162     + add %o2, 4, %o0
    2163     +ENDPROC(U1_o2_4)
    2164     +ENTRY(U1_o2_1)
    2165     + retl
    2166     + add %o2, 1, %o0
    2167     +ENDPROC(U1_o2_1)
    2168     +ENTRY(U1_g1_0)
    2169     + retl
    2170     + add %g1, %o2, %o0
    2171     +ENDPROC(U1_g1_0)
    2172     +ENTRY(U1_g1_1)
    2173     + add %g1, 1, %g1
    2174     + retl
    2175     + add %g1, %o2, %o0
    2176     +ENDPROC(U1_g1_1)
    2177     +ENTRY(U1_gs_0_o2_adj)
    2178     + and %o2, 7, %o2
    2179     + retl
    2180     + add %GLOBAL_SPARE, %o2, %o0
    2181     +ENDPROC(U1_gs_0_o2_adj)
    2182     +ENTRY(U1_gs_8_o2_adj)
    2183     + and %o2, 7, %o2
    2184     + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE
    2185     + retl
    2186     + add %GLOBAL_SPARE, %o2, %o0
    2187     +ENDPROC(U1_gs_8_o2_adj)
    2188     +#endif
    2189     +
    2190     .align 64
    2191    
    2192     .globl FUNC_NAME
    2193     @@ -166,8 +279,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2194     and %g2, 0x38, %g2
    2195    
    2196     1: subcc %g1, 0x1, %g1
    2197     - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
    2198     - EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE))
    2199     + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp)
    2200     + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp)
    2201     bgu,pt %XCC, 1b
    2202     add %o1, 0x1, %o1
    2203    
    2204     @@ -178,20 +291,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2205     be,pt %icc, 3f
    2206     alignaddr %o1, %g0, %o1
    2207    
    2208     - EX_LD_FP(LOAD(ldd, %o1, %f4))
    2209     -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
    2210     + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp)
    2211     +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp)
    2212     add %o1, 0x8, %o1
    2213     subcc %g2, 0x8, %g2
    2214     faligndata %f4, %f6, %f0
    2215     - EX_ST_FP(STORE(std, %f0, %o0))
    2216     + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
    2217     be,pn %icc, 3f
    2218     add %o0, 0x8, %o0
    2219    
    2220     - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
    2221     + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp)
    2222     add %o1, 0x8, %o1
    2223     subcc %g2, 0x8, %g2
    2224     faligndata %f6, %f4, %f0
    2225     - EX_ST_FP(STORE(std, %f0, %o0))
    2226     + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
    2227     bne,pt %icc, 1b
    2228     add %o0, 0x8, %o0
    2229    
    2230     @@ -214,13 +327,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2231     add %g1, %GLOBAL_SPARE, %g1
    2232     subcc %o2, %g3, %o2
    2233    
    2234     - EX_LD_FP(LOAD_BLK(%o1, %f0))
    2235     + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp)
    2236     add %o1, 0x40, %o1
    2237     add %g1, %g3, %g1
    2238     - EX_LD_FP(LOAD_BLK(%o1, %f16))
    2239     + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp)
    2240     add %o1, 0x40, %o1
    2241     sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
    2242     - EX_LD_FP(LOAD_BLK(%o1, %f32))
    2243     + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp)
    2244     add %o1, 0x40, %o1
    2245    
    2246     /* There are 8 instances of the unrolled loop,
    2247     @@ -240,11 +353,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2248    
    2249     .align 64
    2250     1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
    2251     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2252     + LOOP_CHUNK1(o1, o0, 1f)
    2253     FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
    2254     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2255     + LOOP_CHUNK2(o1, o0, 2f)
    2256     FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
    2257     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2258     + LOOP_CHUNK3(o1, o0, 3f)
    2259     ba,pt %xcc, 1b+4
    2260     faligndata %f0, %f2, %f48
    2261     1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
    2262     @@ -261,11 +374,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2263     STORE_JUMP(o0, f48, 56f)
    2264    
    2265     1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
    2266     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2267     + LOOP_CHUNK1(o1, o0, 1f)
    2268     FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
    2269     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2270     + LOOP_CHUNK2(o1, o0, 2f)
    2271     FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
    2272     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2273     + LOOP_CHUNK3(o1, o0, 3f)
    2274     ba,pt %xcc, 1b+4
    2275     faligndata %f2, %f4, %f48
    2276     1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
    2277     @@ -282,11 +395,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2278     STORE_JUMP(o0, f48, 57f)
    2279    
    2280     1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
    2281     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2282     + LOOP_CHUNK1(o1, o0, 1f)
    2283     FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
    2284     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2285     + LOOP_CHUNK2(o1, o0, 2f)
    2286     FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
    2287     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2288     + LOOP_CHUNK3(o1, o0, 3f)
    2289     ba,pt %xcc, 1b+4
    2290     faligndata %f4, %f6, %f48
    2291     1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
    2292     @@ -303,11 +416,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2293     STORE_JUMP(o0, f48, 58f)
    2294    
    2295     1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
    2296     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2297     + LOOP_CHUNK1(o1, o0, 1f)
    2298     FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
    2299     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2300     + LOOP_CHUNK2(o1, o0, 2f)
    2301     FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
    2302     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2303     + LOOP_CHUNK3(o1, o0, 3f)
    2304     ba,pt %xcc, 1b+4
    2305     faligndata %f6, %f8, %f48
    2306     1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
    2307     @@ -324,11 +437,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2308     STORE_JUMP(o0, f48, 59f)
    2309    
    2310     1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
    2311     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2312     + LOOP_CHUNK1(o1, o0, 1f)
    2313     FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
    2314     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2315     + LOOP_CHUNK2(o1, o0, 2f)
    2316     FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
    2317     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2318     + LOOP_CHUNK3(o1, o0, 3f)
    2319     ba,pt %xcc, 1b+4
    2320     faligndata %f8, %f10, %f48
    2321     1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
    2322     @@ -345,11 +458,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2323     STORE_JUMP(o0, f48, 60f)
    2324    
    2325     1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
    2326     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2327     + LOOP_CHUNK1(o1, o0, 1f)
    2328     FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
    2329     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2330     + LOOP_CHUNK2(o1, o0, 2f)
    2331     FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
    2332     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2333     + LOOP_CHUNK3(o1, o0, 3f)
    2334     ba,pt %xcc, 1b+4
    2335     faligndata %f10, %f12, %f48
    2336     1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
    2337     @@ -366,11 +479,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2338     STORE_JUMP(o0, f48, 61f)
    2339    
    2340     1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
    2341     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2342     + LOOP_CHUNK1(o1, o0, 1f)
    2343     FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
    2344     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2345     + LOOP_CHUNK2(o1, o0, 2f)
    2346     FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
    2347     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2348     + LOOP_CHUNK3(o1, o0, 3f)
    2349     ba,pt %xcc, 1b+4
    2350     faligndata %f12, %f14, %f48
    2351     1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
    2352     @@ -387,11 +500,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2353     STORE_JUMP(o0, f48, 62f)
    2354    
    2355     1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
    2356     - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2357     + LOOP_CHUNK1(o1, o0, 1f)
    2358     FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2359     - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2360     + LOOP_CHUNK2(o1, o0, 2f)
    2361     FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
    2362     - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2363     + LOOP_CHUNK3(o1, o0, 3f)
    2364     ba,pt %xcc, 1b+4
    2365     faligndata %f14, %f16, %f48
    2366     1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2367     @@ -407,53 +520,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2368     FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2369     STORE_JUMP(o0, f48, 63f)
    2370    
    2371     -40: FINISH_VISCHUNK(o0, f0, f2, g3)
    2372     -41: FINISH_VISCHUNK(o0, f2, f4, g3)
    2373     -42: FINISH_VISCHUNK(o0, f4, f6, g3)
    2374     -43: FINISH_VISCHUNK(o0, f6, f8, g3)
    2375     -44: FINISH_VISCHUNK(o0, f8, f10, g3)
    2376     -45: FINISH_VISCHUNK(o0, f10, f12, g3)
    2377     -46: FINISH_VISCHUNK(o0, f12, f14, g3)
    2378     -47: UNEVEN_VISCHUNK(o0, f14, f0, g3)
    2379     -48: FINISH_VISCHUNK(o0, f16, f18, g3)
    2380     -49: FINISH_VISCHUNK(o0, f18, f20, g3)
    2381     -50: FINISH_VISCHUNK(o0, f20, f22, g3)
    2382     -51: FINISH_VISCHUNK(o0, f22, f24, g3)
    2383     -52: FINISH_VISCHUNK(o0, f24, f26, g3)
    2384     -53: FINISH_VISCHUNK(o0, f26, f28, g3)
    2385     -54: FINISH_VISCHUNK(o0, f28, f30, g3)
    2386     -55: UNEVEN_VISCHUNK(o0, f30, f0, g3)
    2387     -56: FINISH_VISCHUNK(o0, f32, f34, g3)
    2388     -57: FINISH_VISCHUNK(o0, f34, f36, g3)
    2389     -58: FINISH_VISCHUNK(o0, f36, f38, g3)
    2390     -59: FINISH_VISCHUNK(o0, f38, f40, g3)
    2391     -60: FINISH_VISCHUNK(o0, f40, f42, g3)
    2392     -61: FINISH_VISCHUNK(o0, f42, f44, g3)
    2393     -62: FINISH_VISCHUNK(o0, f44, f46, g3)
    2394     -63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
    2395     -
    2396     -93: EX_LD_FP(LOAD(ldd, %o1, %f2))
    2397     +40: FINISH_VISCHUNK(o0, f0, f2)
    2398     +41: FINISH_VISCHUNK(o0, f2, f4)
    2399     +42: FINISH_VISCHUNK(o0, f4, f6)
    2400     +43: FINISH_VISCHUNK(o0, f6, f8)
    2401     +44: FINISH_VISCHUNK(o0, f8, f10)
    2402     +45: FINISH_VISCHUNK(o0, f10, f12)
    2403     +46: FINISH_VISCHUNK(o0, f12, f14)
    2404     +47: UNEVEN_VISCHUNK(o0, f14, f0)
    2405     +48: FINISH_VISCHUNK(o0, f16, f18)
    2406     +49: FINISH_VISCHUNK(o0, f18, f20)
    2407     +50: FINISH_VISCHUNK(o0, f20, f22)
    2408     +51: FINISH_VISCHUNK(o0, f22, f24)
    2409     +52: FINISH_VISCHUNK(o0, f24, f26)
    2410     +53: FINISH_VISCHUNK(o0, f26, f28)
    2411     +54: FINISH_VISCHUNK(o0, f28, f30)
    2412     +55: UNEVEN_VISCHUNK(o0, f30, f0)
    2413     +56: FINISH_VISCHUNK(o0, f32, f34)
    2414     +57: FINISH_VISCHUNK(o0, f34, f36)
    2415     +58: FINISH_VISCHUNK(o0, f36, f38)
    2416     +59: FINISH_VISCHUNK(o0, f38, f40)
    2417     +60: FINISH_VISCHUNK(o0, f40, f42)
    2418     +61: FINISH_VISCHUNK(o0, f42, f44)
    2419     +62: FINISH_VISCHUNK(o0, f44, f46)
    2420     +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0)
    2421     +
    2422     +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp)
    2423     add %o1, 8, %o1
    2424     subcc %g3, 8, %g3
    2425     faligndata %f0, %f2, %f8
    2426     - EX_ST_FP(STORE(std, %f8, %o0))
    2427     + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
    2428     bl,pn %xcc, 95f
    2429     add %o0, 8, %o0
    2430     - EX_LD_FP(LOAD(ldd, %o1, %f0))
    2431     + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp)
    2432     add %o1, 8, %o1
    2433     subcc %g3, 8, %g3
    2434     faligndata %f2, %f0, %f8
    2435     - EX_ST_FP(STORE(std, %f8, %o0))
    2436     + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
    2437     bge,pt %xcc, 93b
    2438     add %o0, 8, %o0
    2439    
    2440     95: brz,pt %o2, 2f
    2441     mov %g1, %o1
    2442    
    2443     -1: EX_LD_FP(LOAD(ldub, %o1, %o3))
    2444     +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp)
    2445     add %o1, 1, %o1
    2446     subcc %o2, 1, %o2
    2447     - EX_ST_FP(STORE(stb, %o3, %o0))
    2448     + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp)
    2449     bne,pt %xcc, 1b
    2450     add %o0, 1, %o0
    2451    
    2452     @@ -469,27 +582,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2453    
    2454     72: andn %o2, 0xf, %GLOBAL_SPARE
    2455     and %o2, 0xf, %o2
    2456     -1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
    2457     - EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
    2458     +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0)
    2459     + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0)
    2460     subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
    2461     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2462     + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10)
    2463     add %o1, 0x8, %o1
    2464     - EX_ST(STORE(stx, %g1, %o1 + %o3))
    2465     + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8)
    2466     bgu,pt %XCC, 1b
    2467     add %o1, 0x8, %o1
    2468     73: andcc %o2, 0x8, %g0
    2469     be,pt %XCC, 1f
    2470     nop
    2471     - EX_LD(LOAD(ldx, %o1, %o5))
    2472     + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0)
    2473     sub %o2, 0x8, %o2
    2474     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2475     + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8)
    2476     add %o1, 0x8, %o1
    2477     1: andcc %o2, 0x4, %g0
    2478     be,pt %XCC, 1f
    2479     nop
    2480     - EX_LD(LOAD(lduw, %o1, %o5))
    2481     + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0)
    2482     sub %o2, 0x4, %o2
    2483     - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2484     + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4)
    2485     add %o1, 0x4, %o1
    2486     1: cmp %o2, 0
    2487     be,pt %XCC, 85f
    2488     @@ -503,9 +616,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2489     sub %g0, %g1, %g1
    2490     sub %o2, %g1, %o2
    2491    
    2492     -1: EX_LD(LOAD(ldub, %o1, %o5))
    2493     +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0)
    2494     subcc %g1, 1, %g1
    2495     - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2496     + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1)
    2497     bgu,pt %icc, 1b
    2498     add %o1, 1, %o1
    2499    
    2500     @@ -521,16 +634,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2501    
    2502     8: mov 64, %o3
    2503     andn %o1, 0x7, %o1
    2504     - EX_LD(LOAD(ldx, %o1, %g2))
    2505     + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0)
    2506     sub %o3, %g1, %o3
    2507     andn %o2, 0x7, %GLOBAL_SPARE
    2508     sllx %g2, %g1, %g2
    2509     -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
    2510     +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj)
    2511     subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
    2512     add %o1, 0x8, %o1
    2513     srlx %g3, %o3, %o5
    2514     or %o5, %g2, %o5
    2515     - EX_ST(STORE(stx, %o5, %o0))
    2516     + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj)
    2517     add %o0, 0x8, %o0
    2518     bgu,pt %icc, 1b
    2519     sllx %g3, %g1, %g2
    2520     @@ -548,9 +661,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2521     bne,pn %XCC, 90f
    2522     sub %o0, %o1, %o3
    2523    
    2524     -1: EX_LD(LOAD(lduw, %o1, %g1))
    2525     +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0)
    2526     subcc %o2, 4, %o2
    2527     - EX_ST(STORE(stw, %g1, %o1 + %o3))
    2528     + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4)
    2529     bgu,pt %XCC, 1b
    2530     add %o1, 4, %o1
    2531    
    2532     @@ -558,9 +671,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2533     mov EX_RETVAL(%o4), %o0
    2534    
    2535     .align 32
    2536     -90: EX_LD(LOAD(ldub, %o1, %g1))
    2537     +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0)
    2538     subcc %o2, 1, %o2
    2539     - EX_ST(STORE(stb, %g1, %o1 + %o3))
    2540     + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1)
    2541     bgu,pt %XCC, 90b
    2542     add %o1, 1, %o1
    2543     retl
    2544     diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S
    2545     index 88ad73d86fe4..db73010a1af8 100644
    2546     --- a/arch/sparc/lib/U3copy_from_user.S
    2547     +++ b/arch/sparc/lib/U3copy_from_user.S
    2548     @@ -3,19 +3,19 @@
    2549     * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    2550     */
    2551    
    2552     -#define EX_LD(x) \
    2553     +#define EX_LD(x,y) \
    2554     98: x; \
    2555     .section __ex_table,"a";\
    2556     .align 4; \
    2557     - .word 98b, __retl_one; \
    2558     + .word 98b, y; \
    2559     .text; \
    2560     .align 4;
    2561    
    2562     -#define EX_LD_FP(x) \
    2563     +#define EX_LD_FP(x,y) \
    2564     98: x; \
    2565     .section __ex_table,"a";\
    2566     .align 4; \
    2567     - .word 98b, __retl_one_fp;\
    2568     + .word 98b, y##_fp; \
    2569     .text; \
    2570     .align 4;
    2571    
    2572     diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
    2573     index 845139d75537..c4ee858e352a 100644
    2574     --- a/arch/sparc/lib/U3copy_to_user.S
    2575     +++ b/arch/sparc/lib/U3copy_to_user.S
    2576     @@ -3,19 +3,19 @@
    2577     * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    2578     */
    2579    
    2580     -#define EX_ST(x) \
    2581     +#define EX_ST(x,y) \
    2582     98: x; \
    2583     .section __ex_table,"a";\
    2584     .align 4; \
    2585     - .word 98b, __retl_one; \
    2586     + .word 98b, y; \
    2587     .text; \
    2588     .align 4;
    2589    
    2590     -#define EX_ST_FP(x) \
    2591     +#define EX_ST_FP(x,y) \
    2592     98: x; \
    2593     .section __ex_table,"a";\
    2594     .align 4; \
    2595     - .word 98b, __retl_one_fp;\
    2596     + .word 98b, y##_fp; \
    2597     .text; \
    2598     .align 4;
    2599    
    2600     diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S
    2601     index 491ee69e4995..54f98706b03b 100644
    2602     --- a/arch/sparc/lib/U3memcpy.S
    2603     +++ b/arch/sparc/lib/U3memcpy.S
    2604     @@ -4,6 +4,7 @@
    2605     */
    2606    
    2607     #ifdef __KERNEL__
    2608     +#include <linux/linkage.h>
    2609     #include <asm/visasm.h>
    2610     #include <asm/asi.h>
    2611     #define GLOBAL_SPARE %g7
    2612     @@ -22,21 +23,17 @@
    2613     #endif
    2614    
    2615     #ifndef EX_LD
    2616     -#define EX_LD(x) x
    2617     +#define EX_LD(x,y) x
    2618     #endif
    2619     #ifndef EX_LD_FP
    2620     -#define EX_LD_FP(x) x
    2621     +#define EX_LD_FP(x,y) x
    2622     #endif
    2623    
    2624     #ifndef EX_ST
    2625     -#define EX_ST(x) x
    2626     +#define EX_ST(x,y) x
    2627     #endif
    2628     #ifndef EX_ST_FP
    2629     -#define EX_ST_FP(x) x
    2630     -#endif
    2631     -
    2632     -#ifndef EX_RETVAL
    2633     -#define EX_RETVAL(x) x
    2634     +#define EX_ST_FP(x,y) x
    2635     #endif
    2636    
    2637     #ifndef LOAD
    2638     @@ -77,6 +74,87 @@
    2639     */
    2640    
    2641     .text
    2642     +#ifndef EX_RETVAL
    2643     +#define EX_RETVAL(x) x
    2644     +__restore_fp:
    2645     + VISExitHalf
    2646     + retl
    2647     + nop
    2648     +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
    2649     + add %g1, 1, %g1
    2650     + add %g2, %g1, %g2
    2651     + ba,pt %xcc, __restore_fp
    2652     + add %o2, %g2, %o0
    2653     +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
    2654     +ENTRY(U3_retl_o2_plus_g2_fp)
    2655     + ba,pt %xcc, __restore_fp
    2656     + add %o2, %g2, %o0
    2657     +ENDPROC(U3_retl_o2_plus_g2_fp)
    2658     +ENTRY(U3_retl_o2_plus_g2_plus_8_fp)
    2659     + add %g2, 8, %g2
    2660     + ba,pt %xcc, __restore_fp
    2661     + add %o2, %g2, %o0
    2662     +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp)
    2663     +ENTRY(U3_retl_o2)
    2664     + retl
    2665     + mov %o2, %o0
    2666     +ENDPROC(U3_retl_o2)
    2667     +ENTRY(U3_retl_o2_plus_1)
    2668     + retl
    2669     + add %o2, 1, %o0
    2670     +ENDPROC(U3_retl_o2_plus_1)
    2671     +ENTRY(U3_retl_o2_plus_4)
    2672     + retl
    2673     + add %o2, 4, %o0
    2674     +ENDPROC(U3_retl_o2_plus_4)
    2675     +ENTRY(U3_retl_o2_plus_8)
    2676     + retl
    2677     + add %o2, 8, %o0
    2678     +ENDPROC(U3_retl_o2_plus_8)
    2679     +ENTRY(U3_retl_o2_plus_g1_plus_1)
    2680     + add %g1, 1, %g1
    2681     + retl
    2682     + add %o2, %g1, %o0
    2683     +ENDPROC(U3_retl_o2_plus_g1_plus_1)
    2684     +ENTRY(U3_retl_o2_fp)
    2685     + ba,pt %xcc, __restore_fp
    2686     + mov %o2, %o0
    2687     +ENDPROC(U3_retl_o2_fp)
    2688     +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
    2689     + sll %o3, 6, %o3
    2690     + add %o3, 0x80, %o3
    2691     + ba,pt %xcc, __restore_fp
    2692     + add %o2, %o3, %o0
    2693     +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
    2694     +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
    2695     + sll %o3, 6, %o3
    2696     + add %o3, 0x40, %o3
    2697     + ba,pt %xcc, __restore_fp
    2698     + add %o2, %o3, %o0
    2699     +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
    2700     +ENTRY(U3_retl_o2_plus_GS_plus_0x10)
    2701     + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE
    2702     + retl
    2703     + add %o2, GLOBAL_SPARE, %o0
    2704     +ENDPROC(U3_retl_o2_plus_GS_plus_0x10)
    2705     +ENTRY(U3_retl_o2_plus_GS_plus_0x08)
    2706     + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE
    2707     + retl
    2708     + add %o2, GLOBAL_SPARE, %o0
    2709     +ENDPROC(U3_retl_o2_plus_GS_plus_0x08)
    2710     +ENTRY(U3_retl_o2_and_7_plus_GS)
    2711     + and %o2, 7, %o2
    2712     + retl
    2713     + add %o2, GLOBAL_SPARE, %o2
    2714     +ENDPROC(U3_retl_o2_and_7_plus_GS)
    2715     +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8)
    2716     + add GLOBAL_SPARE, 8, GLOBAL_SPARE
    2717     + and %o2, 7, %o2
    2718     + retl
    2719     + add %o2, GLOBAL_SPARE, %o2
    2720     +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8)
    2721     +#endif
    2722     +
    2723     .align 64
    2724    
    2725     /* The cheetah's flexible spine, oversized liver, enlarged heart,
    2726     @@ -126,8 +204,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2727     and %g2, 0x38, %g2
    2728    
    2729     1: subcc %g1, 0x1, %g1
    2730     - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
    2731     - EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
    2732     + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1)
    2733     + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1)
    2734     bgu,pt %XCC, 1b
    2735     add %o1, 0x1, %o1
    2736    
    2737     @@ -138,20 +216,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2738     be,pt %icc, 3f
    2739     alignaddr %o1, %g0, %o1
    2740    
    2741     - EX_LD_FP(LOAD(ldd, %o1, %f4))
    2742     -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
    2743     + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2)
    2744     +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2)
    2745     add %o1, 0x8, %o1
    2746     subcc %g2, 0x8, %g2
    2747     faligndata %f4, %f6, %f0
    2748     - EX_ST_FP(STORE(std, %f0, %o0))
    2749     + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8)
    2750     be,pn %icc, 3f
    2751     add %o0, 0x8, %o0
    2752    
    2753     - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
    2754     + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2)
    2755     add %o1, 0x8, %o1
    2756     subcc %g2, 0x8, %g2
    2757     faligndata %f6, %f4, %f2
    2758     - EX_ST_FP(STORE(std, %f2, %o0))
    2759     + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8)
    2760     bne,pt %icc, 1b
    2761     add %o0, 0x8, %o0
    2762    
    2763     @@ -161,25 +239,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2764     LOAD(prefetch, %o1 + 0x080, #one_read)
    2765     LOAD(prefetch, %o1 + 0x0c0, #one_read)
    2766     LOAD(prefetch, %o1 + 0x100, #one_read)
    2767     - EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0))
    2768     + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2)
    2769     LOAD(prefetch, %o1 + 0x140, #one_read)
    2770     - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2771     + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2)
    2772     LOAD(prefetch, %o1 + 0x180, #one_read)
    2773     - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2774     + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2)
    2775     LOAD(prefetch, %o1 + 0x1c0, #one_read)
    2776     faligndata %f0, %f2, %f16
    2777     - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2778     + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2)
    2779     faligndata %f2, %f4, %f18
    2780     - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2781     + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2)
    2782     faligndata %f4, %f6, %f20
    2783     - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2784     + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2)
    2785     faligndata %f6, %f8, %f22
    2786    
    2787     - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2788     + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2)
    2789     faligndata %f8, %f10, %f24
    2790     - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2791     + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2)
    2792     faligndata %f10, %f12, %f26
    2793     - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2794     + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2)
    2795    
    2796     subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE
    2797     add %o1, 0x40, %o1
    2798     @@ -190,26 +268,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2799    
    2800     .align 64
    2801     1:
    2802     - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2803     + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2804     faligndata %f12, %f14, %f28
    2805     - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2806     + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2807     faligndata %f14, %f0, %f30
    2808     - EX_ST_FP(STORE_BLK(%f16, %o0))
    2809     - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2810     + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2811     + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2812     faligndata %f0, %f2, %f16
    2813     add %o0, 0x40, %o0
    2814    
    2815     - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2816     + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2817     faligndata %f2, %f4, %f18
    2818     - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2819     + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2820     faligndata %f4, %f6, %f20
    2821     - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2822     + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2823     subcc %o3, 0x01, %o3
    2824     faligndata %f6, %f8, %f22
    2825     - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2826     + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2827    
    2828     faligndata %f8, %f10, %f24
    2829     - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2830     + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2831     LOAD(prefetch, %o1 + 0x1c0, #one_read)
    2832     faligndata %f10, %f12, %f26
    2833     bg,pt %XCC, 1b
    2834     @@ -217,29 +295,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2835    
    2836     /* Finally we copy the last full 64-byte block. */
    2837     2:
    2838     - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2839     + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2840     faligndata %f12, %f14, %f28
    2841     - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2842     + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2843     faligndata %f14, %f0, %f30
    2844     - EX_ST_FP(STORE_BLK(%f16, %o0))
    2845     - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2846     + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2847     + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2848     faligndata %f0, %f2, %f16
    2849     - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2850     + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2851     faligndata %f2, %f4, %f18
    2852     - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2853     + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2854     faligndata %f4, %f6, %f20
    2855     - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2856     + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2857     faligndata %f6, %f8, %f22
    2858     - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2859     + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2860     faligndata %f8, %f10, %f24
    2861     cmp %g1, 0
    2862     be,pt %XCC, 1f
    2863     add %o0, 0x40, %o0
    2864     - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2865     + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2866     1: faligndata %f10, %f12, %f26
    2867     faligndata %f12, %f14, %f28
    2868     faligndata %f14, %f0, %f30
    2869     - EX_ST_FP(STORE_BLK(%f16, %o0))
    2870     + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2871     add %o0, 0x40, %o0
    2872     add %o1, 0x40, %o1
    2873     membar #Sync
    2874     @@ -259,20 +337,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2875    
    2876     sub %o2, %g2, %o2
    2877     be,a,pt %XCC, 1f
    2878     - EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0))
    2879     + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2)
    2880    
    2881     -1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2))
    2882     +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2)
    2883     add %o1, 0x8, %o1
    2884     subcc %g2, 0x8, %g2
    2885     faligndata %f0, %f2, %f8
    2886     - EX_ST_FP(STORE(std, %f8, %o0))
    2887     + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
    2888     be,pn %XCC, 2f
    2889     add %o0, 0x8, %o0
    2890     - EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0))
    2891     + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2)
    2892     add %o1, 0x8, %o1
    2893     subcc %g2, 0x8, %g2
    2894     faligndata %f2, %f0, %f8
    2895     - EX_ST_FP(STORE(std, %f8, %o0))
    2896     + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
    2897     bne,pn %XCC, 1b
    2898     add %o0, 0x8, %o0
    2899    
    2900     @@ -292,30 +370,33 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2901     andcc %o2, 0x8, %g0
    2902     be,pt %icc, 1f
    2903     nop
    2904     - EX_LD(LOAD(ldx, %o1, %o5))
    2905     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2906     + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2)
    2907     + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2)
    2908     add %o1, 0x8, %o1
    2909     + sub %o2, 8, %o2
    2910    
    2911     1: andcc %o2, 0x4, %g0
    2912     be,pt %icc, 1f
    2913     nop
    2914     - EX_LD(LOAD(lduw, %o1, %o5))
    2915     - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2916     + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2)
    2917     + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2)
    2918     add %o1, 0x4, %o1
    2919     + sub %o2, 4, %o2
    2920    
    2921     1: andcc %o2, 0x2, %g0
    2922     be,pt %icc, 1f
    2923     nop
    2924     - EX_LD(LOAD(lduh, %o1, %o5))
    2925     - EX_ST(STORE(sth, %o5, %o1 + %o3))
    2926     + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2)
    2927     + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2)
    2928     add %o1, 0x2, %o1
    2929     + sub %o2, 2, %o2
    2930    
    2931     1: andcc %o2, 0x1, %g0
    2932     be,pt %icc, 85f
    2933     nop
    2934     - EX_LD(LOAD(ldub, %o1, %o5))
    2935     + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2)
    2936     ba,pt %xcc, 85f
    2937     - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2938     + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2)
    2939    
    2940     .align 64
    2941     70: /* 16 < len <= 64 */
    2942     @@ -326,26 +407,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2943     andn %o2, 0xf, GLOBAL_SPARE
    2944     and %o2, 0xf, %o2
    2945     1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE
    2946     - EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
    2947     - EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
    2948     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2949     + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10)
    2950     + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10)
    2951     + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10)
    2952     add %o1, 0x8, %o1
    2953     - EX_ST(STORE(stx, %g1, %o1 + %o3))
    2954     + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08)
    2955     bgu,pt %XCC, 1b
    2956     add %o1, 0x8, %o1
    2957     73: andcc %o2, 0x8, %g0
    2958     be,pt %XCC, 1f
    2959     nop
    2960     sub %o2, 0x8, %o2
    2961     - EX_LD(LOAD(ldx, %o1, %o5))
    2962     - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2963     + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8)
    2964     + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8)
    2965     add %o1, 0x8, %o1
    2966     1: andcc %o2, 0x4, %g0
    2967     be,pt %XCC, 1f
    2968     nop
    2969     sub %o2, 0x4, %o2
    2970     - EX_LD(LOAD(lduw, %o1, %o5))
    2971     - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2972     + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4)
    2973     + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4)
    2974     add %o1, 0x4, %o1
    2975     1: cmp %o2, 0
    2976     be,pt %XCC, 85f
    2977     @@ -361,8 +442,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2978     sub %o2, %g1, %o2
    2979    
    2980     1: subcc %g1, 1, %g1
    2981     - EX_LD(LOAD(ldub, %o1, %o5))
    2982     - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2983     + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1)
    2984     + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1)
    2985     bgu,pt %icc, 1b
    2986     add %o1, 1, %o1
    2987    
    2988     @@ -378,16 +459,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2989    
    2990     8: mov 64, %o3
    2991     andn %o1, 0x7, %o1
    2992     - EX_LD(LOAD(ldx, %o1, %g2))
    2993     + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2)
    2994     sub %o3, %g1, %o3
    2995     andn %o2, 0x7, GLOBAL_SPARE
    2996     sllx %g2, %g1, %g2
    2997     -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
    2998     +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS)
    2999     subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE
    3000     add %o1, 0x8, %o1
    3001     srlx %g3, %o3, %o5
    3002     or %o5, %g2, %o5
    3003     - EX_ST(STORE(stx, %o5, %o0))
    3004     + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8)
    3005     add %o0, 0x8, %o0
    3006     bgu,pt %icc, 1b
    3007     sllx %g3, %g1, %g2
    3008     @@ -407,8 +488,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    3009    
    3010     1:
    3011     subcc %o2, 4, %o2
    3012     - EX_LD(LOAD(lduw, %o1, %g1))
    3013     - EX_ST(STORE(stw, %g1, %o1 + %o3))
    3014     + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4)
    3015     + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4)
    3016     bgu,pt %XCC, 1b
    3017     add %o1, 4, %o1
    3018    
    3019     @@ -418,8 +499,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    3020     .align 32
    3021     90:
    3022     subcc %o2, 1, %o2
    3023     - EX_LD(LOAD(ldub, %o1, %g1))
    3024     - EX_ST(STORE(stb, %g1, %o1 + %o3))
    3025     + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1)
    3026     + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1)
    3027     bgu,pt %XCC, 90b
    3028     add %o1, 1, %o1
    3029     retl
    3030     diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
    3031     index 302c0e60dc2c..4c89b486fa0d 100644
    3032     --- a/arch/sparc/lib/copy_in_user.S
    3033     +++ b/arch/sparc/lib/copy_in_user.S
    3034     @@ -8,18 +8,33 @@
    3035    
    3036     #define XCC xcc
    3037    
    3038     -#define EX(x,y) \
    3039     +#define EX(x,y,z) \
    3040     98: x,y; \
    3041     .section __ex_table,"a";\
    3042     .align 4; \
    3043     - .word 98b, __retl_one; \
    3044     + .word 98b, z; \
    3045     .text; \
    3046     .align 4;
    3047    
    3048     +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8)
    3049     +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4)
    3050     +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1)
    3051     +
    3052     .register %g2,#scratch
    3053     .register %g3,#scratch
    3054    
    3055     .text
    3056     +__retl_o4_plus_8:
    3057     + add %o4, %o2, %o4
    3058     + retl
    3059     + add %o4, 8, %o0
    3060     +__retl_o2_plus_4:
    3061     + retl
    3062     + add %o2, 4, %o0
    3063     +__retl_o2_plus_1:
    3064     + retl
    3065     + add %o2, 1, %o0
    3066     +
    3067     .align 32
    3068    
    3069     /* Don't try to get too fancy here, just nice and
    3070     @@ -44,8 +59,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    3071     andn %o2, 0x7, %o4
    3072     and %o2, 0x7, %o2
    3073     1: subcc %o4, 0x8, %o4
    3074     - EX(ldxa [%o1] %asi, %o5)
    3075     - EX(stxa %o5, [%o0] %asi)
    3076     + EX_O4(ldxa [%o1] %asi, %o5)
    3077     + EX_O4(stxa %o5, [%o0] %asi)
    3078     add %o1, 0x8, %o1
    3079     bgu,pt %XCC, 1b
    3080     add %o0, 0x8, %o0
    3081     @@ -53,8 +68,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    3082     be,pt %XCC, 1f
    3083     nop
    3084     sub %o2, 0x4, %o2
    3085     - EX(lduwa [%o1] %asi, %o5)
    3086     - EX(stwa %o5, [%o0] %asi)
    3087     + EX_O2_4(lduwa [%o1] %asi, %o5)
    3088     + EX_O2_4(stwa %o5, [%o0] %asi)
    3089     add %o1, 0x4, %o1
    3090     add %o0, 0x4, %o0
    3091     1: cmp %o2, 0
    3092     @@ -70,8 +85,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    3093    
    3094     82:
    3095     subcc %o2, 4, %o2
    3096     - EX(lduwa [%o1] %asi, %g1)
    3097     - EX(stwa %g1, [%o0] %asi)
    3098     + EX_O2_4(lduwa [%o1] %asi, %g1)
    3099     + EX_O2_4(stwa %g1, [%o0] %asi)
    3100     add %o1, 4, %o1
    3101     bgu,pt %XCC, 82b
    3102     add %o0, 4, %o0
    3103     @@ -82,8 +97,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    3104     .align 32
    3105     90:
    3106     subcc %o2, 1, %o2
    3107     - EX(lduba [%o1] %asi, %g1)
    3108     - EX(stba %g1, [%o0] %asi)
    3109     + EX_O2_1(lduba [%o1] %asi, %g1)
    3110     + EX_O2_1(stba %g1, [%o0] %asi)
    3111     add %o1, 1, %o1
    3112     bgu,pt %XCC, 90b
    3113     add %o0, 1, %o0
    3114     diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c
    3115     deleted file mode 100644
    3116     index ac96ae236709..000000000000
    3117     --- a/arch/sparc/lib/user_fixup.c
    3118     +++ /dev/null
    3119     @@ -1,71 +0,0 @@
    3120     -/* user_fixup.c: Fix up user copy faults.
    3121     - *
    3122     - * Copyright (C) 2004 David S. Miller <davem@redhat.com>
    3123     - */
    3124     -
    3125     -#include <linux/compiler.h>
    3126     -#include <linux/kernel.h>
    3127     -#include <linux/string.h>
    3128     -#include <linux/errno.h>
    3129     -#include <linux/module.h>
    3130     -
    3131     -#include <asm/uaccess.h>
    3132     -
    3133     -/* Calculating the exact fault address when using
    3134     - * block loads and stores can be very complicated.
    3135     - *
    3136     - * Instead of trying to be clever and handling all
    3137     - * of the cases, just fix things up simply here.
    3138     - */
    3139     -
    3140     -static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset)
    3141     -{
    3142     - unsigned long fault_addr = current_thread_info()->fault_address;
    3143     - unsigned long end = start + size;
    3144     -
    3145     - if (fault_addr < start || fault_addr >= end) {
    3146     - *offset = 0;
    3147     - } else {
    3148     - *offset = fault_addr - start;
    3149     - size = end - fault_addr;
    3150     - }
    3151     - return size;
    3152     -}
    3153     -
    3154     -unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size)
    3155     -{
    3156     - unsigned long offset;
    3157     -
    3158     - size = compute_size((unsigned long) from, size, &offset);
    3159     - if (likely(size))
    3160     - memset(to + offset, 0, size);
    3161     -
    3162     - return size;
    3163     -}
    3164     -EXPORT_SYMBOL(copy_from_user_fixup);
    3165     -
    3166     -unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size)
    3167     -{
    3168     - unsigned long offset;
    3169     -
    3170     - return compute_size((unsigned long) to, size, &offset);
    3171     -}
    3172     -EXPORT_SYMBOL(copy_to_user_fixup);
    3173     -
    3174     -unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size)
    3175     -{
    3176     - unsigned long fault_addr = current_thread_info()->fault_address;
    3177     - unsigned long start = (unsigned long) to;
    3178     - unsigned long end = start + size;
    3179     -
    3180     - if (fault_addr >= start && fault_addr < end)
    3181     - return end - fault_addr;
    3182     -
    3183     - start = (unsigned long) from;
    3184     - end = start + size;
    3185     - if (fault_addr >= start && fault_addr < end)
    3186     - return end - fault_addr;
    3187     -
    3188     - return size;
    3189     -}
    3190     -EXPORT_SYMBOL(copy_in_user_fixup);
    3191     diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
    3192     index dbabe5713a15..e15f33715103 100644
    3193     --- a/arch/sparc/mm/fault_64.c
    3194     +++ b/arch/sparc/mm/fault_64.c
    3195     @@ -479,14 +479,14 @@ good_area:
    3196     up_read(&mm->mmap_sem);
    3197    
    3198     mm_rss = get_mm_rss(mm);
    3199     -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3200     - mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE));
    3201     +#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3202     + mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE));
    3203     #endif
    3204     if (unlikely(mm_rss >
    3205     mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
    3206     tsb_grow(mm, MM_TSB_BASE, mm_rss);
    3207     #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3208     - mm_rss = mm->context.huge_pte_count;
    3209     + mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count;
    3210     if (unlikely(mm_rss >
    3211     mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) {
    3212     if (mm->context.tsb_block[MM_TSB_HUGE].tsb)
    3213     diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
    3214     index 364d093f46c6..da1142401bf4 100644
    3215     --- a/arch/sparc/mm/hugetlbpage.c
    3216     +++ b/arch/sparc/mm/hugetlbpage.c
    3217     @@ -180,7 +180,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
    3218     unsigned long nptes;
    3219    
    3220     if (!pte_present(*ptep) && pte_present(entry))
    3221     - mm->context.huge_pte_count++;
    3222     + mm->context.hugetlb_pte_count++;
    3223    
    3224     addr &= HPAGE_MASK;
    3225    
    3226     @@ -212,7 +212,7 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
    3227    
    3228     entry = *ptep;
    3229     if (pte_present(entry))
    3230     - mm->context.huge_pte_count--;
    3231     + mm->context.hugetlb_pte_count--;
    3232    
    3233     addr &= HPAGE_MASK;
    3234     nptes = 1 << HUGETLB_PAGE_ORDER;
    3235     diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
    3236     index 3c4b8975fa76..a5331c336b2a 100644
    3237     --- a/arch/sparc/mm/init_64.c
    3238     +++ b/arch/sparc/mm/init_64.c
    3239     @@ -346,7 +346,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
    3240     spin_lock_irqsave(&mm->context.lock, flags);
    3241    
    3242     #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3243     - if (mm->context.huge_pte_count && is_hugetlb_pte(pte))
    3244     + if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) &&
    3245     + is_hugetlb_pte(pte))
    3246     __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
    3247     address, pte_val(pte));
    3248     else
    3249     diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
    3250     index f81cd9736700..3659d37b4d81 100644
    3251     --- a/arch/sparc/mm/tlb.c
    3252     +++ b/arch/sparc/mm/tlb.c
    3253     @@ -175,9 +175,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
    3254    
    3255     if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
    3256     if (pmd_val(pmd) & _PAGE_PMD_HUGE)
    3257     - mm->context.huge_pte_count++;
    3258     + mm->context.thp_pte_count++;
    3259     else
    3260     - mm->context.huge_pte_count--;
    3261     + mm->context.thp_pte_count--;
    3262    
    3263     /* Do not try to allocate the TSB hash table if we
    3264     * don't have one already. We have various locks held
    3265     diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
    3266     index a0604a493a36..9cdeca0fa955 100644
    3267     --- a/arch/sparc/mm/tsb.c
    3268     +++ b/arch/sparc/mm/tsb.c
    3269     @@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr)
    3270     return (tag == (vaddr >> 22));
    3271     }
    3272    
    3273     +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end)
    3274     +{
    3275     + unsigned long idx;
    3276     +
    3277     + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) {
    3278     + struct tsb *ent = &swapper_tsb[idx];
    3279     + unsigned long match = idx << 13;
    3280     +
    3281     + match |= (ent->tag << 22);
    3282     + if (match >= start && match < end)
    3283     + ent->tag = (1UL << TSB_TAG_INVALID_BIT);
    3284     + }
    3285     +}
    3286     +
    3287     /* TSB flushes need only occur on the processor initiating the address
    3288     * space modification, not on each cpu the address space has run on.
    3289     * Only the TLB flush needs that treatment.
    3290     @@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
    3291     {
    3292     unsigned long v;
    3293    
    3294     + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES)
    3295     + return flush_tsb_kernel_range_scan(start, end);
    3296     +
    3297     for (v = start; v < end; v += PAGE_SIZE) {
    3298     unsigned long hash = tsb_hash(v, PAGE_SHIFT,
    3299     KERNEL_TSB_NENTRIES);
    3300     @@ -470,7 +487,7 @@ retry_tsb_alloc:
    3301     int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
    3302     {
    3303     #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3304     - unsigned long huge_pte_count;
    3305     + unsigned long total_huge_pte_count;
    3306     #endif
    3307     unsigned int i;
    3308    
    3309     @@ -479,12 +496,14 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
    3310     mm->context.sparc64_ctx_val = 0UL;
    3311    
    3312     #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3313     - /* We reset it to zero because the fork() page copying
    3314     + /* We reset them to zero because the fork() page copying
    3315     * will re-increment the counters as the parent PTEs are
    3316     * copied into the child address space.
    3317     */
    3318     - huge_pte_count = mm->context.huge_pte_count;
    3319     - mm->context.huge_pte_count = 0;
    3320     + total_huge_pte_count = mm->context.hugetlb_pte_count +
    3321     + mm->context.thp_pte_count;
    3322     + mm->context.hugetlb_pte_count = 0;
    3323     + mm->context.thp_pte_count = 0;
    3324     #endif
    3325    
    3326     /* copy_mm() copies over the parent's mm_struct before calling
    3327     @@ -500,8 +519,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
    3328     tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm));
    3329    
    3330     #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3331     - if (unlikely(huge_pte_count))
    3332     - tsb_grow(mm, MM_TSB_HUGE, huge_pte_count);
    3333     + if (unlikely(total_huge_pte_count))
    3334     + tsb_grow(mm, MM_TSB_HUGE, total_huge_pte_count);
    3335     #endif
    3336    
    3337     if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb))
    3338     diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
    3339     index b4f4733abc6e..5d2fd6cd3189 100644
    3340     --- a/arch/sparc/mm/ultra.S
    3341     +++ b/arch/sparc/mm/ultra.S
    3342     @@ -30,7 +30,7 @@
    3343     .text
    3344     .align 32
    3345     .globl __flush_tlb_mm
    3346     -__flush_tlb_mm: /* 18 insns */
    3347     +__flush_tlb_mm: /* 19 insns */
    3348     /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
    3349     ldxa [%o1] ASI_DMMU, %g2
    3350     cmp %g2, %o0
    3351     @@ -81,7 +81,7 @@ __flush_tlb_page: /* 22 insns */
    3352    
    3353     .align 32
    3354     .globl __flush_tlb_pending
    3355     -__flush_tlb_pending: /* 26 insns */
    3356     +__flush_tlb_pending: /* 27 insns */
    3357     /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
    3358     rdpr %pstate, %g7
    3359     sllx %o1, 3, %o1
    3360     @@ -113,12 +113,14 @@ __flush_tlb_pending: /* 26 insns */
    3361    
    3362     .align 32
    3363     .globl __flush_tlb_kernel_range
    3364     -__flush_tlb_kernel_range: /* 16 insns */
    3365     +__flush_tlb_kernel_range: /* 31 insns */
    3366     /* %o0=start, %o1=end */
    3367     cmp %o0, %o1
    3368     be,pn %xcc, 2f
    3369     + sub %o1, %o0, %o3
    3370     + srlx %o3, 18, %o4
    3371     + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow
    3372     sethi %hi(PAGE_SIZE), %o4
    3373     - sub %o1, %o0, %o3
    3374     sub %o3, %o4, %o3
    3375     or %o0, 0x20, %o0 ! Nucleus
    3376     1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
    3377     @@ -131,6 +133,41 @@ __flush_tlb_kernel_range: /* 16 insns */
    3378     retl
    3379     nop
    3380     nop
    3381     + nop
    3382     + nop
    3383     + nop
    3384     + nop
    3385     + nop
    3386     + nop
    3387     + nop
    3388     + nop
    3389     + nop
    3390     + nop
    3391     + nop
    3392     + nop
    3393     + nop
    3394     +
    3395     +__spitfire_flush_tlb_kernel_range_slow:
    3396     + mov 63 * 8, %o4
    3397     +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3
    3398     + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */
    3399     + bne,pn %xcc, 2f
    3400     + mov TLB_TAG_ACCESS, %o3
    3401     + stxa %g0, [%o3] ASI_IMMU
    3402     + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS
    3403     + membar #Sync
    3404     +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3
    3405     + andcc %o3, 0x40, %g0
    3406     + bne,pn %xcc, 2f
    3407     + mov TLB_TAG_ACCESS, %o3
    3408     + stxa %g0, [%o3] ASI_DMMU
    3409     + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS
    3410     + membar #Sync
    3411     +2: sub %o4, 8, %o4
    3412     + brgez,pt %o4, 1b
    3413     + nop
    3414     + retl
    3415     + nop
    3416    
    3417     __spitfire_flush_tlb_mm_slow:
    3418     rdpr %pstate, %g1
    3419     @@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */
    3420     retl
    3421     wrpr %g7, 0x0, %pstate
    3422    
    3423     +__cheetah_flush_tlb_kernel_range: /* 31 insns */
    3424     + /* %o0=start, %o1=end */
    3425     + cmp %o0, %o1
    3426     + be,pn %xcc, 2f
    3427     + sub %o1, %o0, %o3
    3428     + srlx %o3, 18, %o4
    3429     + brnz,pn %o4, 3f
    3430     + sethi %hi(PAGE_SIZE), %o4
    3431     + sub %o3, %o4, %o3
    3432     + or %o0, 0x20, %o0 ! Nucleus
    3433     +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
    3434     + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
    3435     + membar #Sync
    3436     + brnz,pt %o3, 1b
    3437     + sub %o3, %o4, %o3
    3438     +2: sethi %hi(KERNBASE), %o3
    3439     + flush %o3
    3440     + retl
    3441     + nop
    3442     +3: mov 0x80, %o4
    3443     + stxa %g0, [%o4] ASI_DMMU_DEMAP
    3444     + membar #Sync
    3445     + stxa %g0, [%o4] ASI_IMMU_DEMAP
    3446     + membar #Sync
    3447     + retl
    3448     + nop
    3449     + nop
    3450     + nop
    3451     + nop
    3452     + nop
    3453     + nop
    3454     + nop
    3455     + nop
    3456     +
    3457     #ifdef DCACHE_ALIASING_POSSIBLE
    3458     __cheetah_flush_dcache_page: /* 11 insns */
    3459     sethi %hi(PAGE_OFFSET), %g1
    3460     @@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error:
    3461     ret
    3462     restore
    3463    
    3464     -__hypervisor_flush_tlb_mm: /* 10 insns */
    3465     +__hypervisor_flush_tlb_mm: /* 19 insns */
    3466     mov %o0, %o2 /* ARG2: mmu context */
    3467     mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3468     mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3469     mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3470     mov HV_FAST_MMU_DEMAP_CTX, %o5
    3471     ta HV_FAST_TRAP
    3472     - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3473     + brnz,pn %o0, 1f
    3474     mov HV_FAST_MMU_DEMAP_CTX, %o1
    3475     retl
    3476     nop
    3477     +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5
    3478     + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0
    3479     + nop
    3480     + nop
    3481     + nop
    3482     + nop
    3483     + nop
    3484     + nop
    3485     + nop
    3486    
    3487     -__hypervisor_flush_tlb_page: /* 11 insns */
    3488     +__hypervisor_flush_tlb_page: /* 22 insns */
    3489     /* %o0 = context, %o1 = vaddr */
    3490     mov %o0, %g2
    3491     mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */
    3492     @@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */
    3493     srlx %o0, PAGE_SHIFT, %o0
    3494     sllx %o0, PAGE_SHIFT, %o0
    3495     ta HV_MMU_UNMAP_ADDR_TRAP
    3496     - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3497     + brnz,pn %o0, 1f
    3498     mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3499     retl
    3500     nop
    3501     +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3502     + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3503     + nop
    3504     + nop
    3505     + nop
    3506     + nop
    3507     + nop
    3508     + nop
    3509     + nop
    3510     + nop
    3511     + nop
    3512    
    3513     -__hypervisor_flush_tlb_pending: /* 16 insns */
    3514     +__hypervisor_flush_tlb_pending: /* 27 insns */
    3515     /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
    3516     sllx %o1, 3, %g1
    3517     mov %o2, %g2
    3518     @@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */
    3519     srlx %o0, PAGE_SHIFT, %o0
    3520     sllx %o0, PAGE_SHIFT, %o0
    3521     ta HV_MMU_UNMAP_ADDR_TRAP
    3522     - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3523     + brnz,pn %o0, 1f
    3524     mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3525     brnz,pt %g1, 1b
    3526     nop
    3527     retl
    3528     nop
    3529     +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3530     + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3531     + nop
    3532     + nop
    3533     + nop
    3534     + nop
    3535     + nop
    3536     + nop
    3537     + nop
    3538     + nop
    3539     + nop
    3540    
    3541     -__hypervisor_flush_tlb_kernel_range: /* 16 insns */
    3542     +__hypervisor_flush_tlb_kernel_range: /* 31 insns */
    3543     /* %o0=start, %o1=end */
    3544     cmp %o0, %o1
    3545     be,pn %xcc, 2f
    3546     - sethi %hi(PAGE_SIZE), %g3
    3547     - mov %o0, %g1
    3548     - sub %o1, %g1, %g2
    3549     + sub %o1, %o0, %g2
    3550     + srlx %g2, 18, %g3
    3551     + brnz,pn %g3, 4f
    3552     + mov %o0, %g1
    3553     + sethi %hi(PAGE_SIZE), %g3
    3554     sub %g2, %g3, %g2
    3555     1: add %g1, %g2, %o0 /* ARG0: virtual address */
    3556     mov 0, %o1 /* ARG1: mmu context */
    3557     mov HV_MMU_ALL, %o2 /* ARG2: flags */
    3558     ta HV_MMU_UNMAP_ADDR_TRAP
    3559     - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3560     + brnz,pn %o0, 3f
    3561     mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3562     brnz,pt %g2, 1b
    3563     sub %g2, %g3, %g2
    3564     2: retl
    3565     nop
    3566     +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3567     + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3568     + nop
    3569     +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3570     + mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3571     + mov 0, %o2 /* ARG2: mmu context == nucleus */
    3572     + mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3573     + mov HV_FAST_MMU_DEMAP_CTX, %o5
    3574     + ta HV_FAST_TRAP
    3575     + brnz,pn %o0, 3b
    3576     + mov HV_FAST_MMU_DEMAP_CTX, %o1
    3577     + retl
    3578     + nop
    3579    
    3580     #ifdef DCACHE_ALIASING_POSSIBLE
    3581     /* XXX Niagara and friends have an 8K cache, so no aliasing is
    3582     @@ -394,43 +511,6 @@ tlb_patch_one:
    3583     retl
    3584     nop
    3585    
    3586     - .globl cheetah_patch_cachetlbops
    3587     -cheetah_patch_cachetlbops:
    3588     - save %sp, -128, %sp
    3589     -
    3590     - sethi %hi(__flush_tlb_mm), %o0
    3591     - or %o0, %lo(__flush_tlb_mm), %o0
    3592     - sethi %hi(__cheetah_flush_tlb_mm), %o1
    3593     - or %o1, %lo(__cheetah_flush_tlb_mm), %o1
    3594     - call tlb_patch_one
    3595     - mov 19, %o2
    3596     -
    3597     - sethi %hi(__flush_tlb_page), %o0
    3598     - or %o0, %lo(__flush_tlb_page), %o0
    3599     - sethi %hi(__cheetah_flush_tlb_page), %o1
    3600     - or %o1, %lo(__cheetah_flush_tlb_page), %o1
    3601     - call tlb_patch_one
    3602     - mov 22, %o2
    3603     -
    3604     - sethi %hi(__flush_tlb_pending), %o0
    3605     - or %o0, %lo(__flush_tlb_pending), %o0
    3606     - sethi %hi(__cheetah_flush_tlb_pending), %o1
    3607     - or %o1, %lo(__cheetah_flush_tlb_pending), %o1
    3608     - call tlb_patch_one
    3609     - mov 27, %o2
    3610     -
    3611     -#ifdef DCACHE_ALIASING_POSSIBLE
    3612     - sethi %hi(__flush_dcache_page), %o0
    3613     - or %o0, %lo(__flush_dcache_page), %o0
    3614     - sethi %hi(__cheetah_flush_dcache_page), %o1
    3615     - or %o1, %lo(__cheetah_flush_dcache_page), %o1
    3616     - call tlb_patch_one
    3617     - mov 11, %o2
    3618     -#endif /* DCACHE_ALIASING_POSSIBLE */
    3619     -
    3620     - ret
    3621     - restore
    3622     -
    3623     #ifdef CONFIG_SMP
    3624     /* These are all called by the slaves of a cross call, at
    3625     * trap level 1, with interrupts fully disabled.
    3626     @@ -447,7 +527,7 @@ cheetah_patch_cachetlbops:
    3627     */
    3628     .align 32
    3629     .globl xcall_flush_tlb_mm
    3630     -xcall_flush_tlb_mm: /* 21 insns */
    3631     +xcall_flush_tlb_mm: /* 24 insns */
    3632     mov PRIMARY_CONTEXT, %g2
    3633     ldxa [%g2] ASI_DMMU, %g3
    3634     srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
    3635     @@ -469,9 +549,12 @@ xcall_flush_tlb_mm: /* 21 insns */
    3636     nop
    3637     nop
    3638     nop
    3639     + nop
    3640     + nop
    3641     + nop
    3642    
    3643     .globl xcall_flush_tlb_page
    3644     -xcall_flush_tlb_page: /* 17 insns */
    3645     +xcall_flush_tlb_page: /* 20 insns */
    3646     /* %g5=context, %g1=vaddr */
    3647     mov PRIMARY_CONTEXT, %g4
    3648     ldxa [%g4] ASI_DMMU, %g2
    3649     @@ -490,15 +573,20 @@ xcall_flush_tlb_page: /* 17 insns */
    3650     retry
    3651     nop
    3652     nop
    3653     + nop
    3654     + nop
    3655     + nop
    3656    
    3657     .globl xcall_flush_tlb_kernel_range
    3658     -xcall_flush_tlb_kernel_range: /* 25 insns */
    3659     +xcall_flush_tlb_kernel_range: /* 44 insns */
    3660     sethi %hi(PAGE_SIZE - 1), %g2
    3661     or %g2, %lo(PAGE_SIZE - 1), %g2
    3662     andn %g1, %g2, %g1
    3663     andn %g7, %g2, %g7
    3664     sub %g7, %g1, %g3
    3665     - add %g2, 1, %g2
    3666     + srlx %g3, 18, %g2
    3667     + brnz,pn %g2, 2f
    3668     + add %g2, 1, %g2
    3669     sub %g3, %g2, %g3
    3670     or %g1, 0x20, %g1 ! Nucleus
    3671     1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
    3672     @@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */
    3673     brnz,pt %g3, 1b
    3674     sub %g3, %g2, %g3
    3675     retry
    3676     - nop
    3677     - nop
    3678     +2: mov 63 * 8, %g1
    3679     +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2
    3680     + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */
    3681     + bne,pn %xcc, 2f
    3682     + mov TLB_TAG_ACCESS, %g2
    3683     + stxa %g0, [%g2] ASI_IMMU
    3684     + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS
    3685     + membar #Sync
    3686     +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2
    3687     + andcc %g2, 0x40, %g0
    3688     + bne,pn %xcc, 2f
    3689     + mov TLB_TAG_ACCESS, %g2
    3690     + stxa %g0, [%g2] ASI_DMMU
    3691     + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS
    3692     + membar #Sync
    3693     +2: sub %g1, 8, %g1
    3694     + brgez,pt %g1, 1b
    3695     + nop
    3696     + retry
    3697     nop
    3698     nop
    3699     nop
    3700     @@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4:
    3701    
    3702     retry
    3703    
    3704     +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */
    3705     + sethi %hi(PAGE_SIZE - 1), %g2
    3706     + or %g2, %lo(PAGE_SIZE - 1), %g2
    3707     + andn %g1, %g2, %g1
    3708     + andn %g7, %g2, %g7
    3709     + sub %g7, %g1, %g3
    3710     + srlx %g3, 18, %g2
    3711     + brnz,pn %g2, 2f
    3712     + add %g2, 1, %g2
    3713     + sub %g3, %g2, %g3
    3714     + or %g1, 0x20, %g1 ! Nucleus
    3715     +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
    3716     + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
    3717     + membar #Sync
    3718     + brnz,pt %g3, 1b
    3719     + sub %g3, %g2, %g3
    3720     + retry
    3721     +2: mov 0x80, %g2
    3722     + stxa %g0, [%g2] ASI_DMMU_DEMAP
    3723     + membar #Sync
    3724     + stxa %g0, [%g2] ASI_IMMU_DEMAP
    3725     + membar #Sync
    3726     + retry
    3727     + nop
    3728     + nop
    3729     + nop
    3730     + nop
    3731     + nop
    3732     + nop
    3733     + nop
    3734     + nop
    3735     + nop
    3736     + nop
    3737     + nop
    3738     + nop
    3739     + nop
    3740     + nop
    3741     + nop
    3742     + nop
    3743     + nop
    3744     + nop
    3745     + nop
    3746     + nop
    3747     + nop
    3748     + nop
    3749     +
    3750     #ifdef DCACHE_ALIASING_POSSIBLE
    3751     .align 32
    3752     .globl xcall_flush_dcache_page_cheetah
    3753     @@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error:
    3754     ba,a,pt %xcc, rtrap
    3755    
    3756     .globl __hypervisor_xcall_flush_tlb_mm
    3757     -__hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3758     +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
    3759     /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
    3760     mov %o0, %g2
    3761     mov %o1, %g3
    3762     @@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3763     mov HV_FAST_MMU_DEMAP_CTX, %o5
    3764     ta HV_FAST_TRAP
    3765     mov HV_FAST_MMU_DEMAP_CTX, %g6
    3766     - brnz,pn %o0, __hypervisor_tlb_xcall_error
    3767     + brnz,pn %o0, 1f
    3768     mov %o0, %g5
    3769     mov %g2, %o0
    3770     mov %g3, %o1
    3771     @@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3772     mov %g7, %o5
    3773     membar #Sync
    3774     retry
    3775     +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3776     + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3777     + nop
    3778    
    3779     .globl __hypervisor_xcall_flush_tlb_page
    3780     -__hypervisor_xcall_flush_tlb_page: /* 17 insns */
    3781     +__hypervisor_xcall_flush_tlb_page: /* 20 insns */
    3782     /* %g5=ctx, %g1=vaddr */
    3783     mov %o0, %g2
    3784     mov %o1, %g3
    3785     @@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */
    3786     sllx %o0, PAGE_SHIFT, %o0
    3787     ta HV_MMU_UNMAP_ADDR_TRAP
    3788     mov HV_MMU_UNMAP_ADDR_TRAP, %g6
    3789     - brnz,a,pn %o0, __hypervisor_tlb_xcall_error
    3790     + brnz,a,pn %o0, 1f
    3791     mov %o0, %g5
    3792     mov %g2, %o0
    3793     mov %g3, %o1
    3794     mov %g4, %o2
    3795     membar #Sync
    3796     retry
    3797     +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3798     + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3799     + nop
    3800    
    3801     .globl __hypervisor_xcall_flush_tlb_kernel_range
    3802     -__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
    3803     +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
    3804     /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
    3805     sethi %hi(PAGE_SIZE - 1), %g2
    3806     or %g2, %lo(PAGE_SIZE - 1), %g2
    3807     andn %g1, %g2, %g1
    3808     andn %g7, %g2, %g7
    3809     sub %g7, %g1, %g3
    3810     + srlx %g3, 18, %g7
    3811     add %g2, 1, %g2
    3812     sub %g3, %g2, %g3
    3813     mov %o0, %g2
    3814     mov %o1, %g4
    3815     - mov %o2, %g7
    3816     + brnz,pn %g7, 2f
    3817     + mov %o2, %g7
    3818     1: add %g1, %g3, %o0 /* ARG0: virtual address */
    3819     mov 0, %o1 /* ARG1: mmu context */
    3820     mov HV_MMU_ALL, %o2 /* ARG2: flags */
    3821     ta HV_MMU_UNMAP_ADDR_TRAP
    3822     mov HV_MMU_UNMAP_ADDR_TRAP, %g6
    3823     - brnz,pn %o0, __hypervisor_tlb_xcall_error
    3824     + brnz,pn %o0, 1f
    3825     mov %o0, %g5
    3826     sethi %hi(PAGE_SIZE), %o2
    3827     brnz,pt %g3, 1b
    3828     sub %g3, %o2, %g3
    3829     - mov %g2, %o0
    3830     +5: mov %g2, %o0
    3831     mov %g4, %o1
    3832     mov %g7, %o2
    3833     membar #Sync
    3834     retry
    3835     +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3836     + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3837     + nop
    3838     +2: mov %o3, %g1
    3839     + mov %o5, %g3
    3840     + mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3841     + mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3842     + mov 0, %o2 /* ARG2: mmu context == nucleus */
    3843     + mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3844     + mov HV_FAST_MMU_DEMAP_CTX, %o5
    3845     + ta HV_FAST_TRAP
    3846     + mov %g1, %o3
    3847     + brz,pt %o0, 5b
    3848     + mov %g3, %o5
    3849     + mov HV_FAST_MMU_DEMAP_CTX, %g6
    3850     + ba,pt %xcc, 1b
    3851     + clr %g5
    3852    
    3853     /* These just get rescheduled to PIL vectors. */
    3854     .globl xcall_call_function
    3855     @@ -809,6 +985,58 @@ xcall_kgdb_capture:
    3856    
    3857     #endif /* CONFIG_SMP */
    3858    
    3859     + .globl cheetah_patch_cachetlbops
    3860     +cheetah_patch_cachetlbops:
    3861     + save %sp, -128, %sp
    3862     +
    3863     + sethi %hi(__flush_tlb_mm), %o0
    3864     + or %o0, %lo(__flush_tlb_mm), %o0
    3865     + sethi %hi(__cheetah_flush_tlb_mm), %o1
    3866     + or %o1, %lo(__cheetah_flush_tlb_mm), %o1
    3867     + call tlb_patch_one
    3868     + mov 19, %o2
    3869     +
    3870     + sethi %hi(__flush_tlb_page), %o0
    3871     + or %o0, %lo(__flush_tlb_page), %o0
    3872     + sethi %hi(__cheetah_flush_tlb_page), %o1
    3873     + or %o1, %lo(__cheetah_flush_tlb_page), %o1
    3874     + call tlb_patch_one
    3875     + mov 22, %o2
    3876     +
    3877     + sethi %hi(__flush_tlb_pending), %o0
    3878     + or %o0, %lo(__flush_tlb_pending), %o0
    3879     + sethi %hi(__cheetah_flush_tlb_pending), %o1
    3880     + or %o1, %lo(__cheetah_flush_tlb_pending), %o1
    3881     + call tlb_patch_one
    3882     + mov 27, %o2
    3883     +
    3884     + sethi %hi(__flush_tlb_kernel_range), %o0
    3885     + or %o0, %lo(__flush_tlb_kernel_range), %o0
    3886     + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1
    3887     + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
    3888     + call tlb_patch_one
    3889     + mov 31, %o2
    3890     +
    3891     +#ifdef DCACHE_ALIASING_POSSIBLE
    3892     + sethi %hi(__flush_dcache_page), %o0
    3893     + or %o0, %lo(__flush_dcache_page), %o0
    3894     + sethi %hi(__cheetah_flush_dcache_page), %o1
    3895     + or %o1, %lo(__cheetah_flush_dcache_page), %o1
    3896     + call tlb_patch_one
    3897     + mov 11, %o2
    3898     +#endif /* DCACHE_ALIASING_POSSIBLE */
    3899     +
    3900     +#ifdef CONFIG_SMP
    3901     + sethi %hi(xcall_flush_tlb_kernel_range), %o0
    3902     + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
    3903     + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
    3904     + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
    3905     + call tlb_patch_one
    3906     + mov 44, %o2
    3907     +#endif /* CONFIG_SMP */
    3908     +
    3909     + ret
    3910     + restore
    3911    
    3912     .globl hypervisor_patch_cachetlbops
    3913     hypervisor_patch_cachetlbops:
    3914     @@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops:
    3915     sethi %hi(__hypervisor_flush_tlb_mm), %o1
    3916     or %o1, %lo(__hypervisor_flush_tlb_mm), %o1
    3917     call tlb_patch_one
    3918     - mov 10, %o2
    3919     + mov 19, %o2
    3920    
    3921     sethi %hi(__flush_tlb_page), %o0
    3922     or %o0, %lo(__flush_tlb_page), %o0
    3923     sethi %hi(__hypervisor_flush_tlb_page), %o1
    3924     or %o1, %lo(__hypervisor_flush_tlb_page), %o1
    3925     call tlb_patch_one
    3926     - mov 11, %o2
    3927     + mov 22, %o2
    3928    
    3929     sethi %hi(__flush_tlb_pending), %o0
    3930     or %o0, %lo(__flush_tlb_pending), %o0
    3931     sethi %hi(__hypervisor_flush_tlb_pending), %o1
    3932     or %o1, %lo(__hypervisor_flush_tlb_pending), %o1
    3933     call tlb_patch_one
    3934     - mov 16, %o2
    3935     + mov 27, %o2
    3936    
    3937     sethi %hi(__flush_tlb_kernel_range), %o0
    3938     or %o0, %lo(__flush_tlb_kernel_range), %o0
    3939     sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
    3940     or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
    3941     call tlb_patch_one
    3942     - mov 16, %o2
    3943     + mov 31, %o2
    3944    
    3945     #ifdef DCACHE_ALIASING_POSSIBLE
    3946     sethi %hi(__flush_dcache_page), %o0
    3947     @@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops:
    3948     sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1
    3949     or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
    3950     call tlb_patch_one
    3951     - mov 21, %o2
    3952     + mov 24, %o2
    3953    
    3954     sethi %hi(xcall_flush_tlb_page), %o0
    3955     or %o0, %lo(xcall_flush_tlb_page), %o0
    3956     sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1
    3957     or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
    3958     call tlb_patch_one
    3959     - mov 17, %o2
    3960     + mov 20, %o2
    3961    
    3962     sethi %hi(xcall_flush_tlb_kernel_range), %o0
    3963     or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
    3964     sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
    3965     or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
    3966     call tlb_patch_one
    3967     - mov 25, %o2
    3968     + mov 44, %o2
    3969     #endif /* CONFIG_SMP */
    3970    
    3971     ret
    3972     diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
    3973     index c32f5d32f811..b56c9c581359 100644
    3974     --- a/drivers/net/ethernet/broadcom/bgmac.c
    3975     +++ b/drivers/net/ethernet/broadcom/bgmac.c
    3976     @@ -314,6 +314,10 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
    3977     u32 ctl;
    3978    
    3979     ctl = bgmac_read(bgmac, ring->mmio_base + BGMAC_DMA_RX_CTL);
    3980     +
    3981     + /* preserve ONLY bits 16-17 from current hardware value */
    3982     + ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
    3983     +
    3984     if (bgmac->core->id.rev >= 4) {
    3985     ctl &= ~BGMAC_DMA_RX_BL_MASK;
    3986     ctl |= BGMAC_DMA_RX_BL_128 << BGMAC_DMA_RX_BL_SHIFT;
    3987     @@ -324,7 +328,6 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
    3988     ctl &= ~BGMAC_DMA_RX_PT_MASK;
    3989     ctl |= BGMAC_DMA_RX_PT_1 << BGMAC_DMA_RX_PT_SHIFT;
    3990     }
    3991     - ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
    3992     ctl |= BGMAC_DMA_RX_ENABLE;
    3993     ctl |= BGMAC_DMA_RX_PARITY_DISABLE;
    3994     ctl |= BGMAC_DMA_RX_OVERFLOW_CONT;
    3995     diff --git a/drivers/tty/serial/sunhv.c b/drivers/tty/serial/sunhv.c
    3996     index ca0d3802f2af..4e603d060e80 100644
    3997     --- a/drivers/tty/serial/sunhv.c
    3998     +++ b/drivers/tty/serial/sunhv.c
    3999     @@ -490,12 +490,6 @@ static void sunhv_console_write_bychar(struct console *con, const char *s, unsig
    4000     locked = spin_trylock_irqsave(&port->lock, flags);
    4001     else
    4002     spin_lock_irqsave(&port->lock, flags);
    4003     - if (port->sysrq) {
    4004     - locked = 0;
    4005     - } else if (oops_in_progress) {
    4006     - locked = spin_trylock(&port->lock);
    4007     - } else
    4008     - spin_lock(&port->lock);
    4009    
    4010     for (i = 0; i < n; i++) {
    4011     if (*s == '\n')
    4012     diff --git a/drivers/tty/tty_ldisc.c b/drivers/tty/tty_ldisc.c
    4013     index 629e3c865072..9bee25cfa0be 100644
    4014     --- a/drivers/tty/tty_ldisc.c
    4015     +++ b/drivers/tty/tty_ldisc.c
    4016     @@ -417,6 +417,10 @@ EXPORT_SYMBOL_GPL(tty_ldisc_flush);
    4017     * they are not on hot paths so a little discipline won't do
    4018     * any harm.
    4019     *
    4020     + * The line discipline-related tty_struct fields are reset to
    4021     + * prevent the ldisc driver from re-using stale information for
    4022     + * the new ldisc instance.
    4023     + *
    4024     * Locking: takes termios_rwsem
    4025     */
    4026    
    4027     @@ -425,6 +429,9 @@ static void tty_set_termios_ldisc(struct tty_struct *tty, int num)
    4028     down_write(&tty->termios_rwsem);
    4029     tty->termios.c_line = num;
    4030     up_write(&tty->termios_rwsem);
    4031     +
    4032     + tty->disc_data = NULL;
    4033     + tty->receive_room = 0;
    4034     }
    4035    
    4036     /**
    4037     diff --git a/include/linux/filter.h b/include/linux/filter.h
    4038     index 5110d4211866..ccb98b459c59 100644
    4039     --- a/include/linux/filter.h
    4040     +++ b/include/linux/filter.h
    4041     @@ -421,7 +421,11 @@ static inline void bpf_prog_unlock_ro(struct bpf_prog *fp)
    4042     }
    4043     #endif /* CONFIG_DEBUG_SET_MODULE_RONX */
    4044    
    4045     -int sk_filter(struct sock *sk, struct sk_buff *skb);
    4046     +int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);
    4047     +static inline int sk_filter(struct sock *sk, struct sk_buff *skb)
    4048     +{
    4049     + return sk_filter_trim_cap(sk, skb, 1);
    4050     +}
    4051    
    4052     int bpf_prog_select_runtime(struct bpf_prog *fp);
    4053     void bpf_prog_free(struct bpf_prog *fp);
    4054     diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
    4055     index ff788b665277..9c2c044153f6 100644
    4056     --- a/include/net/ip6_tunnel.h
    4057     +++ b/include/net/ip6_tunnel.h
    4058     @@ -86,6 +86,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb,
    4059     struct net_device_stats *stats = &dev->stats;
    4060     int pkt_len, err;
    4061    
    4062     + memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
    4063     pkt_len = skb->len - skb_inner_network_offset(skb);
    4064     err = ip6_local_out(dev_net(skb_dst(skb)->dev), sk, skb);
    4065    
    4066     diff --git a/include/net/tcp.h b/include/net/tcp.h
    4067     index 9c3ab544d3a8..e9d7a8ef9a6d 100644
    4068     --- a/include/net/tcp.h
    4069     +++ b/include/net/tcp.h
    4070     @@ -1156,6 +1156,7 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp)
    4071     }
    4072    
    4073     bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
    4074     +int tcp_filter(struct sock *sk, struct sk_buff *skb);
    4075    
    4076     #undef STATE_TRACE
    4077    
    4078     diff --git a/net/core/dev.c b/net/core/dev.c
    4079     index b3fa4b86ab4c..9ca749c81b6c 100644
    4080     --- a/net/core/dev.c
    4081     +++ b/net/core/dev.c
    4082     @@ -2462,7 +2462,7 @@ int skb_checksum_help(struct sk_buff *skb)
    4083     goto out;
    4084     }
    4085    
    4086     - *(__sum16 *)(skb->data + offset) = csum_fold(csum);
    4087     + *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
    4088     out_set_summed:
    4089     skb->ip_summed = CHECKSUM_NONE;
    4090     out:
    4091     diff --git a/net/core/filter.c b/net/core/filter.c
    4092     index 75e9b2b2336d..e94355452166 100644
    4093     --- a/net/core/filter.c
    4094     +++ b/net/core/filter.c
    4095     @@ -52,9 +52,10 @@
    4096     #include <net/dst.h>
    4097    
    4098     /**
    4099     - * sk_filter - run a packet through a socket filter
    4100     + * sk_filter_trim_cap - run a packet through a socket filter
    4101     * @sk: sock associated with &sk_buff
    4102     * @skb: buffer to filter
    4103     + * @cap: limit on how short the eBPF program may trim the packet
    4104     *
    4105     * Run the eBPF program and then cut skb->data to correct size returned by
    4106     * the program. If pkt_len is 0 we toss packet. If skb->len is smaller
    4107     @@ -63,7 +64,7 @@
    4108     * be accepted or -EPERM if the packet should be tossed.
    4109     *
    4110     */
    4111     -int sk_filter(struct sock *sk, struct sk_buff *skb)
    4112     +int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap)
    4113     {
    4114     int err;
    4115     struct sk_filter *filter;
    4116     @@ -84,14 +85,13 @@ int sk_filter(struct sock *sk, struct sk_buff *skb)
    4117     filter = rcu_dereference(sk->sk_filter);
    4118     if (filter) {
    4119     unsigned int pkt_len = bpf_prog_run_save_cb(filter->prog, skb);
    4120     -
    4121     - err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM;
    4122     + err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM;
    4123     }
    4124     rcu_read_unlock();
    4125    
    4126     return err;
    4127     }
    4128     -EXPORT_SYMBOL(sk_filter);
    4129     +EXPORT_SYMBOL(sk_filter_trim_cap);
    4130    
    4131     static u64 __skb_get_pay_offset(u64 ctx, u64 a, u64 x, u64 r4, u64 r5)
    4132     {
    4133     diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
    4134     index 4ab6ead3d8ee..9aba9e93c0a2 100644
    4135     --- a/net/core/flow_dissector.c
    4136     +++ b/net/core/flow_dissector.c
    4137     @@ -131,7 +131,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
    4138     struct flow_dissector_key_tags *key_tags;
    4139     struct flow_dissector_key_keyid *key_keyid;
    4140     u8 ip_proto = 0;
    4141     - bool ret = false;
    4142     + bool ret;
    4143    
    4144     if (!data) {
    4145     data = skb->data;
    4146     @@ -492,12 +492,17 @@ ip_proto_again:
    4147     out_good:
    4148     ret = true;
    4149    
    4150     -out_bad:
    4151     + key_control->thoff = (u16)nhoff;
    4152     +out:
    4153     key_basic->n_proto = proto;
    4154     key_basic->ip_proto = ip_proto;
    4155     - key_control->thoff = (u16)nhoff;
    4156    
    4157     return ret;
    4158     +
    4159     +out_bad:
    4160     + ret = false;
    4161     + key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
    4162     + goto out;
    4163     }
    4164     EXPORT_SYMBOL(__skb_flow_dissect);
    4165    
    4166     diff --git a/net/core/sock.c b/net/core/sock.c
    4167     index 0d91f7dca751..88f017854509 100644
    4168     --- a/net/core/sock.c
    4169     +++ b/net/core/sock.c
    4170     @@ -1562,6 +1562,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
    4171     }
    4172    
    4173     newsk->sk_err = 0;
    4174     + newsk->sk_err_soft = 0;
    4175     newsk->sk_priority = 0;
    4176     newsk->sk_incoming_cpu = raw_smp_processor_id();
    4177     atomic64_set(&newsk->sk_cookie, 0);
    4178     diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
    4179     index 8be8f27bfacc..861e1fa25d5e 100644
    4180     --- a/net/dccp/ipv4.c
    4181     +++ b/net/dccp/ipv4.c
    4182     @@ -235,7 +235,7 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
    4183     {
    4184     const struct iphdr *iph = (struct iphdr *)skb->data;
    4185     const u8 offset = iph->ihl << 2;
    4186     - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
    4187     + const struct dccp_hdr *dh;
    4188     struct dccp_sock *dp;
    4189     struct inet_sock *inet;
    4190     const int type = icmp_hdr(skb)->type;
    4191     @@ -245,11 +245,13 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
    4192     int err;
    4193     struct net *net = dev_net(skb->dev);
    4194    
    4195     - if (skb->len < offset + sizeof(*dh) ||
    4196     - skb->len < offset + __dccp_basic_hdr_len(dh)) {
    4197     - ICMP_INC_STATS_BH(net, ICMP_MIB_INERRORS);
    4198     - return;
    4199     - }
    4200     + /* Only need dccph_dport & dccph_sport which are the first
    4201     + * 4 bytes in dccp header.
    4202     + * Our caller (icmp_socket_deliver()) already pulled 8 bytes for us.
    4203     + */
    4204     + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
    4205     + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
    4206     + dh = (struct dccp_hdr *)(skb->data + offset);
    4207    
    4208     sk = __inet_lookup_established(net, &dccp_hashinfo,
    4209     iph->daddr, dh->dccph_dport,
    4210     diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
    4211     index b8608b71a66d..27c4e81efa24 100644
    4212     --- a/net/dccp/ipv6.c
    4213     +++ b/net/dccp/ipv6.c
    4214     @@ -70,7 +70,7 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
    4215     u8 type, u8 code, int offset, __be32 info)
    4216     {
    4217     const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
    4218     - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
    4219     + const struct dccp_hdr *dh;
    4220     struct dccp_sock *dp;
    4221     struct ipv6_pinfo *np;
    4222     struct sock *sk;
    4223     @@ -78,12 +78,13 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
    4224     __u64 seq;
    4225     struct net *net = dev_net(skb->dev);
    4226    
    4227     - if (skb->len < offset + sizeof(*dh) ||
    4228     - skb->len < offset + __dccp_basic_hdr_len(dh)) {
    4229     - ICMP6_INC_STATS_BH(net, __in6_dev_get(skb->dev),
    4230     - ICMP6_MIB_INERRORS);
    4231     - return;
    4232     - }
    4233     + /* Only need dccph_dport & dccph_sport which are the first
    4234     + * 4 bytes in dccp header.
    4235     + * Our caller (icmpv6_notify()) already pulled 8 bytes for us.
    4236     + */
    4237     + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
    4238     + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
    4239     + dh = (struct dccp_hdr *)(skb->data + offset);
    4240    
    4241     sk = __inet6_lookup_established(net, &dccp_hashinfo,
    4242     &hdr->daddr, dh->dccph_dport,
    4243     @@ -947,6 +948,7 @@ static const struct inet_connection_sock_af_ops dccp_ipv6_mapped = {
    4244     .getsockopt = ipv6_getsockopt,
    4245     .addr2sockaddr = inet6_csk_addr2sockaddr,
    4246     .sockaddr_len = sizeof(struct sockaddr_in6),
    4247     + .bind_conflict = inet6_csk_bind_conflict,
    4248     #ifdef CONFIG_COMPAT
    4249     .compat_setsockopt = compat_ipv6_setsockopt,
    4250     .compat_getsockopt = compat_ipv6_getsockopt,
    4251     diff --git a/net/dccp/proto.c b/net/dccp/proto.c
    4252     index 41e65804ddf5..9fe25bf63296 100644
    4253     --- a/net/dccp/proto.c
    4254     +++ b/net/dccp/proto.c
    4255     @@ -1009,6 +1009,10 @@ void dccp_close(struct sock *sk, long timeout)
    4256     __kfree_skb(skb);
    4257     }
    4258    
    4259     + /* If socket has been already reset kill it. */
    4260     + if (sk->sk_state == DCCP_CLOSED)
    4261     + goto adjudge_to_death;
    4262     +
    4263     if (data_was_unread) {
    4264     /* Unread data was tossed, send an appropriate Reset Code */
    4265     DCCP_WARN("ABORT with %u bytes unread\n", data_was_unread);
    4266     diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
    4267     index e5a3ff210fec..7c52afb98c42 100644
    4268     --- a/net/ipv4/fib_trie.c
    4269     +++ b/net/ipv4/fib_trie.c
    4270     @@ -2456,22 +2456,19 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
    4271     struct key_vector *l, **tp = &iter->tnode;
    4272     t_key key;
    4273    
    4274     - /* use cache location of next-to-find key */
    4275     + /* use cached location of previously found key */
    4276     if (iter->pos > 0 && pos >= iter->pos) {
    4277     - pos -= iter->pos;
    4278     key = iter->key;
    4279     } else {
    4280     - iter->pos = 0;
    4281     + iter->pos = 1;
    4282     key = 0;
    4283     }
    4284    
    4285     - while ((l = leaf_walk_rcu(tp, key)) != NULL) {
    4286     + pos -= iter->pos;
    4287     +
    4288     + while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) {
    4289     key = l->key + 1;
    4290     iter->pos++;
    4291     -
    4292     - if (--pos <= 0)
    4293     - break;
    4294     -
    4295     l = NULL;
    4296    
    4297     /* handle unlikely case of a key wrap */
    4298     @@ -2480,7 +2477,7 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
    4299     }
    4300    
    4301     if (l)
    4302     - iter->key = key; /* remember it */
    4303     + iter->key = l->key; /* remember it */
    4304     else
    4305     iter->pos = 0; /* forget it */
    4306    
    4307     @@ -2508,7 +2505,7 @@ static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos)
    4308     return fib_route_get_idx(iter, *pos);
    4309    
    4310     iter->pos = 0;
    4311     - iter->key = 0;
    4312     + iter->key = KEY_MAX;
    4313    
    4314     return SEQ_START_TOKEN;
    4315     }
    4316     @@ -2517,7 +2514,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
    4317     {
    4318     struct fib_route_iter *iter = seq->private;
    4319     struct key_vector *l = NULL;
    4320     - t_key key = iter->key;
    4321     + t_key key = iter->key + 1;
    4322    
    4323     ++*pos;
    4324    
    4325     @@ -2526,7 +2523,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
    4326     l = leaf_walk_rcu(&iter->tnode, key);
    4327    
    4328     if (l) {
    4329     - iter->key = l->key + 1;
    4330     + iter->key = l->key;
    4331     iter->pos++;
    4332     } else {
    4333     iter->pos = 0;
    4334     diff --git a/net/ipv4/route.c b/net/ipv4/route.c
    4335     index 8533a75a9328..7ceb8a574a50 100644
    4336     --- a/net/ipv4/route.c
    4337     +++ b/net/ipv4/route.c
    4338     @@ -747,7 +747,9 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
    4339     goto reject_redirect;
    4340     }
    4341    
    4342     - n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
    4343     + n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
    4344     + if (!n)
    4345     + n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
    4346     if (!IS_ERR(n)) {
    4347     if (!(n->nud_state & NUD_VALID)) {
    4348     neigh_event_send(n, NULL);
    4349     diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
    4350     index 036a76ba2ac2..69daa81736f6 100644
    4351     --- a/net/ipv4/tcp.c
    4352     +++ b/net/ipv4/tcp.c
    4353     @@ -1212,7 +1212,7 @@ new_segment:
    4354    
    4355     if (!skb_can_coalesce(skb, i, pfrag->page,
    4356     pfrag->offset)) {
    4357     - if (i == sysctl_max_skb_frags || !sg) {
    4358     + if (i >= sysctl_max_skb_frags || !sg) {
    4359     tcp_mark_push(tp, skb);
    4360     goto new_segment;
    4361     }
    4362     diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c
    4363     index 7e538f71f5fb..55d7da1d2ce9 100644
    4364     --- a/net/ipv4/tcp_dctcp.c
    4365     +++ b/net/ipv4/tcp_dctcp.c
    4366     @@ -56,6 +56,7 @@ struct dctcp {
    4367     u32 next_seq;
    4368     u32 ce_state;
    4369     u32 delayed_ack_reserved;
    4370     + u32 loss_cwnd;
    4371     };
    4372    
    4373     static unsigned int dctcp_shift_g __read_mostly = 4; /* g = 1/2^4 */
    4374     @@ -96,6 +97,7 @@ static void dctcp_init(struct sock *sk)
    4375     ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
    4376    
    4377     ca->delayed_ack_reserved = 0;
    4378     + ca->loss_cwnd = 0;
    4379     ca->ce_state = 0;
    4380    
    4381     dctcp_reset(tp, ca);
    4382     @@ -111,9 +113,10 @@ static void dctcp_init(struct sock *sk)
    4383    
    4384     static u32 dctcp_ssthresh(struct sock *sk)
    4385     {
    4386     - const struct dctcp *ca = inet_csk_ca(sk);
    4387     + struct dctcp *ca = inet_csk_ca(sk);
    4388     struct tcp_sock *tp = tcp_sk(sk);
    4389    
    4390     + ca->loss_cwnd = tp->snd_cwnd;
    4391     return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
    4392     }
    4393    
    4394     @@ -308,12 +311,20 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr,
    4395     return 0;
    4396     }
    4397    
    4398     +static u32 dctcp_cwnd_undo(struct sock *sk)
    4399     +{
    4400     + const struct dctcp *ca = inet_csk_ca(sk);
    4401     +
    4402     + return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
    4403     +}
    4404     +
    4405     static struct tcp_congestion_ops dctcp __read_mostly = {
    4406     .init = dctcp_init,
    4407     .in_ack_event = dctcp_update_alpha,
    4408     .cwnd_event = dctcp_cwnd_event,
    4409     .ssthresh = dctcp_ssthresh,
    4410     .cong_avoid = tcp_reno_cong_avoid,
    4411     + .undo_cwnd = dctcp_cwnd_undo,
    4412     .set_state = dctcp_state,
    4413     .get_info = dctcp_get_info,
    4414     .flags = TCP_CONG_NEEDS_ECN,
    4415     diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
    4416     index b5853cac3269..b58a38eea059 100644
    4417     --- a/net/ipv4/tcp_ipv4.c
    4418     +++ b/net/ipv4/tcp_ipv4.c
    4419     @@ -1533,6 +1533,21 @@ bool tcp_prequeue(struct sock *sk, struct sk_buff *skb)
    4420     }
    4421     EXPORT_SYMBOL(tcp_prequeue);
    4422    
    4423     +int tcp_filter(struct sock *sk, struct sk_buff *skb)
    4424     +{
    4425     + struct tcphdr *th = (struct tcphdr *)skb->data;
    4426     + unsigned int eaten = skb->len;
    4427     + int err;
    4428     +
    4429     + err = sk_filter_trim_cap(sk, skb, th->doff * 4);
    4430     + if (!err) {
    4431     + eaten -= skb->len;
    4432     + TCP_SKB_CB(skb)->end_seq -= eaten;
    4433     + }
    4434     + return err;
    4435     +}
    4436     +EXPORT_SYMBOL(tcp_filter);
    4437     +
    4438     /*
    4439     * From tcp_input.c
    4440     */
    4441     @@ -1638,8 +1653,10 @@ process:
    4442    
    4443     nf_reset(skb);
    4444    
    4445     - if (sk_filter(sk, skb))
    4446     + if (tcp_filter(sk, skb))
    4447     goto discard_and_relse;
    4448     + th = (const struct tcphdr *)skb->data;
    4449     + iph = ip_hdr(skb);
    4450    
    4451     skb->dev = NULL;
    4452    
    4453     diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
    4454     index fbd521fdae53..5f581616bf6a 100644
    4455     --- a/net/ipv6/tcp_ipv6.c
    4456     +++ b/net/ipv6/tcp_ipv6.c
    4457     @@ -1214,7 +1214,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
    4458     if (skb->protocol == htons(ETH_P_IP))
    4459     return tcp_v4_do_rcv(sk, skb);
    4460    
    4461     - if (sk_filter(sk, skb))
    4462     + if (tcp_filter(sk, skb))
    4463     goto discard;
    4464    
    4465     /*
    4466     @@ -1438,8 +1438,10 @@ process:
    4467     if (tcp_v6_inbound_md5_hash(sk, skb))
    4468     goto discard_and_relse;
    4469    
    4470     - if (sk_filter(sk, skb))
    4471     + if (tcp_filter(sk, skb))
    4472     goto discard_and_relse;
    4473     + th = (const struct tcphdr *)skb->data;
    4474     + hdr = ipv6_hdr(skb);
    4475    
    4476     skb->dev = NULL;
    4477    
    4478     diff --git a/net/sctp/socket.c b/net/sctp/socket.c
    4479     index 402817be3873..b5fd4ab56156 100644
    4480     --- a/net/sctp/socket.c
    4481     +++ b/net/sctp/socket.c
    4482     @@ -1212,9 +1212,12 @@ static int __sctp_connect(struct sock *sk,
    4483    
    4484     timeo = sock_sndtimeo(sk, f_flags & O_NONBLOCK);
    4485    
    4486     - err = sctp_wait_for_connect(asoc, &timeo);
    4487     - if ((err == 0 || err == -EINPROGRESS) && assoc_id)
    4488     + if (assoc_id)
    4489     *assoc_id = asoc->assoc_id;
    4490     + err = sctp_wait_for_connect(asoc, &timeo);
    4491     + /* Note: the asoc may be freed after the return of
    4492     + * sctp_wait_for_connect.
    4493     + */
    4494    
    4495     /* Don't free association on exit. */
    4496     asoc = NULL;
    4497     diff --git a/net/socket.c b/net/socket.c
    4498     index 263b334ec5e4..0090225eeb1e 100644
    4499     --- a/net/socket.c
    4500     +++ b/net/socket.c
    4501     @@ -2041,6 +2041,8 @@ int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen,
    4502     if (err)
    4503     break;
    4504     ++datagrams;
    4505     + if (msg_data_left(&msg_sys))
    4506     + break;
    4507     }
    4508    
    4509     fput_light(sock->file, fput_needed);