Magellan Linux

  • Back to homepage
    • /[pkg-src]/trunk/kernel-alx/patches-4.4/0133-4.4.34-all-fixes.patch

    Contents of /trunk/kernel-alx/patches-4.4/0133-4.4.34-all-fixes.patch

    Parent Directory Parent Directory | Revision Log Revision Log

    Revision 2869 - (show annotations) (download)
    Mon Mar 27 13:49:12 2017 UTC (7 years, 1 month ago) by niro
    File size: 133458 byte(s) 
    linux-4.4.34
    1 diff --git a/Makefile b/Makefile
    2 index a513c045c8de..30924aabf1b4 100644
    3 --- a/Makefile
    4 +++ b/Makefile
    5 @@ -1,6 +1,6 @@
    6 VERSION = 4
    7 PATCHLEVEL = 4
    8 -SUBLEVEL = 33
    9 +SUBLEVEL = 34
    10 EXTRAVERSION =
    11 NAME = Blurry Fish Butt
    12
    13 diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h
    14 index 70067ce184b1..f7de0dbc38af 100644
    15 --- a/arch/sparc/include/asm/mmu_64.h
    16 +++ b/arch/sparc/include/asm/mmu_64.h
    17 @@ -92,7 +92,8 @@ struct tsb_config {
    18 typedef struct {
    19 spinlock_t lock;
    20 unsigned long sparc64_ctx_val;
    21 - unsigned long huge_pte_count;
    22 + unsigned long hugetlb_pte_count;
    23 + unsigned long thp_pte_count;
    24 struct tsb_config tsb_block[MM_NUM_TSBS];
    25 struct hv_tsb_descr tsb_descr[MM_NUM_TSBS];
    26 } mm_context_t;
    27 diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
    28 index ea6e9a20f3ff..f428512481f9 100644
    29 --- a/arch/sparc/include/asm/uaccess_64.h
    30 +++ b/arch/sparc/include/asm/uaccess_64.h
    31 @@ -98,7 +98,6 @@ struct exception_table_entry {
    32 unsigned int insn, fixup;
    33 };
    34
    35 -void __ret_efault(void);
    36 void __retl_efault(void);
    37
    38 /* Uh, these should become the main single-value transfer routines..
    39 @@ -179,20 +178,6 @@ int __put_user_bad(void);
    40 __gu_ret; \
    41 })
    42
    43 -#define __get_user_nocheck_ret(data, addr, size, type, retval) ({ \
    44 - register unsigned long __gu_val __asm__ ("l1"); \
    45 - switch (size) { \
    46 - case 1: __get_user_asm_ret(__gu_val, ub, addr, retval); break; \
    47 - case 2: __get_user_asm_ret(__gu_val, uh, addr, retval); break; \
    48 - case 4: __get_user_asm_ret(__gu_val, uw, addr, retval); break; \
    49 - case 8: __get_user_asm_ret(__gu_val, x, addr, retval); break; \
    50 - default: \
    51 - if (__get_user_bad()) \
    52 - return retval; \
    53 - } \
    54 - data = (__force type) __gu_val; \
    55 -})
    56 -
    57 #define __get_user_asm(x, size, addr, ret) \
    58 __asm__ __volatile__( \
    59 "/* Get user asm, inline. */\n" \
    60 @@ -214,80 +199,35 @@ __asm__ __volatile__( \
    61 : "=r" (ret), "=r" (x) : "r" (__m(addr)), \
    62 "i" (-EFAULT))
    63
    64 -#define __get_user_asm_ret(x, size, addr, retval) \
    65 -if (__builtin_constant_p(retval) && retval == -EFAULT) \
    66 - __asm__ __volatile__( \
    67 - "/* Get user asm ret, inline. */\n" \
    68 - "1:\t" "ld"#size "a [%1] %%asi, %0\n\n\t" \
    69 - ".section __ex_table,\"a\"\n\t" \
    70 - ".align 4\n\t" \
    71 - ".word 1b,__ret_efault\n\n\t" \
    72 - ".previous\n\t" \
    73 - : "=r" (x) : "r" (__m(addr))); \
    74 -else \
    75 - __asm__ __volatile__( \
    76 - "/* Get user asm ret, inline. */\n" \
    77 - "1:\t" "ld"#size "a [%1] %%asi, %0\n\n\t" \
    78 - ".section .fixup,#alloc,#execinstr\n\t" \
    79 - ".align 4\n" \
    80 - "3:\n\t" \
    81 - "ret\n\t" \
    82 - " restore %%g0, %2, %%o0\n\n\t" \
    83 - ".previous\n\t" \
    84 - ".section __ex_table,\"a\"\n\t" \
    85 - ".align 4\n\t" \
    86 - ".word 1b, 3b\n\n\t" \
    87 - ".previous\n\t" \
    88 - : "=r" (x) : "r" (__m(addr)), "i" (retval))
    89 -
    90 int __get_user_bad(void);
    91
    92 unsigned long __must_check ___copy_from_user(void *to,
    93 const void __user *from,
    94 unsigned long size);
    95 -unsigned long copy_from_user_fixup(void *to, const void __user *from,
    96 - unsigned long size);
    97 static inline unsigned long __must_check
    98 copy_from_user(void *to, const void __user *from, unsigned long size)
    99 {
    100 - unsigned long ret = ___copy_from_user(to, from, size);
    101 -
    102 - if (unlikely(ret))
    103 - ret = copy_from_user_fixup(to, from, size);
    104 -
    105 - return ret;
    106 + return ___copy_from_user(to, from, size);
    107 }
    108 #define __copy_from_user copy_from_user
    109
    110 unsigned long __must_check ___copy_to_user(void __user *to,
    111 const void *from,
    112 unsigned long size);
    113 -unsigned long copy_to_user_fixup(void __user *to, const void *from,
    114 - unsigned long size);
    115 static inline unsigned long __must_check
    116 copy_to_user(void __user *to, const void *from, unsigned long size)
    117 {
    118 - unsigned long ret = ___copy_to_user(to, from, size);
    119 -
    120 - if (unlikely(ret))
    121 - ret = copy_to_user_fixup(to, from, size);
    122 - return ret;
    123 + return ___copy_to_user(to, from, size);
    124 }
    125 #define __copy_to_user copy_to_user
    126
    127 unsigned long __must_check ___copy_in_user(void __user *to,
    128 const void __user *from,
    129 unsigned long size);
    130 -unsigned long copy_in_user_fixup(void __user *to, void __user *from,
    131 - unsigned long size);
    132 static inline unsigned long __must_check
    133 copy_in_user(void __user *to, void __user *from, unsigned long size)
    134 {
    135 - unsigned long ret = ___copy_in_user(to, from, size);
    136 -
    137 - if (unlikely(ret))
    138 - ret = copy_in_user_fixup(to, from, size);
    139 - return ret;
    140 + return ___copy_in_user(to, from, size);
    141 }
    142 #define __copy_in_user copy_in_user
    143
    144 diff --git a/arch/sparc/kernel/dtlb_prot.S b/arch/sparc/kernel/dtlb_prot.S
    145 index d668ca149e64..4087a62f96b0 100644
    146 --- a/arch/sparc/kernel/dtlb_prot.S
    147 +++ b/arch/sparc/kernel/dtlb_prot.S
    148 @@ -25,13 +25,13 @@
    149
    150 /* PROT ** ICACHE line 2: More real fault processing */
    151 ldxa [%g4] ASI_DMMU, %g5 ! Put tagaccess in %g5
    152 + srlx %g5, PAGE_SHIFT, %g5
    153 + sllx %g5, PAGE_SHIFT, %g5 ! Clear context ID bits
    154 bgu,pn %xcc, winfix_trampoline ! Yes, perform winfixup
    155 mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
    156 ba,pt %xcc, sparc64_realfault_common ! Nope, normal fault
    157 nop
    158 nop
    159 - nop
    160 - nop
    161
    162 /* PROT ** ICACHE line 3: Unused... */
    163 nop
    164 diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
    165 index 51faf92ace00..7eeeb1d5a410 100644
    166 --- a/arch/sparc/kernel/head_64.S
    167 +++ b/arch/sparc/kernel/head_64.S
    168 @@ -922,47 +922,11 @@ prom_tba: .xword 0
    169 tlb_type: .word 0 /* Must NOT end up in BSS */
    170 .section ".fixup",#alloc,#execinstr
    171
    172 - .globl __ret_efault, __retl_efault, __ret_one, __retl_one
    173 -ENTRY(__ret_efault)
    174 - ret
    175 - restore %g0, -EFAULT, %o0
    176 -ENDPROC(__ret_efault)
    177 -
    178 ENTRY(__retl_efault)
    179 retl
    180 mov -EFAULT, %o0
    181 ENDPROC(__retl_efault)
    182
    183 -ENTRY(__retl_one)
    184 - retl
    185 - mov 1, %o0
    186 -ENDPROC(__retl_one)
    187 -
    188 -ENTRY(__retl_one_fp)
    189 - VISExitHalf
    190 - retl
    191 - mov 1, %o0
    192 -ENDPROC(__retl_one_fp)
    193 -
    194 -ENTRY(__ret_one_asi)
    195 - wr %g0, ASI_AIUS, %asi
    196 - ret
    197 - restore %g0, 1, %o0
    198 -ENDPROC(__ret_one_asi)
    199 -
    200 -ENTRY(__retl_one_asi)
    201 - wr %g0, ASI_AIUS, %asi
    202 - retl
    203 - mov 1, %o0
    204 -ENDPROC(__retl_one_asi)
    205 -
    206 -ENTRY(__retl_one_asi_fp)
    207 - wr %g0, ASI_AIUS, %asi
    208 - VISExitHalf
    209 - retl
    210 - mov 1, %o0
    211 -ENDPROC(__retl_one_asi_fp)
    212 -
    213 ENTRY(__retl_o1)
    214 retl
    215 mov %o1, %o0
    216 diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
    217 index 59bbeff55024..07933b9e9ce0 100644
    218 --- a/arch/sparc/kernel/jump_label.c
    219 +++ b/arch/sparc/kernel/jump_label.c
    220 @@ -13,19 +13,30 @@
    221 void arch_jump_label_transform(struct jump_entry *entry,
    222 enum jump_label_type type)
    223 {
    224 - u32 val;
    225 u32 *insn = (u32 *) (unsigned long) entry->code;
    226 + u32 val;
    227
    228 if (type == JUMP_LABEL_JMP) {
    229 s32 off = (s32)entry->target - (s32)entry->code;
    230 + bool use_v9_branch = false;
    231 +
    232 + BUG_ON(off & 3);
    233
    234 #ifdef CONFIG_SPARC64
    235 - /* ba,pt %xcc, . + (off << 2) */
    236 - val = 0x10680000 | ((u32) off >> 2);
    237 -#else
    238 - /* ba . + (off << 2) */
    239 - val = 0x10800000 | ((u32) off >> 2);
    240 + if (off <= 0xfffff && off >= -0x100000)
    241 + use_v9_branch = true;
    242 #endif
    243 + if (use_v9_branch) {
    244 + /* WDISP19 - target is . + immed << 2 */
    245 + /* ba,pt %xcc, . + off */
    246 + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff);
    247 + } else {
    248 + /* WDISP22 - target is . + immed << 2 */
    249 + BUG_ON(off > 0x7fffff);
    250 + BUG_ON(off < -0x800000);
    251 + /* ba . + off */
    252 + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff);
    253 + }
    254 } else {
    255 val = 0x01000000;
    256 }
    257 diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S
    258 index ef0d8e9e1210..f22bec0db645 100644
    259 --- a/arch/sparc/kernel/ktlb.S
    260 +++ b/arch/sparc/kernel/ktlb.S
    261 @@ -20,6 +20,10 @@ kvmap_itlb:
    262 mov TLB_TAG_ACCESS, %g4
    263 ldxa [%g4] ASI_IMMU, %g4
    264
    265 + /* The kernel executes in context zero, therefore we do not
    266 + * need to clear the context ID bits out of %g4 here.
    267 + */
    268 +
    269 /* sun4v_itlb_miss branches here with the missing virtual
    270 * address already loaded into %g4
    271 */
    272 @@ -128,6 +132,10 @@ kvmap_dtlb:
    273 mov TLB_TAG_ACCESS, %g4
    274 ldxa [%g4] ASI_DMMU, %g4
    275
    276 + /* The kernel executes in context zero, therefore we do not
    277 + * need to clear the context ID bits out of %g4 here.
    278 + */
    279 +
    280 /* sun4v_dtlb_miss branches here with the missing virtual
    281 * address already loaded into %g4
    282 */
    283 @@ -251,6 +259,10 @@ kvmap_dtlb_longpath:
    284 nop
    285 .previous
    286
    287 + /* The kernel executes in context zero, therefore we do not
    288 + * need to clear the context ID bits out of %g5 here.
    289 + */
    290 +
    291 be,pt %xcc, sparc64_realfault_common
    292 mov FAULT_CODE_DTLB, %g4
    293 ba,pt %xcc, winfix_trampoline
    294 diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c
    295 index a92d5d2c46a3..51b25325a961 100644
    296 --- a/arch/sparc/kernel/sparc_ksyms_64.c
    297 +++ b/arch/sparc/kernel/sparc_ksyms_64.c
    298 @@ -27,7 +27,6 @@ EXPORT_SYMBOL(__flushw_user);
    299 EXPORT_SYMBOL_GPL(real_hard_smp_processor_id);
    300
    301 /* from head_64.S */
    302 -EXPORT_SYMBOL(__ret_efault);
    303 EXPORT_SYMBOL(tlb_type);
    304 EXPORT_SYMBOL(sun4v_chip_type);
    305 EXPORT_SYMBOL(prom_root_node);
    306 diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S
    307 index be98685c14c6..d568c8207af7 100644
    308 --- a/arch/sparc/kernel/tsb.S
    309 +++ b/arch/sparc/kernel/tsb.S
    310 @@ -29,13 +29,17 @@
    311 */
    312 tsb_miss_dtlb:
    313 mov TLB_TAG_ACCESS, %g4
    314 + ldxa [%g4] ASI_DMMU, %g4
    315 + srlx %g4, PAGE_SHIFT, %g4
    316 ba,pt %xcc, tsb_miss_page_table_walk
    317 - ldxa [%g4] ASI_DMMU, %g4
    318 + sllx %g4, PAGE_SHIFT, %g4
    319
    320 tsb_miss_itlb:
    321 mov TLB_TAG_ACCESS, %g4
    322 + ldxa [%g4] ASI_IMMU, %g4
    323 + srlx %g4, PAGE_SHIFT, %g4
    324 ba,pt %xcc, tsb_miss_page_table_walk
    325 - ldxa [%g4] ASI_IMMU, %g4
    326 + sllx %g4, PAGE_SHIFT, %g4
    327
    328 /* At this point we have:
    329 * %g1 -- PAGE_SIZE TSB entry address
    330 @@ -284,6 +288,10 @@ tsb_do_dtlb_fault:
    331 nop
    332 .previous
    333
    334 + /* Clear context ID bits. */
    335 + srlx %g5, PAGE_SHIFT, %g5
    336 + sllx %g5, PAGE_SHIFT, %g5
    337 +
    338 be,pt %xcc, sparc64_realfault_common
    339 mov FAULT_CODE_DTLB, %g4
    340 ba,pt %xcc, winfix_trampoline
    341 diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
    342 index b7d0bd6b1406..69a439fa2fc1 100644
    343 --- a/arch/sparc/lib/GENcopy_from_user.S
    344 +++ b/arch/sparc/lib/GENcopy_from_user.S
    345 @@ -3,11 +3,11 @@
    346 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    347 */
    348
    349 -#define EX_LD(x) \
    350 +#define EX_LD(x,y) \
    351 98: x; \
    352 .section __ex_table,"a";\
    353 .align 4; \
    354 - .word 98b, __retl_one; \
    355 + .word 98b, y; \
    356 .text; \
    357 .align 4;
    358
    359 diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
    360 index 780550e1afc7..9947427ce354 100644
    361 --- a/arch/sparc/lib/GENcopy_to_user.S
    362 +++ b/arch/sparc/lib/GENcopy_to_user.S
    363 @@ -3,11 +3,11 @@
    364 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    365 */
    366
    367 -#define EX_ST(x) \
    368 +#define EX_ST(x,y) \
    369 98: x; \
    370 .section __ex_table,"a";\
    371 .align 4; \
    372 - .word 98b, __retl_one; \
    373 + .word 98b, y; \
    374 .text; \
    375 .align 4;
    376
    377 diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S
    378 index 89358ee94851..059ea24ad73d 100644
    379 --- a/arch/sparc/lib/GENmemcpy.S
    380 +++ b/arch/sparc/lib/GENmemcpy.S
    381 @@ -4,21 +4,18 @@
    382 */
    383
    384 #ifdef __KERNEL__
    385 +#include <linux/linkage.h>
    386 #define GLOBAL_SPARE %g7
    387 #else
    388 #define GLOBAL_SPARE %g5
    389 #endif
    390
    391 #ifndef EX_LD
    392 -#define EX_LD(x) x
    393 +#define EX_LD(x,y) x
    394 #endif
    395
    396 #ifndef EX_ST
    397 -#define EX_ST(x) x
    398 -#endif
    399 -
    400 -#ifndef EX_RETVAL
    401 -#define EX_RETVAL(x) x
    402 +#define EX_ST(x,y) x
    403 #endif
    404
    405 #ifndef LOAD
    406 @@ -45,6 +42,29 @@
    407 .register %g3,#scratch
    408
    409 .text
    410 +
    411 +#ifndef EX_RETVAL
    412 +#define EX_RETVAL(x) x
    413 +ENTRY(GEN_retl_o4_1)
    414 + add %o4, %o2, %o4
    415 + retl
    416 + add %o4, 1, %o0
    417 +ENDPROC(GEN_retl_o4_1)
    418 +ENTRY(GEN_retl_g1_8)
    419 + add %g1, %o2, %g1
    420 + retl
    421 + add %g1, 8, %o0
    422 +ENDPROC(GEN_retl_g1_8)
    423 +ENTRY(GEN_retl_o2_4)
    424 + retl
    425 + add %o2, 4, %o0
    426 +ENDPROC(GEN_retl_o2_4)
    427 +ENTRY(GEN_retl_o2_1)
    428 + retl
    429 + add %o2, 1, %o0
    430 +ENDPROC(GEN_retl_o2_1)
    431 +#endif
    432 +
    433 .align 64
    434
    435 .globl FUNC_NAME
    436 @@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    437 sub %g0, %o4, %o4
    438 sub %o2, %o4, %o2
    439 1: subcc %o4, 1, %o4
    440 - EX_LD(LOAD(ldub, %o1, %g1))
    441 - EX_ST(STORE(stb, %g1, %o0))
    442 + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1)
    443 + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1)
    444 add %o1, 1, %o1
    445 bne,pt %XCC, 1b
    446 add %o0, 1, %o0
    447 @@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    448 andn %o2, 0x7, %g1
    449 sub %o2, %g1, %o2
    450 1: subcc %g1, 0x8, %g1
    451 - EX_LD(LOAD(ldx, %o1, %g2))
    452 - EX_ST(STORE(stx, %g2, %o0))
    453 + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8)
    454 + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8)
    455 add %o1, 0x8, %o1
    456 bne,pt %XCC, 1b
    457 add %o0, 0x8, %o0
    458 @@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    459
    460 1:
    461 subcc %o2, 4, %o2
    462 - EX_LD(LOAD(lduw, %o1, %g1))
    463 - EX_ST(STORE(stw, %g1, %o1 + %o3))
    464 + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4)
    465 + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4)
    466 bgu,pt %XCC, 1b
    467 add %o1, 4, %o1
    468
    469 @@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    470 .align 32
    471 90:
    472 subcc %o2, 1, %o2
    473 - EX_LD(LOAD(ldub, %o1, %g1))
    474 - EX_ST(STORE(stb, %g1, %o1 + %o3))
    475 + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1)
    476 + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1)
    477 bgu,pt %XCC, 90b
    478 add %o1, 1, %o1
    479 retl
    480 diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
    481 index 3269b0234093..4f2384a4286a 100644
    482 --- a/arch/sparc/lib/Makefile
    483 +++ b/arch/sparc/lib/Makefile
    484 @@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o
    485 lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
    486 lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
    487
    488 -lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o
    489 +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
    490 lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
    491
    492 obj-$(CONFIG_SPARC64) += iomap.o
    493 diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
    494 index d5242b8c4f94..b79a6998d87c 100644
    495 --- a/arch/sparc/lib/NG2copy_from_user.S
    496 +++ b/arch/sparc/lib/NG2copy_from_user.S
    497 @@ -3,19 +3,19 @@
    498 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    499 */
    500
    501 -#define EX_LD(x) \
    502 +#define EX_LD(x,y) \
    503 98: x; \
    504 .section __ex_table,"a";\
    505 .align 4; \
    506 - .word 98b, __retl_one_asi;\
    507 + .word 98b, y; \
    508 .text; \
    509 .align 4;
    510
    511 -#define EX_LD_FP(x) \
    512 +#define EX_LD_FP(x,y) \
    513 98: x; \
    514 .section __ex_table,"a";\
    515 .align 4; \
    516 - .word 98b, __retl_one_asi_fp;\
    517 + .word 98b, y##_fp; \
    518 .text; \
    519 .align 4;
    520
    521 diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
    522 index 4e962d993b10..dcec55f254ab 100644
    523 --- a/arch/sparc/lib/NG2copy_to_user.S
    524 +++ b/arch/sparc/lib/NG2copy_to_user.S
    525 @@ -3,19 +3,19 @@
    526 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
    527 */
    528
    529 -#define EX_ST(x) \
    530 +#define EX_ST(x,y) \
    531 98: x; \
    532 .section __ex_table,"a";\
    533 .align 4; \
    534 - .word 98b, __retl_one_asi;\
    535 + .word 98b, y; \
    536 .text; \
    537 .align 4;
    538
    539 -#define EX_ST_FP(x) \
    540 +#define EX_ST_FP(x,y) \
    541 98: x; \
    542 .section __ex_table,"a";\
    543 .align 4; \
    544 - .word 98b, __retl_one_asi_fp;\
    545 + .word 98b, y##_fp; \
    546 .text; \
    547 .align 4;
    548
    549 diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
    550 index d5f585df2f3f..c629dbd121b6 100644
    551 --- a/arch/sparc/lib/NG2memcpy.S
    552 +++ b/arch/sparc/lib/NG2memcpy.S
    553 @@ -4,6 +4,7 @@
    554 */
    555
    556 #ifdef __KERNEL__
    557 +#include <linux/linkage.h>
    558 #include <asm/visasm.h>
    559 #include <asm/asi.h>
    560 #define GLOBAL_SPARE %g7
    561 @@ -32,21 +33,17 @@
    562 #endif
    563
    564 #ifndef EX_LD
    565 -#define EX_LD(x) x
    566 +#define EX_LD(x,y) x
    567 #endif
    568 #ifndef EX_LD_FP
    569 -#define EX_LD_FP(x) x
    570 +#define EX_LD_FP(x,y) x
    571 #endif
    572
    573 #ifndef EX_ST
    574 -#define EX_ST(x) x
    575 +#define EX_ST(x,y) x
    576 #endif
    577 #ifndef EX_ST_FP
    578 -#define EX_ST_FP(x) x
    579 -#endif
    580 -
    581 -#ifndef EX_RETVAL
    582 -#define EX_RETVAL(x) x
    583 +#define EX_ST_FP(x,y) x
    584 #endif
    585
    586 #ifndef LOAD
    587 @@ -140,45 +137,110 @@
    588 fsrc2 %x6, %f12; \
    589 fsrc2 %x7, %f14;
    590 #define FREG_LOAD_1(base, x0) \
    591 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0))
    592 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1)
    593 #define FREG_LOAD_2(base, x0, x1) \
    594 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    595 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1));
    596 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    597 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1);
    598 #define FREG_LOAD_3(base, x0, x1, x2) \
    599 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    600 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    601 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2));
    602 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    603 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    604 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1);
    605 #define FREG_LOAD_4(base, x0, x1, x2, x3) \
    606 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    607 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    608 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    609 - EX_LD_FP(LOAD(ldd, base + 0x18, %x3));
    610 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    611 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    612 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    613 + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1);
    614 #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
    615 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    616 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    617 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    618 - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    619 - EX_LD_FP(LOAD(ldd, base + 0x20, %x4));
    620 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    621 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    622 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    623 + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    624 + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1);
    625 #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
    626 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    627 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    628 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    629 - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    630 - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
    631 - EX_LD_FP(LOAD(ldd, base + 0x28, %x5));
    632 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    633 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    634 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    635 + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    636 + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
    637 + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1);
    638 #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
    639 - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
    640 - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
    641 - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
    642 - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
    643 - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
    644 - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \
    645 - EX_LD_FP(LOAD(ldd, base + 0x30, %x6));
    646 + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
    647 + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
    648 + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
    649 + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
    650 + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
    651 + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \
    652 + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1);
    653
    654 .register %g2,#scratch
    655 .register %g3,#scratch
    656
    657 .text
    658 +#ifndef EX_RETVAL
    659 +#define EX_RETVAL(x) x
    660 +__restore_fp:
    661 + VISExitHalf
    662 +__restore_asi:
    663 + retl
    664 + wr %g0, ASI_AIUS, %asi
    665 +ENTRY(NG2_retl_o2)
    666 + ba,pt %xcc, __restore_asi
    667 + mov %o2, %o0
    668 +ENDPROC(NG2_retl_o2)
    669 +ENTRY(NG2_retl_o2_plus_1)
    670 + ba,pt %xcc, __restore_asi
    671 + add %o2, 1, %o0
    672 +ENDPROC(NG2_retl_o2_plus_1)
    673 +ENTRY(NG2_retl_o2_plus_4)
    674 + ba,pt %xcc, __restore_asi
    675 + add %o2, 4, %o0
    676 +ENDPROC(NG2_retl_o2_plus_4)
    677 +ENTRY(NG2_retl_o2_plus_8)
    678 + ba,pt %xcc, __restore_asi
    679 + add %o2, 8, %o0
    680 +ENDPROC(NG2_retl_o2_plus_8)
    681 +ENTRY(NG2_retl_o2_plus_o4_plus_1)
    682 + add %o4, 1, %o4
    683 + ba,pt %xcc, __restore_asi
    684 + add %o2, %o4, %o0
    685 +ENDPROC(NG2_retl_o2_plus_o4_plus_1)
    686 +ENTRY(NG2_retl_o2_plus_o4_plus_8)
    687 + add %o4, 8, %o4
    688 + ba,pt %xcc, __restore_asi
    689 + add %o2, %o4, %o0
    690 +ENDPROC(NG2_retl_o2_plus_o4_plus_8)
    691 +ENTRY(NG2_retl_o2_plus_o4_plus_16)
    692 + add %o4, 16, %o4
    693 + ba,pt %xcc, __restore_asi
    694 + add %o2, %o4, %o0
    695 +ENDPROC(NG2_retl_o2_plus_o4_plus_16)
    696 +ENTRY(NG2_retl_o2_plus_g1_fp)
    697 + ba,pt %xcc, __restore_fp
    698 + add %o2, %g1, %o0
    699 +ENDPROC(NG2_retl_o2_plus_g1_fp)
    700 +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp)
    701 + add %g1, 64, %g1
    702 + ba,pt %xcc, __restore_fp
    703 + add %o2, %g1, %o0
    704 +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp)
    705 +ENTRY(NG2_retl_o2_plus_g1_plus_1)
    706 + add %g1, 1, %g1
    707 + ba,pt %xcc, __restore_asi
    708 + add %o2, %g1, %o0
    709 +ENDPROC(NG2_retl_o2_plus_g1_plus_1)
    710 +ENTRY(NG2_retl_o2_and_7_plus_o4)
    711 + and %o2, 7, %o2
    712 + ba,pt %xcc, __restore_asi
    713 + add %o2, %o4, %o0
    714 +ENDPROC(NG2_retl_o2_and_7_plus_o4)
    715 +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8)
    716 + and %o2, 7, %o2
    717 + add %o4, 8, %o4
    718 + ba,pt %xcc, __restore_asi
    719 + add %o2, %o4, %o0
    720 +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8)
    721 +#endif
    722 +
    723 .align 64
    724
    725 .globl FUNC_NAME
    726 @@ -230,8 +292,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    727 sub %g0, %o4, %o4 ! bytes to align dst
    728 sub %o2, %o4, %o2
    729 1: subcc %o4, 1, %o4
    730 - EX_LD(LOAD(ldub, %o1, %g1))
    731 - EX_ST(STORE(stb, %g1, %o0))
    732 + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1)
    733 + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1)
    734 add %o1, 1, %o1
    735 bne,pt %XCC, 1b
    736 add %o0, 1, %o0
    737 @@ -281,11 +343,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    738 nop
    739 /* fall through for 0 < low bits < 8 */
    740 110: sub %o4, 64, %g2
    741 - EX_LD_FP(LOAD_BLK(%g2, %f0))
    742 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    743 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    744 + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1)
    745 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    746 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    747 FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
    748 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    749 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    750 FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
    751 subcc %g1, 64, %g1
    752 add %o4, 64, %o4
    753 @@ -296,10 +358,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    754
    755 120: sub %o4, 56, %g2
    756 FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
    757 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    758 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    759 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    760 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    761 FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
    762 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    763 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    764 FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
    765 subcc %g1, 64, %g1
    766 add %o4, 64, %o4
    767 @@ -310,10 +372,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    768
    769 130: sub %o4, 48, %g2
    770 FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
    771 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    772 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    773 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    774 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    775 FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
    776 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    777 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    778 FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
    779 subcc %g1, 64, %g1
    780 add %o4, 64, %o4
    781 @@ -324,10 +386,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    782
    783 140: sub %o4, 40, %g2
    784 FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
    785 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    786 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    787 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    788 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    789 FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
    790 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    791 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    792 FREG_MOVE_5(f22, f24, f26, f28, f30)
    793 subcc %g1, 64, %g1
    794 add %o4, 64, %o4
    795 @@ -338,10 +400,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    796
    797 150: sub %o4, 32, %g2
    798 FREG_LOAD_4(%g2, f0, f2, f4, f6)
    799 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    800 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    801 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    802 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    803 FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
    804 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    805 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    806 FREG_MOVE_4(f24, f26, f28, f30)
    807 subcc %g1, 64, %g1
    808 add %o4, 64, %o4
    809 @@ -352,10 +414,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    810
    811 160: sub %o4, 24, %g2
    812 FREG_LOAD_3(%g2, f0, f2, f4)
    813 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    814 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    815 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    816 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    817 FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
    818 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    819 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    820 FREG_MOVE_3(f26, f28, f30)
    821 subcc %g1, 64, %g1
    822 add %o4, 64, %o4
    823 @@ -366,10 +428,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    824
    825 170: sub %o4, 16, %g2
    826 FREG_LOAD_2(%g2, f0, f2)
    827 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    828 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    829 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    830 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    831 FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
    832 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    833 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    834 FREG_MOVE_2(f28, f30)
    835 subcc %g1, 64, %g1
    836 add %o4, 64, %o4
    837 @@ -380,10 +442,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    838
    839 180: sub %o4, 8, %g2
    840 FREG_LOAD_1(%g2, f0)
    841 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    842 - EX_LD_FP(LOAD_BLK(%o4, %f16))
    843 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    844 + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
    845 FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
    846 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    847 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
    848 FREG_MOVE_1(f30)
    849 subcc %g1, 64, %g1
    850 add %o4, 64, %o4
    851 @@ -393,10 +455,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    852 nop
    853
    854 190:
    855 -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
    856 +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
    857 subcc %g1, 64, %g1
    858 - EX_LD_FP(LOAD_BLK(%o4, %f0))
    859 - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
    860 + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64)
    861 + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64)
    862 add %o4, 64, %o4
    863 bne,pt %xcc, 1b
    864 LOAD(prefetch, %o4 + 64, #one_read)
    865 @@ -423,28 +485,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    866 andn %o2, 0xf, %o4
    867 and %o2, 0xf, %o2
    868 1: subcc %o4, 0x10, %o4
    869 - EX_LD(LOAD(ldx, %o1, %o5))
    870 + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16)
    871 add %o1, 0x08, %o1
    872 - EX_LD(LOAD(ldx, %o1, %g1))
    873 + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16)
    874 sub %o1, 0x08, %o1
    875 - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
    876 + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16)
    877 add %o1, 0x8, %o1
    878 - EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE))
    879 + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8)
    880 bgu,pt %XCC, 1b
    881 add %o1, 0x8, %o1
    882 73: andcc %o2, 0x8, %g0
    883 be,pt %XCC, 1f
    884 nop
    885 sub %o2, 0x8, %o2
    886 - EX_LD(LOAD(ldx, %o1, %o5))
    887 - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
    888 + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8)
    889 + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8)
    890 add %o1, 0x8, %o1
    891 1: andcc %o2, 0x4, %g0
    892 be,pt %XCC, 1f
    893 nop
    894 sub %o2, 0x4, %o2
    895 - EX_LD(LOAD(lduw, %o1, %o5))
    896 - EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE))
    897 + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4)
    898 + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
    899 add %o1, 0x4, %o1
    900 1: cmp %o2, 0
    901 be,pt %XCC, 85f
    902 @@ -460,8 +522,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    903 sub %o2, %g1, %o2
    904
    905 1: subcc %g1, 1, %g1
    906 - EX_LD(LOAD(ldub, %o1, %o5))
    907 - EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE))
    908 + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1)
    909 + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1)
    910 bgu,pt %icc, 1b
    911 add %o1, 1, %o1
    912
    913 @@ -477,16 +539,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    914
    915 8: mov 64, GLOBAL_SPARE
    916 andn %o1, 0x7, %o1
    917 - EX_LD(LOAD(ldx, %o1, %g2))
    918 + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2)
    919 sub GLOBAL_SPARE, %g1, GLOBAL_SPARE
    920 andn %o2, 0x7, %o4
    921 sllx %g2, %g1, %g2
    922 1: add %o1, 0x8, %o1
    923 - EX_LD(LOAD(ldx, %o1, %g3))
    924 + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4)
    925 subcc %o4, 0x8, %o4
    926 srlx %g3, GLOBAL_SPARE, %o5
    927 or %o5, %g2, %o5
    928 - EX_ST(STORE(stx, %o5, %o0))
    929 + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8)
    930 add %o0, 0x8, %o0
    931 bgu,pt %icc, 1b
    932 sllx %g3, %g1, %g2
    933 @@ -506,8 +568,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    934
    935 1:
    936 subcc %o2, 4, %o2
    937 - EX_LD(LOAD(lduw, %o1, %g1))
    938 - EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE))
    939 + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4)
    940 + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
    941 bgu,pt %XCC, 1b
    942 add %o1, 4, %o1
    943
    944 @@ -517,8 +579,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    945 .align 32
    946 90:
    947 subcc %o2, 1, %o2
    948 - EX_LD(LOAD(ldub, %o1, %g1))
    949 - EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE))
    950 + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1)
    951 + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1)
    952 bgu,pt %XCC, 90b
    953 add %o1, 1, %o1
    954 retl
    955 diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S
    956 index 2e8ee7ad07a9..16a286c1a528 100644
    957 --- a/arch/sparc/lib/NG4copy_from_user.S
    958 +++ b/arch/sparc/lib/NG4copy_from_user.S
    959 @@ -3,19 +3,19 @@
    960 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
    961 */
    962
    963 -#define EX_LD(x) \
    964 +#define EX_LD(x, y) \
    965 98: x; \
    966 .section __ex_table,"a";\
    967 .align 4; \
    968 - .word 98b, __retl_one_asi;\
    969 + .word 98b, y; \
    970 .text; \
    971 .align 4;
    972
    973 -#define EX_LD_FP(x) \
    974 +#define EX_LD_FP(x,y) \
    975 98: x; \
    976 .section __ex_table,"a";\
    977 .align 4; \
    978 - .word 98b, __retl_one_asi_fp;\
    979 + .word 98b, y##_fp; \
    980 .text; \
    981 .align 4;
    982
    983 diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S
    984 index be0bf4590df8..6b0276ffc858 100644
    985 --- a/arch/sparc/lib/NG4copy_to_user.S
    986 +++ b/arch/sparc/lib/NG4copy_to_user.S
    987 @@ -3,19 +3,19 @@
    988 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
    989 */
    990
    991 -#define EX_ST(x) \
    992 +#define EX_ST(x,y) \
    993 98: x; \
    994 .section __ex_table,"a";\
    995 .align 4; \
    996 - .word 98b, __retl_one_asi;\
    997 + .word 98b, y; \
    998 .text; \
    999 .align 4;
    1000
    1001 -#define EX_ST_FP(x) \
    1002 +#define EX_ST_FP(x,y) \
    1003 98: x; \
    1004 .section __ex_table,"a";\
    1005 .align 4; \
    1006 - .word 98b, __retl_one_asi_fp;\
    1007 + .word 98b, y##_fp; \
    1008 .text; \
    1009 .align 4;
    1010
    1011 diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S
    1012 index 8e13ee1f4454..75bb93b1437f 100644
    1013 --- a/arch/sparc/lib/NG4memcpy.S
    1014 +++ b/arch/sparc/lib/NG4memcpy.S
    1015 @@ -4,6 +4,7 @@
    1016 */
    1017
    1018 #ifdef __KERNEL__
    1019 +#include <linux/linkage.h>
    1020 #include <asm/visasm.h>
    1021 #include <asm/asi.h>
    1022 #define GLOBAL_SPARE %g7
    1023 @@ -46,22 +47,19 @@
    1024 #endif
    1025
    1026 #ifndef EX_LD
    1027 -#define EX_LD(x) x
    1028 +#define EX_LD(x,y) x
    1029 #endif
    1030 #ifndef EX_LD_FP
    1031 -#define EX_LD_FP(x) x
    1032 +#define EX_LD_FP(x,y) x
    1033 #endif
    1034
    1035 #ifndef EX_ST
    1036 -#define EX_ST(x) x
    1037 +#define EX_ST(x,y) x
    1038 #endif
    1039 #ifndef EX_ST_FP
    1040 -#define EX_ST_FP(x) x
    1041 +#define EX_ST_FP(x,y) x
    1042 #endif
    1043
    1044 -#ifndef EX_RETVAL
    1045 -#define EX_RETVAL(x) x
    1046 -#endif
    1047
    1048 #ifndef LOAD
    1049 #define LOAD(type,addr,dest) type [addr], dest
    1050 @@ -94,6 +92,158 @@
    1051 .register %g3,#scratch
    1052
    1053 .text
    1054 +#ifndef EX_RETVAL
    1055 +#define EX_RETVAL(x) x
    1056 +__restore_asi_fp:
    1057 + VISExitHalf
    1058 +__restore_asi:
    1059 + retl
    1060 + wr %g0, ASI_AIUS, %asi
    1061 +
    1062 +ENTRY(NG4_retl_o2)
    1063 + ba,pt %xcc, __restore_asi
    1064 + mov %o2, %o0
    1065 +ENDPROC(NG4_retl_o2)
    1066 +ENTRY(NG4_retl_o2_plus_1)
    1067 + ba,pt %xcc, __restore_asi
    1068 + add %o2, 1, %o0
    1069 +ENDPROC(NG4_retl_o2_plus_1)
    1070 +ENTRY(NG4_retl_o2_plus_4)
    1071 + ba,pt %xcc, __restore_asi
    1072 + add %o2, 4, %o0
    1073 +ENDPROC(NG4_retl_o2_plus_4)
    1074 +ENTRY(NG4_retl_o2_plus_o5)
    1075 + ba,pt %xcc, __restore_asi
    1076 + add %o2, %o5, %o0
    1077 +ENDPROC(NG4_retl_o2_plus_o5)
    1078 +ENTRY(NG4_retl_o2_plus_o5_plus_4)
    1079 + add %o5, 4, %o5
    1080 + ba,pt %xcc, __restore_asi
    1081 + add %o2, %o5, %o0
    1082 +ENDPROC(NG4_retl_o2_plus_o5_plus_4)
    1083 +ENTRY(NG4_retl_o2_plus_o5_plus_8)
    1084 + add %o5, 8, %o5
    1085 + ba,pt %xcc, __restore_asi
    1086 + add %o2, %o5, %o0
    1087 +ENDPROC(NG4_retl_o2_plus_o5_plus_8)
    1088 +ENTRY(NG4_retl_o2_plus_o5_plus_16)
    1089 + add %o5, 16, %o5
    1090 + ba,pt %xcc, __restore_asi
    1091 + add %o2, %o5, %o0
    1092 +ENDPROC(NG4_retl_o2_plus_o5_plus_16)
    1093 +ENTRY(NG4_retl_o2_plus_o5_plus_24)
    1094 + add %o5, 24, %o5
    1095 + ba,pt %xcc, __restore_asi
    1096 + add %o2, %o5, %o0
    1097 +ENDPROC(NG4_retl_o2_plus_o5_plus_24)
    1098 +ENTRY(NG4_retl_o2_plus_o5_plus_32)
    1099 + add %o5, 32, %o5
    1100 + ba,pt %xcc, __restore_asi
    1101 + add %o2, %o5, %o0
    1102 +ENDPROC(NG4_retl_o2_plus_o5_plus_32)
    1103 +ENTRY(NG4_retl_o2_plus_g1)
    1104 + ba,pt %xcc, __restore_asi
    1105 + add %o2, %g1, %o0
    1106 +ENDPROC(NG4_retl_o2_plus_g1)
    1107 +ENTRY(NG4_retl_o2_plus_g1_plus_1)
    1108 + add %g1, 1, %g1
    1109 + ba,pt %xcc, __restore_asi
    1110 + add %o2, %g1, %o0
    1111 +ENDPROC(NG4_retl_o2_plus_g1_plus_1)
    1112 +ENTRY(NG4_retl_o2_plus_g1_plus_8)
    1113 + add %g1, 8, %g1
    1114 + ba,pt %xcc, __restore_asi
    1115 + add %o2, %g1, %o0
    1116 +ENDPROC(NG4_retl_o2_plus_g1_plus_8)
    1117 +ENTRY(NG4_retl_o2_plus_o4)
    1118 + ba,pt %xcc, __restore_asi
    1119 + add %o2, %o4, %o0
    1120 +ENDPROC(NG4_retl_o2_plus_o4)
    1121 +ENTRY(NG4_retl_o2_plus_o4_plus_8)
    1122 + add %o4, 8, %o4
    1123 + ba,pt %xcc, __restore_asi
    1124 + add %o2, %o4, %o0
    1125 +ENDPROC(NG4_retl_o2_plus_o4_plus_8)
    1126 +ENTRY(NG4_retl_o2_plus_o4_plus_16)
    1127 + add %o4, 16, %o4
    1128 + ba,pt %xcc, __restore_asi
    1129 + add %o2, %o4, %o0
    1130 +ENDPROC(NG4_retl_o2_plus_o4_plus_16)
    1131 +ENTRY(NG4_retl_o2_plus_o4_plus_24)
    1132 + add %o4, 24, %o4
    1133 + ba,pt %xcc, __restore_asi
    1134 + add %o2, %o4, %o0
    1135 +ENDPROC(NG4_retl_o2_plus_o4_plus_24)
    1136 +ENTRY(NG4_retl_o2_plus_o4_plus_32)
    1137 + add %o4, 32, %o4
    1138 + ba,pt %xcc, __restore_asi
    1139 + add %o2, %o4, %o0
    1140 +ENDPROC(NG4_retl_o2_plus_o4_plus_32)
    1141 +ENTRY(NG4_retl_o2_plus_o4_plus_40)
    1142 + add %o4, 40, %o4
    1143 + ba,pt %xcc, __restore_asi
    1144 + add %o2, %o4, %o0
    1145 +ENDPROC(NG4_retl_o2_plus_o4_plus_40)
    1146 +ENTRY(NG4_retl_o2_plus_o4_plus_48)
    1147 + add %o4, 48, %o4
    1148 + ba,pt %xcc, __restore_asi
    1149 + add %o2, %o4, %o0
    1150 +ENDPROC(NG4_retl_o2_plus_o4_plus_48)
    1151 +ENTRY(NG4_retl_o2_plus_o4_plus_56)
    1152 + add %o4, 56, %o4
    1153 + ba,pt %xcc, __restore_asi
    1154 + add %o2, %o4, %o0
    1155 +ENDPROC(NG4_retl_o2_plus_o4_plus_56)
    1156 +ENTRY(NG4_retl_o2_plus_o4_plus_64)
    1157 + add %o4, 64, %o4
    1158 + ba,pt %xcc, __restore_asi
    1159 + add %o2, %o4, %o0
    1160 +ENDPROC(NG4_retl_o2_plus_o4_plus_64)
    1161 +ENTRY(NG4_retl_o2_plus_o4_fp)
    1162 + ba,pt %xcc, __restore_asi_fp
    1163 + add %o2, %o4, %o0
    1164 +ENDPROC(NG4_retl_o2_plus_o4_fp)
    1165 +ENTRY(NG4_retl_o2_plus_o4_plus_8_fp)
    1166 + add %o4, 8, %o4
    1167 + ba,pt %xcc, __restore_asi_fp
    1168 + add %o2, %o4, %o0
    1169 +ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp)
    1170 +ENTRY(NG4_retl_o2_plus_o4_plus_16_fp)
    1171 + add %o4, 16, %o4
    1172 + ba,pt %xcc, __restore_asi_fp
    1173 + add %o2, %o4, %o0
    1174 +ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp)
    1175 +ENTRY(NG4_retl_o2_plus_o4_plus_24_fp)
    1176 + add %o4, 24, %o4
    1177 + ba,pt %xcc, __restore_asi_fp
    1178 + add %o2, %o4, %o0
    1179 +ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp)
    1180 +ENTRY(NG4_retl_o2_plus_o4_plus_32_fp)
    1181 + add %o4, 32, %o4
    1182 + ba,pt %xcc, __restore_asi_fp
    1183 + add %o2, %o4, %o0
    1184 +ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp)
    1185 +ENTRY(NG4_retl_o2_plus_o4_plus_40_fp)
    1186 + add %o4, 40, %o4
    1187 + ba,pt %xcc, __restore_asi_fp
    1188 + add %o2, %o4, %o0
    1189 +ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp)
    1190 +ENTRY(NG4_retl_o2_plus_o4_plus_48_fp)
    1191 + add %o4, 48, %o4
    1192 + ba,pt %xcc, __restore_asi_fp
    1193 + add %o2, %o4, %o0
    1194 +ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp)
    1195 +ENTRY(NG4_retl_o2_plus_o4_plus_56_fp)
    1196 + add %o4, 56, %o4
    1197 + ba,pt %xcc, __restore_asi_fp
    1198 + add %o2, %o4, %o0
    1199 +ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp)
    1200 +ENTRY(NG4_retl_o2_plus_o4_plus_64_fp)
    1201 + add %o4, 64, %o4
    1202 + ba,pt %xcc, __restore_asi_fp
    1203 + add %o2, %o4, %o0
    1204 +ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp)
    1205 +#endif
    1206 .align 64
    1207
    1208 .globl FUNC_NAME
    1209 @@ -124,12 +274,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1210 brz,pt %g1, 51f
    1211 sub %o2, %g1, %o2
    1212
    1213 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
    1214 +
    1215 +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1216 add %o1, 1, %o1
    1217 subcc %g1, 1, %g1
    1218 add %o0, 1, %o0
    1219 bne,pt %icc, 1b
    1220 - EX_ST(STORE(stb, %g2, %o0 - 0x01))
    1221 + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
    1222
    1223 51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong)
    1224 LOAD(prefetch, %o1 + 0x080, #n_reads_strong)
    1225 @@ -154,43 +305,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1226 brz,pt %g1, .Llarge_aligned
    1227 sub %o2, %g1, %o2
    1228
    1229 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2))
    1230 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1231 add %o1, 8, %o1
    1232 subcc %g1, 8, %g1
    1233 add %o0, 8, %o0
    1234 bne,pt %icc, 1b
    1235 - EX_ST(STORE(stx, %g2, %o0 - 0x08))
    1236 + EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8)
    1237
    1238 .Llarge_aligned:
    1239 /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */
    1240 andn %o2, 0x3f, %o4
    1241 sub %o2, %o4, %o2
    1242
    1243 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1244 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4)
    1245 add %o1, 0x40, %o1
    1246 - EX_LD(LOAD(ldx, %o1 - 0x38, %g2))
    1247 + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4)
    1248 subcc %o4, 0x40, %o4
    1249 - EX_LD(LOAD(ldx, %o1 - 0x30, %g3))
    1250 - EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE))
    1251 - EX_LD(LOAD(ldx, %o1 - 0x20, %o5))
    1252 - EX_ST(STORE_INIT(%g1, %o0))
    1253 + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64)
    1254 + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64)
    1255 + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64)
    1256 + EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64)
    1257 add %o0, 0x08, %o0
    1258 - EX_ST(STORE_INIT(%g2, %o0))
    1259 + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56)
    1260 add %o0, 0x08, %o0
    1261 - EX_LD(LOAD(ldx, %o1 - 0x18, %g2))
    1262 - EX_ST(STORE_INIT(%g3, %o0))
    1263 + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48)
    1264 + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48)
    1265 add %o0, 0x08, %o0
    1266 - EX_LD(LOAD(ldx, %o1 - 0x10, %g3))
    1267 - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
    1268 + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40)
    1269 + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40)
    1270 add %o0, 0x08, %o0
    1271 - EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE))
    1272 - EX_ST(STORE_INIT(%o5, %o0))
    1273 + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32)
    1274 + EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32)
    1275 add %o0, 0x08, %o0
    1276 - EX_ST(STORE_INIT(%g2, %o0))
    1277 + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24)
    1278 add %o0, 0x08, %o0
    1279 - EX_ST(STORE_INIT(%g3, %o0))
    1280 + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16)
    1281 add %o0, 0x08, %o0
    1282 - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
    1283 + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8)
    1284 add %o0, 0x08, %o0
    1285 bne,pt %icc, 1b
    1286 LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
    1287 @@ -216,17 +367,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1288 sub %o2, %o4, %o2
    1289 alignaddr %o1, %g0, %g1
    1290 add %o1, %o4, %o1
    1291 - EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0))
    1292 -1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2))
    1293 + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4)
    1294 +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4)
    1295 subcc %o4, 0x40, %o4
    1296 - EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4))
    1297 - EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6))
    1298 - EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8))
    1299 - EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10))
    1300 - EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12))
    1301 - EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14))
    1302 + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64)
    1303 + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64)
    1304 + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64)
    1305 + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64)
    1306 + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64)
    1307 + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64)
    1308 faligndata %f0, %f2, %f16
    1309 - EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0))
    1310 + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64)
    1311 faligndata %f2, %f4, %f18
    1312 add %g1, 0x40, %g1
    1313 faligndata %f4, %f6, %f20
    1314 @@ -235,14 +386,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1315 faligndata %f10, %f12, %f26
    1316 faligndata %f12, %f14, %f28
    1317 faligndata %f14, %f0, %f30
    1318 - EX_ST_FP(STORE(std, %f16, %o0 + 0x00))
    1319 - EX_ST_FP(STORE(std, %f18, %o0 + 0x08))
    1320 - EX_ST_FP(STORE(std, %f20, %o0 + 0x10))
    1321 - EX_ST_FP(STORE(std, %f22, %o0 + 0x18))
    1322 - EX_ST_FP(STORE(std, %f24, %o0 + 0x20))
    1323 - EX_ST_FP(STORE(std, %f26, %o0 + 0x28))
    1324 - EX_ST_FP(STORE(std, %f28, %o0 + 0x30))
    1325 - EX_ST_FP(STORE(std, %f30, %o0 + 0x38))
    1326 + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64)
    1327 + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56)
    1328 + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48)
    1329 + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40)
    1330 + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32)
    1331 + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24)
    1332 + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16)
    1333 + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8)
    1334 add %o0, 0x40, %o0
    1335 bne,pt %icc, 1b
    1336 LOAD(prefetch, %g1 + 0x200, #n_reads_strong)
    1337 @@ -270,37 +421,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1338 andncc %o2, 0x20 - 1, %o5
    1339 be,pn %icc, 2f
    1340 sub %o2, %o5, %o2
    1341 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1342 - EX_LD(LOAD(ldx, %o1 + 0x08, %g2))
    1343 - EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE))
    1344 - EX_LD(LOAD(ldx, %o1 + 0x18, %o4))
    1345 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1346 + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5)
    1347 + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5)
    1348 + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5)
    1349 add %o1, 0x20, %o1
    1350 subcc %o5, 0x20, %o5
    1351 - EX_ST(STORE(stx, %g1, %o0 + 0x00))
    1352 - EX_ST(STORE(stx, %g2, %o0 + 0x08))
    1353 - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10))
    1354 - EX_ST(STORE(stx, %o4, %o0 + 0x18))
    1355 + EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32)
    1356 + EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24)
    1357 + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24)
    1358 + EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8)
    1359 bne,pt %icc, 1b
    1360 add %o0, 0x20, %o0
    1361 2: andcc %o2, 0x18, %o5
    1362 be,pt %icc, 3f
    1363 sub %o2, %o5, %o2
    1364 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
    1365 +
    1366 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1367 add %o1, 0x08, %o1
    1368 add %o0, 0x08, %o0
    1369 subcc %o5, 0x08, %o5
    1370 bne,pt %icc, 1b
    1371 - EX_ST(STORE(stx, %g1, %o0 - 0x08))
    1372 + EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8)
    1373 3: brz,pt %o2, .Lexit
    1374 cmp %o2, 0x04
    1375 bl,pn %icc, .Ltiny
    1376 nop
    1377 - EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
    1378 + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2)
    1379 add %o1, 0x04, %o1
    1380 add %o0, 0x04, %o0
    1381 subcc %o2, 0x04, %o2
    1382 bne,pn %icc, .Ltiny
    1383 - EX_ST(STORE(stw, %g1, %o0 - 0x04))
    1384 + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4)
    1385 ba,a,pt %icc, .Lexit
    1386 .Lmedium_unaligned:
    1387 /* First get dest 8 byte aligned. */
    1388 @@ -309,12 +461,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1389 brz,pt %g1, 2f
    1390 sub %o2, %g1, %o2
    1391
    1392 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
    1393 +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
    1394 add %o1, 1, %o1
    1395 subcc %g1, 1, %g1
    1396 add %o0, 1, %o0
    1397 bne,pt %icc, 1b
    1398 - EX_ST(STORE(stb, %g2, %o0 - 0x01))
    1399 + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
    1400 2:
    1401 and %o1, 0x7, %g1
    1402 brz,pn %g1, .Lmedium_noprefetch
    1403 @@ -322,16 +474,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1404 mov 64, %g2
    1405 sub %g2, %g1, %g2
    1406 andn %o1, 0x7, %o1
    1407 - EX_LD(LOAD(ldx, %o1 + 0x00, %o4))
    1408 + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2)
    1409 sllx %o4, %g1, %o4
    1410 andn %o2, 0x08 - 1, %o5
    1411 sub %o2, %o5, %o2
    1412 -1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3))
    1413 +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5)
    1414 add %o1, 0x08, %o1
    1415 subcc %o5, 0x08, %o5
    1416 srlx %g3, %g2, GLOBAL_SPARE
    1417 or GLOBAL_SPARE, %o4, GLOBAL_SPARE
    1418 - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00))
    1419 + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8)
    1420 add %o0, 0x08, %o0
    1421 bne,pt %icc, 1b
    1422 sllx %g3, %g1, %o4
    1423 @@ -342,17 +494,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1424 ba,pt %icc, .Lsmall_unaligned
    1425
    1426 .Ltiny:
    1427 - EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
    1428 + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
    1429 subcc %o2, 1, %o2
    1430 be,pn %icc, .Lexit
    1431 - EX_ST(STORE(stb, %g1, %o0 + 0x00))
    1432 - EX_LD(LOAD(ldub, %o1 + 0x01, %g1))
    1433 + EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1)
    1434 + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2)
    1435 subcc %o2, 1, %o2
    1436 be,pn %icc, .Lexit
    1437 - EX_ST(STORE(stb, %g1, %o0 + 0x01))
    1438 - EX_LD(LOAD(ldub, %o1 + 0x02, %g1))
    1439 + EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1)
    1440 + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2)
    1441 ba,pt %icc, .Lexit
    1442 - EX_ST(STORE(stb, %g1, %o0 + 0x02))
    1443 + EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2)
    1444
    1445 .Lsmall:
    1446 andcc %g2, 0x3, %g0
    1447 @@ -360,22 +512,22 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    1448 andn %o2, 0x4 - 1, %o5
    1449 sub %o2, %o5, %o2
    1450 1:
    1451 - EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
    1452 + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
    1453 add %o1, 0x04, %o1
    1454 subcc %o5, 0x04, %o5
    1455 add %o0, 0x04, %o0
    1456 bne,pt %icc, 1b
    1457 - EX_ST(STORE(stw, %g1, %o0 - 0x04))
    1458 + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4)
    1459 brz,pt %o2, .Lexit
    1460 nop
    1461 ba,a,pt %icc, .Ltiny
    1462
    1463 .Lsmall_unaligned:
    1464 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
    1465 +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
    1466 add %o1, 1, %o1
    1467 add %o0, 1, %o0
    1468 subcc %o2, 1, %o2
    1469 bne,pt %icc, 1b
    1470 - EX_ST(STORE(stb, %g1, %o0 - 0x01))
    1471 + EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1)
    1472 ba,a,pt %icc, .Lexit
    1473 .size FUNC_NAME, .-FUNC_NAME
    1474 diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
    1475 index 5d1e4d1ac21e..9cd42fcbc781 100644
    1476 --- a/arch/sparc/lib/NGcopy_from_user.S
    1477 +++ b/arch/sparc/lib/NGcopy_from_user.S
    1478 @@ -3,11 +3,11 @@
    1479 * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
    1480 */
    1481
    1482 -#define EX_LD(x) \
    1483 +#define EX_LD(x,y) \
    1484 98: x; \
    1485 .section __ex_table,"a";\
    1486 .align 4; \
    1487 - .word 98b, __ret_one_asi;\
    1488 + .word 98b, y; \
    1489 .text; \
    1490 .align 4;
    1491
    1492 diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
    1493 index ff630dcb273c..5c358afd464e 100644
    1494 --- a/arch/sparc/lib/NGcopy_to_user.S
    1495 +++ b/arch/sparc/lib/NGcopy_to_user.S
    1496 @@ -3,11 +3,11 @@
    1497 * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
    1498 */
    1499
    1500 -#define EX_ST(x) \
    1501 +#define EX_ST(x,y) \
    1502 98: x; \
    1503 .section __ex_table,"a";\
    1504 .align 4; \
    1505 - .word 98b, __ret_one_asi;\
    1506 + .word 98b, y; \
    1507 .text; \
    1508 .align 4;
    1509
    1510 diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S
    1511 index 96a14caf6966..d88c4ed50a00 100644
    1512 --- a/arch/sparc/lib/NGmemcpy.S
    1513 +++ b/arch/sparc/lib/NGmemcpy.S
    1514 @@ -4,6 +4,7 @@
    1515 */
    1516
    1517 #ifdef __KERNEL__
    1518 +#include <linux/linkage.h>
    1519 #include <asm/asi.h>
    1520 #include <asm/thread_info.h>
    1521 #define GLOBAL_SPARE %g7
    1522 @@ -27,15 +28,11 @@
    1523 #endif
    1524
    1525 #ifndef EX_LD
    1526 -#define EX_LD(x) x
    1527 +#define EX_LD(x,y) x
    1528 #endif
    1529
    1530 #ifndef EX_ST
    1531 -#define EX_ST(x) x
    1532 -#endif
    1533 -
    1534 -#ifndef EX_RETVAL
    1535 -#define EX_RETVAL(x) x
    1536 +#define EX_ST(x,y) x
    1537 #endif
    1538
    1539 #ifndef LOAD
    1540 @@ -79,6 +76,92 @@
    1541 .register %g3,#scratch
    1542
    1543 .text
    1544 +#ifndef EX_RETVAL
    1545 +#define EX_RETVAL(x) x
    1546 +__restore_asi:
    1547 + ret
    1548 + wr %g0, ASI_AIUS, %asi
    1549 + restore
    1550 +ENTRY(NG_ret_i2_plus_i4_plus_1)
    1551 + ba,pt %xcc, __restore_asi
    1552 + add %i2, %i5, %i0
    1553 +ENDPROC(NG_ret_i2_plus_i4_plus_1)
    1554 +ENTRY(NG_ret_i2_plus_g1)
    1555 + ba,pt %xcc, __restore_asi
    1556 + add %i2, %g1, %i0
    1557 +ENDPROC(NG_ret_i2_plus_g1)
    1558 +ENTRY(NG_ret_i2_plus_g1_minus_8)
    1559 + sub %g1, 8, %g1
    1560 + ba,pt %xcc, __restore_asi
    1561 + add %i2, %g1, %i0
    1562 +ENDPROC(NG_ret_i2_plus_g1_minus_8)
    1563 +ENTRY(NG_ret_i2_plus_g1_minus_16)
    1564 + sub %g1, 16, %g1
    1565 + ba,pt %xcc, __restore_asi
    1566 + add %i2, %g1, %i0
    1567 +ENDPROC(NG_ret_i2_plus_g1_minus_16)
    1568 +ENTRY(NG_ret_i2_plus_g1_minus_24)
    1569 + sub %g1, 24, %g1
    1570 + ba,pt %xcc, __restore_asi
    1571 + add %i2, %g1, %i0
    1572 +ENDPROC(NG_ret_i2_plus_g1_minus_24)
    1573 +ENTRY(NG_ret_i2_plus_g1_minus_32)
    1574 + sub %g1, 32, %g1
    1575 + ba,pt %xcc, __restore_asi
    1576 + add %i2, %g1, %i0
    1577 +ENDPROC(NG_ret_i2_plus_g1_minus_32)
    1578 +ENTRY(NG_ret_i2_plus_g1_minus_40)
    1579 + sub %g1, 40, %g1
    1580 + ba,pt %xcc, __restore_asi
    1581 + add %i2, %g1, %i0
    1582 +ENDPROC(NG_ret_i2_plus_g1_minus_40)
    1583 +ENTRY(NG_ret_i2_plus_g1_minus_48)
    1584 + sub %g1, 48, %g1
    1585 + ba,pt %xcc, __restore_asi
    1586 + add %i2, %g1, %i0
    1587 +ENDPROC(NG_ret_i2_plus_g1_minus_48)
    1588 +ENTRY(NG_ret_i2_plus_g1_minus_56)
    1589 + sub %g1, 56, %g1
    1590 + ba,pt %xcc, __restore_asi
    1591 + add %i2, %g1, %i0
    1592 +ENDPROC(NG_ret_i2_plus_g1_minus_56)
    1593 +ENTRY(NG_ret_i2_plus_i4)
    1594 + ba,pt %xcc, __restore_asi
    1595 + add %i2, %i4, %i0
    1596 +ENDPROC(NG_ret_i2_plus_i4)
    1597 +ENTRY(NG_ret_i2_plus_i4_minus_8)
    1598 + sub %i4, 8, %i4
    1599 + ba,pt %xcc, __restore_asi
    1600 + add %i2, %i4, %i0
    1601 +ENDPROC(NG_ret_i2_plus_i4_minus_8)
    1602 +ENTRY(NG_ret_i2_plus_8)
    1603 + ba,pt %xcc, __restore_asi
    1604 + add %i2, 8, %i0
    1605 +ENDPROC(NG_ret_i2_plus_8)
    1606 +ENTRY(NG_ret_i2_plus_4)
    1607 + ba,pt %xcc, __restore_asi
    1608 + add %i2, 4, %i0
    1609 +ENDPROC(NG_ret_i2_plus_4)
    1610 +ENTRY(NG_ret_i2_plus_1)
    1611 + ba,pt %xcc, __restore_asi
    1612 + add %i2, 1, %i0
    1613 +ENDPROC(NG_ret_i2_plus_1)
    1614 +ENTRY(NG_ret_i2_plus_g1_plus_1)
    1615 + add %g1, 1, %g1
    1616 + ba,pt %xcc, __restore_asi
    1617 + add %i2, %g1, %i0
    1618 +ENDPROC(NG_ret_i2_plus_g1_plus_1)
    1619 +ENTRY(NG_ret_i2)
    1620 + ba,pt %xcc, __restore_asi
    1621 + mov %i2, %i0
    1622 +ENDPROC(NG_ret_i2)
    1623 +ENTRY(NG_ret_i2_and_7_plus_i4)
    1624 + and %i2, 7, %i2
    1625 + ba,pt %xcc, __restore_asi
    1626 + add %i2, %i4, %i0
    1627 +ENDPROC(NG_ret_i2_and_7_plus_i4)
    1628 +#endif
    1629 +
    1630 .align 64
    1631
    1632 .globl FUNC_NAME
    1633 @@ -126,8 +209,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1634 sub %g0, %i4, %i4 ! bytes to align dst
    1635 sub %i2, %i4, %i2
    1636 1: subcc %i4, 1, %i4
    1637 - EX_LD(LOAD(ldub, %i1, %g1))
    1638 - EX_ST(STORE(stb, %g1, %o0))
    1639 + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1)
    1640 + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1)
    1641 add %i1, 1, %i1
    1642 bne,pt %XCC, 1b
    1643 add %o0, 1, %o0
    1644 @@ -160,7 +243,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1645 and %i4, 0x7, GLOBAL_SPARE
    1646 sll GLOBAL_SPARE, 3, GLOBAL_SPARE
    1647 mov 64, %i5
    1648 - EX_LD(LOAD_TWIN(%i1, %g2, %g3))
    1649 + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1)
    1650 sub %i5, GLOBAL_SPARE, %i5
    1651 mov 16, %o4
    1652 mov 32, %o5
    1653 @@ -178,31 +261,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1654 srlx WORD3, PRE_SHIFT, TMP; \
    1655 or WORD2, TMP, WORD2;
    1656
    1657 -8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
    1658 +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
    1659 MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
    1660 LOAD(prefetch, %i1 + %i3, #one_read)
    1661
    1662 - EX_ST(STORE_INIT(%g2, %o0 + 0x00))
    1663 - EX_ST(STORE_INIT(%g3, %o0 + 0x08))
    1664 + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1)
    1665 + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1666
    1667 - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
    1668 + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
    1669 MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
    1670
    1671 - EX_ST(STORE_INIT(%o2, %o0 + 0x10))
    1672 - EX_ST(STORE_INIT(%o3, %o0 + 0x18))
    1673 + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1674 + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1675
    1676 - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1677 + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1678 MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
    1679
    1680 - EX_ST(STORE_INIT(%g2, %o0 + 0x20))
    1681 - EX_ST(STORE_INIT(%g3, %o0 + 0x28))
    1682 + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1683 + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1684
    1685 - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
    1686 + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
    1687 add %i1, 64, %i1
    1688 MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
    1689
    1690 - EX_ST(STORE_INIT(%o2, %o0 + 0x30))
    1691 - EX_ST(STORE_INIT(%o3, %o0 + 0x38))
    1692 + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1693 + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1694
    1695 subcc %g1, 64, %g1
    1696 bne,pt %XCC, 8b
    1697 @@ -211,31 +294,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1698 ba,pt %XCC, 60f
    1699 add %i1, %i4, %i1
    1700
    1701 -9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
    1702 +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
    1703 MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
    1704 LOAD(prefetch, %i1 + %i3, #one_read)
    1705
    1706 - EX_ST(STORE_INIT(%g3, %o0 + 0x00))
    1707 - EX_ST(STORE_INIT(%o2, %o0 + 0x08))
    1708 + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1)
    1709 + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1710
    1711 - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
    1712 + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
    1713 MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
    1714
    1715 - EX_ST(STORE_INIT(%o3, %o0 + 0x10))
    1716 - EX_ST(STORE_INIT(%g2, %o0 + 0x18))
    1717 + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1718 + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1719
    1720 - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1721 + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1722 MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
    1723
    1724 - EX_ST(STORE_INIT(%g3, %o0 + 0x20))
    1725 - EX_ST(STORE_INIT(%o2, %o0 + 0x28))
    1726 + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1727 + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1728
    1729 - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
    1730 + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
    1731 add %i1, 64, %i1
    1732 MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
    1733
    1734 - EX_ST(STORE_INIT(%o3, %o0 + 0x30))
    1735 - EX_ST(STORE_INIT(%g2, %o0 + 0x38))
    1736 + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1737 + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1738
    1739 subcc %g1, 64, %g1
    1740 bne,pt %XCC, 9b
    1741 @@ -249,25 +332,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1742 * one twin load ahead, then add 8 back into source when
    1743 * we finish the loop.
    1744 */
    1745 - EX_LD(LOAD_TWIN(%i1, %o4, %o5))
    1746 + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1)
    1747 mov 16, %o7
    1748 mov 32, %g2
    1749 mov 48, %g3
    1750 mov 64, %o1
    1751 -1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1752 +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
    1753 LOAD(prefetch, %i1 + %o1, #one_read)
    1754 - EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line
    1755 - EX_ST(STORE_INIT(%o2, %o0 + 0x08))
    1756 - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
    1757 - EX_ST(STORE_INIT(%o3, %o0 + 0x10))
    1758 - EX_ST(STORE_INIT(%o4, %o0 + 0x18))
    1759 - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
    1760 - EX_ST(STORE_INIT(%o5, %o0 + 0x20))
    1761 - EX_ST(STORE_INIT(%o2, %o0 + 0x28))
    1762 - EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5))
    1763 + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line
    1764 + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1765 + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
    1766 + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1767 + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1768 + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1769 + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1770 + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1771 + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48)
    1772 add %i1, 64, %i1
    1773 - EX_ST(STORE_INIT(%o3, %o0 + 0x30))
    1774 - EX_ST(STORE_INIT(%o4, %o0 + 0x38))
    1775 + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1776 + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1777 subcc %g1, 64, %g1
    1778 bne,pt %XCC, 1b
    1779 add %o0, 64, %o0
    1780 @@ -282,20 +365,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1781 mov 32, %g2
    1782 mov 48, %g3
    1783 mov 64, %o1
    1784 -1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5))
    1785 - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
    1786 +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1)
    1787 + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
    1788 LOAD(prefetch, %i1 + %o1, #one_read)
    1789 - EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line
    1790 - EX_ST(STORE_INIT(%o5, %o0 + 0x08))
    1791 - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
    1792 - EX_ST(STORE_INIT(%o2, %o0 + 0x10))
    1793 - EX_ST(STORE_INIT(%o3, %o0 + 0x18))
    1794 - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
    1795 + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line
    1796 + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
    1797 + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
    1798 + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
    1799 + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
    1800 + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
    1801 add %i1, 64, %i1
    1802 - EX_ST(STORE_INIT(%o4, %o0 + 0x20))
    1803 - EX_ST(STORE_INIT(%o5, %o0 + 0x28))
    1804 - EX_ST(STORE_INIT(%o2, %o0 + 0x30))
    1805 - EX_ST(STORE_INIT(%o3, %o0 + 0x38))
    1806 + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
    1807 + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
    1808 + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
    1809 + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
    1810 subcc %g1, 64, %g1
    1811 bne,pt %XCC, 1b
    1812 add %o0, 64, %o0
    1813 @@ -321,28 +404,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1814 andn %i2, 0xf, %i4
    1815 and %i2, 0xf, %i2
    1816 1: subcc %i4, 0x10, %i4
    1817 - EX_LD(LOAD(ldx, %i1, %o4))
    1818 + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4)
    1819 add %i1, 0x08, %i1
    1820 - EX_LD(LOAD(ldx, %i1, %g1))
    1821 + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4)
    1822 sub %i1, 0x08, %i1
    1823 - EX_ST(STORE(stx, %o4, %i1 + %i3))
    1824 + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4)
    1825 add %i1, 0x8, %i1
    1826 - EX_ST(STORE(stx, %g1, %i1 + %i3))
    1827 + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8)
    1828 bgu,pt %XCC, 1b
    1829 add %i1, 0x8, %i1
    1830 73: andcc %i2, 0x8, %g0
    1831 be,pt %XCC, 1f
    1832 nop
    1833 sub %i2, 0x8, %i2
    1834 - EX_LD(LOAD(ldx, %i1, %o4))
    1835 - EX_ST(STORE(stx, %o4, %i1 + %i3))
    1836 + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8)
    1837 + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8)
    1838 add %i1, 0x8, %i1
    1839 1: andcc %i2, 0x4, %g0
    1840 be,pt %XCC, 1f
    1841 nop
    1842 sub %i2, 0x4, %i2
    1843 - EX_LD(LOAD(lduw, %i1, %i5))
    1844 - EX_ST(STORE(stw, %i5, %i1 + %i3))
    1845 + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4)
    1846 + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4)
    1847 add %i1, 0x4, %i1
    1848 1: cmp %i2, 0
    1849 be,pt %XCC, 85f
    1850 @@ -358,8 +441,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1851 sub %i2, %g1, %i2
    1852
    1853 1: subcc %g1, 1, %g1
    1854 - EX_LD(LOAD(ldub, %i1, %i5))
    1855 - EX_ST(STORE(stb, %i5, %i1 + %i3))
    1856 + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1)
    1857 + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1)
    1858 bgu,pt %icc, 1b
    1859 add %i1, 1, %i1
    1860
    1861 @@ -375,16 +458,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1862
    1863 8: mov 64, %i3
    1864 andn %i1, 0x7, %i1
    1865 - EX_LD(LOAD(ldx, %i1, %g2))
    1866 + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2)
    1867 sub %i3, %g1, %i3
    1868 andn %i2, 0x7, %i4
    1869 sllx %g2, %g1, %g2
    1870 1: add %i1, 0x8, %i1
    1871 - EX_LD(LOAD(ldx, %i1, %g3))
    1872 + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4)
    1873 subcc %i4, 0x8, %i4
    1874 srlx %g3, %i3, %i5
    1875 or %i5, %g2, %i5
    1876 - EX_ST(STORE(stx, %i5, %o0))
    1877 + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4)
    1878 add %o0, 0x8, %o0
    1879 bgu,pt %icc, 1b
    1880 sllx %g3, %g1, %g2
    1881 @@ -404,8 +487,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1882
    1883 1:
    1884 subcc %i2, 4, %i2
    1885 - EX_LD(LOAD(lduw, %i1, %g1))
    1886 - EX_ST(STORE(stw, %g1, %i1 + %i3))
    1887 + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4)
    1888 + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4)
    1889 bgu,pt %XCC, 1b
    1890 add %i1, 4, %i1
    1891
    1892 @@ -415,8 +498,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */
    1893 .align 32
    1894 90:
    1895 subcc %i2, 1, %i2
    1896 - EX_LD(LOAD(ldub, %i1, %g1))
    1897 - EX_ST(STORE(stb, %g1, %i1 + %i3))
    1898 + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1)
    1899 + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1)
    1900 bgu,pt %XCC, 90b
    1901 add %i1, 1, %i1
    1902 ret
    1903 diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
    1904 index ecc5692fa2b4..bb6ff73229e3 100644
    1905 --- a/arch/sparc/lib/U1copy_from_user.S
    1906 +++ b/arch/sparc/lib/U1copy_from_user.S
    1907 @@ -3,19 +3,19 @@
    1908 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    1909 */
    1910
    1911 -#define EX_LD(x) \
    1912 +#define EX_LD(x,y) \
    1913 98: x; \
    1914 .section __ex_table,"a";\
    1915 .align 4; \
    1916 - .word 98b, __retl_one; \
    1917 + .word 98b, y; \
    1918 .text; \
    1919 .align 4;
    1920
    1921 -#define EX_LD_FP(x) \
    1922 +#define EX_LD_FP(x,y) \
    1923 98: x; \
    1924 .section __ex_table,"a";\
    1925 .align 4; \
    1926 - .word 98b, __retl_one_fp;\
    1927 + .word 98b, y; \
    1928 .text; \
    1929 .align 4;
    1930
    1931 diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
    1932 index 9eea392e44d4..ed92ce739558 100644
    1933 --- a/arch/sparc/lib/U1copy_to_user.S
    1934 +++ b/arch/sparc/lib/U1copy_to_user.S
    1935 @@ -3,19 +3,19 @@
    1936 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    1937 */
    1938
    1939 -#define EX_ST(x) \
    1940 +#define EX_ST(x,y) \
    1941 98: x; \
    1942 .section __ex_table,"a";\
    1943 .align 4; \
    1944 - .word 98b, __retl_one; \
    1945 + .word 98b, y; \
    1946 .text; \
    1947 .align 4;
    1948
    1949 -#define EX_ST_FP(x) \
    1950 +#define EX_ST_FP(x,y) \
    1951 98: x; \
    1952 .section __ex_table,"a";\
    1953 .align 4; \
    1954 - .word 98b, __retl_one_fp;\
    1955 + .word 98b, y; \
    1956 .text; \
    1957 .align 4;
    1958
    1959 diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S
    1960 index 3e6209ebb7d7..f30d2ab2c371 100644
    1961 --- a/arch/sparc/lib/U1memcpy.S
    1962 +++ b/arch/sparc/lib/U1memcpy.S
    1963 @@ -5,6 +5,7 @@
    1964 */
    1965
    1966 #ifdef __KERNEL__
    1967 +#include <linux/linkage.h>
    1968 #include <asm/visasm.h>
    1969 #include <asm/asi.h>
    1970 #define GLOBAL_SPARE g7
    1971 @@ -23,21 +24,17 @@
    1972 #endif
    1973
    1974 #ifndef EX_LD
    1975 -#define EX_LD(x) x
    1976 +#define EX_LD(x,y) x
    1977 #endif
    1978 #ifndef EX_LD_FP
    1979 -#define EX_LD_FP(x) x
    1980 +#define EX_LD_FP(x,y) x
    1981 #endif
    1982
    1983 #ifndef EX_ST
    1984 -#define EX_ST(x) x
    1985 +#define EX_ST(x,y) x
    1986 #endif
    1987 #ifndef EX_ST_FP
    1988 -#define EX_ST_FP(x) x
    1989 -#endif
    1990 -
    1991 -#ifndef EX_RETVAL
    1992 -#define EX_RETVAL(x) x
    1993 +#define EX_ST_FP(x,y) x
    1994 #endif
    1995
    1996 #ifndef LOAD
    1997 @@ -78,53 +75,169 @@
    1998 faligndata %f7, %f8, %f60; \
    1999 faligndata %f8, %f9, %f62;
    2000
    2001 -#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
    2002 - EX_LD_FP(LOAD_BLK(%src, %fdest)); \
    2003 - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    2004 - add %src, 0x40, %src; \
    2005 - subcc %len, 0x40, %len; \
    2006 - be,pn %xcc, jmptgt; \
    2007 - add %dest, 0x40, %dest; \
    2008 -
    2009 -#define LOOP_CHUNK1(src, dest, len, branch_dest) \
    2010 - MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
    2011 -#define LOOP_CHUNK2(src, dest, len, branch_dest) \
    2012 - MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
    2013 -#define LOOP_CHUNK3(src, dest, len, branch_dest) \
    2014 - MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
    2015 +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \
    2016 + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \
    2017 + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
    2018 + add %src, 0x40, %src; \
    2019 + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \
    2020 + be,pn %xcc, jmptgt; \
    2021 + add %dest, 0x40, %dest; \
    2022 +
    2023 +#define LOOP_CHUNK1(src, dest, branch_dest) \
    2024 + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest)
    2025 +#define LOOP_CHUNK2(src, dest, branch_dest) \
    2026 + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest)
    2027 +#define LOOP_CHUNK3(src, dest, branch_dest) \
    2028 + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest)
    2029
    2030 #define DO_SYNC membar #Sync;
    2031 #define STORE_SYNC(dest, fsrc) \
    2032 - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    2033 + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \
    2034 add %dest, 0x40, %dest; \
    2035 DO_SYNC
    2036
    2037 #define STORE_JUMP(dest, fsrc, target) \
    2038 - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
    2039 + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \
    2040 add %dest, 0x40, %dest; \
    2041 ba,pt %xcc, target; \
    2042 nop;
    2043
    2044 -#define FINISH_VISCHUNK(dest, f0, f1, left) \
    2045 - subcc %left, 8, %left;\
    2046 - bl,pn %xcc, 95f; \
    2047 - faligndata %f0, %f1, %f48; \
    2048 - EX_ST_FP(STORE(std, %f48, %dest)); \
    2049 +#define FINISH_VISCHUNK(dest, f0, f1) \
    2050 + subcc %g3, 8, %g3; \
    2051 + bl,pn %xcc, 95f; \
    2052 + faligndata %f0, %f1, %f48; \
    2053 + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \
    2054 add %dest, 8, %dest;
    2055
    2056 -#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
    2057 - subcc %left, 8, %left; \
    2058 - bl,pn %xcc, 95f; \
    2059 +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
    2060 + subcc %g3, 8, %g3; \
    2061 + bl,pn %xcc, 95f; \
    2062 fsrc2 %f0, %f1;
    2063
    2064 -#define UNEVEN_VISCHUNK(dest, f0, f1, left) \
    2065 - UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
    2066 +#define UNEVEN_VISCHUNK(dest, f0, f1) \
    2067 + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \
    2068 ba,a,pt %xcc, 93f;
    2069
    2070 .register %g2,#scratch
    2071 .register %g3,#scratch
    2072
    2073 .text
    2074 +#ifndef EX_RETVAL
    2075 +#define EX_RETVAL(x) x
    2076 +ENTRY(U1_g1_1_fp)
    2077 + VISExitHalf
    2078 + add %g1, 1, %g1
    2079 + add %g1, %g2, %g1
    2080 + retl
    2081 + add %g1, %o2, %o0
    2082 +ENDPROC(U1_g1_1_fp)
    2083 +ENTRY(U1_g2_0_fp)
    2084 + VISExitHalf
    2085 + retl
    2086 + add %g2, %o2, %o0
    2087 +ENDPROC(U1_g2_0_fp)
    2088 +ENTRY(U1_g2_8_fp)
    2089 + VISExitHalf
    2090 + add %g2, 8, %g2
    2091 + retl
    2092 + add %g2, %o2, %o0
    2093 +ENDPROC(U1_g2_8_fp)
    2094 +ENTRY(U1_gs_0_fp)
    2095 + VISExitHalf
    2096 + add %GLOBAL_SPARE, %g3, %o0
    2097 + retl
    2098 + add %o0, %o2, %o0
    2099 +ENDPROC(U1_gs_0_fp)
    2100 +ENTRY(U1_gs_80_fp)
    2101 + VISExitHalf
    2102 + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
    2103 + add %GLOBAL_SPARE, %g3, %o0
    2104 + retl
    2105 + add %o0, %o2, %o0
    2106 +ENDPROC(U1_gs_80_fp)
    2107 +ENTRY(U1_gs_40_fp)
    2108 + VISExitHalf
    2109 + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE
    2110 + add %GLOBAL_SPARE, %g3, %o0
    2111 + retl
    2112 + add %o0, %o2, %o0
    2113 +ENDPROC(U1_gs_40_fp)
    2114 +ENTRY(U1_g3_0_fp)
    2115 + VISExitHalf
    2116 + retl
    2117 + add %g3, %o2, %o0
    2118 +ENDPROC(U1_g3_0_fp)
    2119 +ENTRY(U1_g3_8_fp)
    2120 + VISExitHalf
    2121 + add %g3, 8, %g3
    2122 + retl
    2123 + add %g3, %o2, %o0
    2124 +ENDPROC(U1_g3_8_fp)
    2125 +ENTRY(U1_o2_0_fp)
    2126 + VISExitHalf
    2127 + retl
    2128 + mov %o2, %o0
    2129 +ENDPROC(U1_o2_0_fp)
    2130 +ENTRY(U1_o2_1_fp)
    2131 + VISExitHalf
    2132 + retl
    2133 + add %o2, 1, %o0
    2134 +ENDPROC(U1_o2_1_fp)
    2135 +ENTRY(U1_gs_0)
    2136 + VISExitHalf
    2137 + retl
    2138 + add %GLOBAL_SPARE, %o2, %o0
    2139 +ENDPROC(U1_gs_0)
    2140 +ENTRY(U1_gs_8)
    2141 + VISExitHalf
    2142 + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
    2143 + retl
    2144 + add %GLOBAL_SPARE, 0x8, %o0
    2145 +ENDPROC(U1_gs_8)
    2146 +ENTRY(U1_gs_10)
    2147 + VISExitHalf
    2148 + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE
    2149 + retl
    2150 + add %GLOBAL_SPARE, 0x10, %o0
    2151 +ENDPROC(U1_gs_10)
    2152 +ENTRY(U1_o2_0)
    2153 + retl
    2154 + mov %o2, %o0
    2155 +ENDPROC(U1_o2_0)
    2156 +ENTRY(U1_o2_8)
    2157 + retl
    2158 + add %o2, 8, %o0
    2159 +ENDPROC(U1_o2_8)
    2160 +ENTRY(U1_o2_4)
    2161 + retl
    2162 + add %o2, 4, %o0
    2163 +ENDPROC(U1_o2_4)
    2164 +ENTRY(U1_o2_1)
    2165 + retl
    2166 + add %o2, 1, %o0
    2167 +ENDPROC(U1_o2_1)
    2168 +ENTRY(U1_g1_0)
    2169 + retl
    2170 + add %g1, %o2, %o0
    2171 +ENDPROC(U1_g1_0)
    2172 +ENTRY(U1_g1_1)
    2173 + add %g1, 1, %g1
    2174 + retl
    2175 + add %g1, %o2, %o0
    2176 +ENDPROC(U1_g1_1)
    2177 +ENTRY(U1_gs_0_o2_adj)
    2178 + and %o2, 7, %o2
    2179 + retl
    2180 + add %GLOBAL_SPARE, %o2, %o0
    2181 +ENDPROC(U1_gs_0_o2_adj)
    2182 +ENTRY(U1_gs_8_o2_adj)
    2183 + and %o2, 7, %o2
    2184 + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE
    2185 + retl
    2186 + add %GLOBAL_SPARE, %o2, %o0
    2187 +ENDPROC(U1_gs_8_o2_adj)
    2188 +#endif
    2189 +
    2190 .align 64
    2191
    2192 .globl FUNC_NAME
    2193 @@ -166,8 +279,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2194 and %g2, 0x38, %g2
    2195
    2196 1: subcc %g1, 0x1, %g1
    2197 - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
    2198 - EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE))
    2199 + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp)
    2200 + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp)
    2201 bgu,pt %XCC, 1b
    2202 add %o1, 0x1, %o1
    2203
    2204 @@ -178,20 +291,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2205 be,pt %icc, 3f
    2206 alignaddr %o1, %g0, %o1
    2207
    2208 - EX_LD_FP(LOAD(ldd, %o1, %f4))
    2209 -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
    2210 + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp)
    2211 +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp)
    2212 add %o1, 0x8, %o1
    2213 subcc %g2, 0x8, %g2
    2214 faligndata %f4, %f6, %f0
    2215 - EX_ST_FP(STORE(std, %f0, %o0))
    2216 + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
    2217 be,pn %icc, 3f
    2218 add %o0, 0x8, %o0
    2219
    2220 - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
    2221 + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp)
    2222 add %o1, 0x8, %o1
    2223 subcc %g2, 0x8, %g2
    2224 faligndata %f6, %f4, %f0
    2225 - EX_ST_FP(STORE(std, %f0, %o0))
    2226 + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
    2227 bne,pt %icc, 1b
    2228 add %o0, 0x8, %o0
    2229
    2230 @@ -214,13 +327,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2231 add %g1, %GLOBAL_SPARE, %g1
    2232 subcc %o2, %g3, %o2
    2233
    2234 - EX_LD_FP(LOAD_BLK(%o1, %f0))
    2235 + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp)
    2236 add %o1, 0x40, %o1
    2237 add %g1, %g3, %g1
    2238 - EX_LD_FP(LOAD_BLK(%o1, %f16))
    2239 + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp)
    2240 add %o1, 0x40, %o1
    2241 sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
    2242 - EX_LD_FP(LOAD_BLK(%o1, %f32))
    2243 + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp)
    2244 add %o1, 0x40, %o1
    2245
    2246 /* There are 8 instances of the unrolled loop,
    2247 @@ -240,11 +353,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2248
    2249 .align 64
    2250 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
    2251 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2252 + LOOP_CHUNK1(o1, o0, 1f)
    2253 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
    2254 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2255 + LOOP_CHUNK2(o1, o0, 2f)
    2256 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
    2257 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2258 + LOOP_CHUNK3(o1, o0, 3f)
    2259 ba,pt %xcc, 1b+4
    2260 faligndata %f0, %f2, %f48
    2261 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
    2262 @@ -261,11 +374,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2263 STORE_JUMP(o0, f48, 56f)
    2264
    2265 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
    2266 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2267 + LOOP_CHUNK1(o1, o0, 1f)
    2268 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
    2269 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2270 + LOOP_CHUNK2(o1, o0, 2f)
    2271 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
    2272 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2273 + LOOP_CHUNK3(o1, o0, 3f)
    2274 ba,pt %xcc, 1b+4
    2275 faligndata %f2, %f4, %f48
    2276 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
    2277 @@ -282,11 +395,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2278 STORE_JUMP(o0, f48, 57f)
    2279
    2280 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
    2281 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2282 + LOOP_CHUNK1(o1, o0, 1f)
    2283 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
    2284 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2285 + LOOP_CHUNK2(o1, o0, 2f)
    2286 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
    2287 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2288 + LOOP_CHUNK3(o1, o0, 3f)
    2289 ba,pt %xcc, 1b+4
    2290 faligndata %f4, %f6, %f48
    2291 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
    2292 @@ -303,11 +416,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2293 STORE_JUMP(o0, f48, 58f)
    2294
    2295 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
    2296 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2297 + LOOP_CHUNK1(o1, o0, 1f)
    2298 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
    2299 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2300 + LOOP_CHUNK2(o1, o0, 2f)
    2301 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
    2302 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2303 + LOOP_CHUNK3(o1, o0, 3f)
    2304 ba,pt %xcc, 1b+4
    2305 faligndata %f6, %f8, %f48
    2306 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
    2307 @@ -324,11 +437,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2308 STORE_JUMP(o0, f48, 59f)
    2309
    2310 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
    2311 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2312 + LOOP_CHUNK1(o1, o0, 1f)
    2313 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
    2314 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2315 + LOOP_CHUNK2(o1, o0, 2f)
    2316 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
    2317 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2318 + LOOP_CHUNK3(o1, o0, 3f)
    2319 ba,pt %xcc, 1b+4
    2320 faligndata %f8, %f10, %f48
    2321 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
    2322 @@ -345,11 +458,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2323 STORE_JUMP(o0, f48, 60f)
    2324
    2325 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
    2326 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2327 + LOOP_CHUNK1(o1, o0, 1f)
    2328 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
    2329 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2330 + LOOP_CHUNK2(o1, o0, 2f)
    2331 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
    2332 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2333 + LOOP_CHUNK3(o1, o0, 3f)
    2334 ba,pt %xcc, 1b+4
    2335 faligndata %f10, %f12, %f48
    2336 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
    2337 @@ -366,11 +479,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2338 STORE_JUMP(o0, f48, 61f)
    2339
    2340 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
    2341 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2342 + LOOP_CHUNK1(o1, o0, 1f)
    2343 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
    2344 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2345 + LOOP_CHUNK2(o1, o0, 2f)
    2346 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
    2347 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2348 + LOOP_CHUNK3(o1, o0, 3f)
    2349 ba,pt %xcc, 1b+4
    2350 faligndata %f12, %f14, %f48
    2351 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
    2352 @@ -387,11 +500,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2353 STORE_JUMP(o0, f48, 62f)
    2354
    2355 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
    2356 - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
    2357 + LOOP_CHUNK1(o1, o0, 1f)
    2358 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2359 - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
    2360 + LOOP_CHUNK2(o1, o0, 2f)
    2361 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
    2362 - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
    2363 + LOOP_CHUNK3(o1, o0, 3f)
    2364 ba,pt %xcc, 1b+4
    2365 faligndata %f14, %f16, %f48
    2366 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2367 @@ -407,53 +520,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2368 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
    2369 STORE_JUMP(o0, f48, 63f)
    2370
    2371 -40: FINISH_VISCHUNK(o0, f0, f2, g3)
    2372 -41: FINISH_VISCHUNK(o0, f2, f4, g3)
    2373 -42: FINISH_VISCHUNK(o0, f4, f6, g3)
    2374 -43: FINISH_VISCHUNK(o0, f6, f8, g3)
    2375 -44: FINISH_VISCHUNK(o0, f8, f10, g3)
    2376 -45: FINISH_VISCHUNK(o0, f10, f12, g3)
    2377 -46: FINISH_VISCHUNK(o0, f12, f14, g3)
    2378 -47: UNEVEN_VISCHUNK(o0, f14, f0, g3)
    2379 -48: FINISH_VISCHUNK(o0, f16, f18, g3)
    2380 -49: FINISH_VISCHUNK(o0, f18, f20, g3)
    2381 -50: FINISH_VISCHUNK(o0, f20, f22, g3)
    2382 -51: FINISH_VISCHUNK(o0, f22, f24, g3)
    2383 -52: FINISH_VISCHUNK(o0, f24, f26, g3)
    2384 -53: FINISH_VISCHUNK(o0, f26, f28, g3)
    2385 -54: FINISH_VISCHUNK(o0, f28, f30, g3)
    2386 -55: UNEVEN_VISCHUNK(o0, f30, f0, g3)
    2387 -56: FINISH_VISCHUNK(o0, f32, f34, g3)
    2388 -57: FINISH_VISCHUNK(o0, f34, f36, g3)
    2389 -58: FINISH_VISCHUNK(o0, f36, f38, g3)
    2390 -59: FINISH_VISCHUNK(o0, f38, f40, g3)
    2391 -60: FINISH_VISCHUNK(o0, f40, f42, g3)
    2392 -61: FINISH_VISCHUNK(o0, f42, f44, g3)
    2393 -62: FINISH_VISCHUNK(o0, f44, f46, g3)
    2394 -63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
    2395 -
    2396 -93: EX_LD_FP(LOAD(ldd, %o1, %f2))
    2397 +40: FINISH_VISCHUNK(o0, f0, f2)
    2398 +41: FINISH_VISCHUNK(o0, f2, f4)
    2399 +42: FINISH_VISCHUNK(o0, f4, f6)
    2400 +43: FINISH_VISCHUNK(o0, f6, f8)
    2401 +44: FINISH_VISCHUNK(o0, f8, f10)
    2402 +45: FINISH_VISCHUNK(o0, f10, f12)
    2403 +46: FINISH_VISCHUNK(o0, f12, f14)
    2404 +47: UNEVEN_VISCHUNK(o0, f14, f0)
    2405 +48: FINISH_VISCHUNK(o0, f16, f18)
    2406 +49: FINISH_VISCHUNK(o0, f18, f20)
    2407 +50: FINISH_VISCHUNK(o0, f20, f22)
    2408 +51: FINISH_VISCHUNK(o0, f22, f24)
    2409 +52: FINISH_VISCHUNK(o0, f24, f26)
    2410 +53: FINISH_VISCHUNK(o0, f26, f28)
    2411 +54: FINISH_VISCHUNK(o0, f28, f30)
    2412 +55: UNEVEN_VISCHUNK(o0, f30, f0)
    2413 +56: FINISH_VISCHUNK(o0, f32, f34)
    2414 +57: FINISH_VISCHUNK(o0, f34, f36)
    2415 +58: FINISH_VISCHUNK(o0, f36, f38)
    2416 +59: FINISH_VISCHUNK(o0, f38, f40)
    2417 +60: FINISH_VISCHUNK(o0, f40, f42)
    2418 +61: FINISH_VISCHUNK(o0, f42, f44)
    2419 +62: FINISH_VISCHUNK(o0, f44, f46)
    2420 +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0)
    2421 +
    2422 +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp)
    2423 add %o1, 8, %o1
    2424 subcc %g3, 8, %g3
    2425 faligndata %f0, %f2, %f8
    2426 - EX_ST_FP(STORE(std, %f8, %o0))
    2427 + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
    2428 bl,pn %xcc, 95f
    2429 add %o0, 8, %o0
    2430 - EX_LD_FP(LOAD(ldd, %o1, %f0))
    2431 + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp)
    2432 add %o1, 8, %o1
    2433 subcc %g3, 8, %g3
    2434 faligndata %f2, %f0, %f8
    2435 - EX_ST_FP(STORE(std, %f8, %o0))
    2436 + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
    2437 bge,pt %xcc, 93b
    2438 add %o0, 8, %o0
    2439
    2440 95: brz,pt %o2, 2f
    2441 mov %g1, %o1
    2442
    2443 -1: EX_LD_FP(LOAD(ldub, %o1, %o3))
    2444 +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp)
    2445 add %o1, 1, %o1
    2446 subcc %o2, 1, %o2
    2447 - EX_ST_FP(STORE(stb, %o3, %o0))
    2448 + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp)
    2449 bne,pt %xcc, 1b
    2450 add %o0, 1, %o0
    2451
    2452 @@ -469,27 +582,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2453
    2454 72: andn %o2, 0xf, %GLOBAL_SPARE
    2455 and %o2, 0xf, %o2
    2456 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
    2457 - EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
    2458 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0)
    2459 + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0)
    2460 subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
    2461 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2462 + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10)
    2463 add %o1, 0x8, %o1
    2464 - EX_ST(STORE(stx, %g1, %o1 + %o3))
    2465 + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8)
    2466 bgu,pt %XCC, 1b
    2467 add %o1, 0x8, %o1
    2468 73: andcc %o2, 0x8, %g0
    2469 be,pt %XCC, 1f
    2470 nop
    2471 - EX_LD(LOAD(ldx, %o1, %o5))
    2472 + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0)
    2473 sub %o2, 0x8, %o2
    2474 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2475 + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8)
    2476 add %o1, 0x8, %o1
    2477 1: andcc %o2, 0x4, %g0
    2478 be,pt %XCC, 1f
    2479 nop
    2480 - EX_LD(LOAD(lduw, %o1, %o5))
    2481 + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0)
    2482 sub %o2, 0x4, %o2
    2483 - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2484 + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4)
    2485 add %o1, 0x4, %o1
    2486 1: cmp %o2, 0
    2487 be,pt %XCC, 85f
    2488 @@ -503,9 +616,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2489 sub %g0, %g1, %g1
    2490 sub %o2, %g1, %o2
    2491
    2492 -1: EX_LD(LOAD(ldub, %o1, %o5))
    2493 +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0)
    2494 subcc %g1, 1, %g1
    2495 - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2496 + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1)
    2497 bgu,pt %icc, 1b
    2498 add %o1, 1, %o1
    2499
    2500 @@ -521,16 +634,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2501
    2502 8: mov 64, %o3
    2503 andn %o1, 0x7, %o1
    2504 - EX_LD(LOAD(ldx, %o1, %g2))
    2505 + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0)
    2506 sub %o3, %g1, %o3
    2507 andn %o2, 0x7, %GLOBAL_SPARE
    2508 sllx %g2, %g1, %g2
    2509 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
    2510 +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj)
    2511 subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
    2512 add %o1, 0x8, %o1
    2513 srlx %g3, %o3, %o5
    2514 or %o5, %g2, %o5
    2515 - EX_ST(STORE(stx, %o5, %o0))
    2516 + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj)
    2517 add %o0, 0x8, %o0
    2518 bgu,pt %icc, 1b
    2519 sllx %g3, %g1, %g2
    2520 @@ -548,9 +661,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2521 bne,pn %XCC, 90f
    2522 sub %o0, %o1, %o3
    2523
    2524 -1: EX_LD(LOAD(lduw, %o1, %g1))
    2525 +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0)
    2526 subcc %o2, 4, %o2
    2527 - EX_ST(STORE(stw, %g1, %o1 + %o3))
    2528 + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4)
    2529 bgu,pt %XCC, 1b
    2530 add %o1, 4, %o1
    2531
    2532 @@ -558,9 +671,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2533 mov EX_RETVAL(%o4), %o0
    2534
    2535 .align 32
    2536 -90: EX_LD(LOAD(ldub, %o1, %g1))
    2537 +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0)
    2538 subcc %o2, 1, %o2
    2539 - EX_ST(STORE(stb, %g1, %o1 + %o3))
    2540 + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1)
    2541 bgu,pt %XCC, 90b
    2542 add %o1, 1, %o1
    2543 retl
    2544 diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S
    2545 index 88ad73d86fe4..db73010a1af8 100644
    2546 --- a/arch/sparc/lib/U3copy_from_user.S
    2547 +++ b/arch/sparc/lib/U3copy_from_user.S
    2548 @@ -3,19 +3,19 @@
    2549 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    2550 */
    2551
    2552 -#define EX_LD(x) \
    2553 +#define EX_LD(x,y) \
    2554 98: x; \
    2555 .section __ex_table,"a";\
    2556 .align 4; \
    2557 - .word 98b, __retl_one; \
    2558 + .word 98b, y; \
    2559 .text; \
    2560 .align 4;
    2561
    2562 -#define EX_LD_FP(x) \
    2563 +#define EX_LD_FP(x,y) \
    2564 98: x; \
    2565 .section __ex_table,"a";\
    2566 .align 4; \
    2567 - .word 98b, __retl_one_fp;\
    2568 + .word 98b, y##_fp; \
    2569 .text; \
    2570 .align 4;
    2571
    2572 diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
    2573 index 845139d75537..c4ee858e352a 100644
    2574 --- a/arch/sparc/lib/U3copy_to_user.S
    2575 +++ b/arch/sparc/lib/U3copy_to_user.S
    2576 @@ -3,19 +3,19 @@
    2577 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
    2578 */
    2579
    2580 -#define EX_ST(x) \
    2581 +#define EX_ST(x,y) \
    2582 98: x; \
    2583 .section __ex_table,"a";\
    2584 .align 4; \
    2585 - .word 98b, __retl_one; \
    2586 + .word 98b, y; \
    2587 .text; \
    2588 .align 4;
    2589
    2590 -#define EX_ST_FP(x) \
    2591 +#define EX_ST_FP(x,y) \
    2592 98: x; \
    2593 .section __ex_table,"a";\
    2594 .align 4; \
    2595 - .word 98b, __retl_one_fp;\
    2596 + .word 98b, y##_fp; \
    2597 .text; \
    2598 .align 4;
    2599
    2600 diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S
    2601 index 491ee69e4995..54f98706b03b 100644
    2602 --- a/arch/sparc/lib/U3memcpy.S
    2603 +++ b/arch/sparc/lib/U3memcpy.S
    2604 @@ -4,6 +4,7 @@
    2605 */
    2606
    2607 #ifdef __KERNEL__
    2608 +#include <linux/linkage.h>
    2609 #include <asm/visasm.h>
    2610 #include <asm/asi.h>
    2611 #define GLOBAL_SPARE %g7
    2612 @@ -22,21 +23,17 @@
    2613 #endif
    2614
    2615 #ifndef EX_LD
    2616 -#define EX_LD(x) x
    2617 +#define EX_LD(x,y) x
    2618 #endif
    2619 #ifndef EX_LD_FP
    2620 -#define EX_LD_FP(x) x
    2621 +#define EX_LD_FP(x,y) x
    2622 #endif
    2623
    2624 #ifndef EX_ST
    2625 -#define EX_ST(x) x
    2626 +#define EX_ST(x,y) x
    2627 #endif
    2628 #ifndef EX_ST_FP
    2629 -#define EX_ST_FP(x) x
    2630 -#endif
    2631 -
    2632 -#ifndef EX_RETVAL
    2633 -#define EX_RETVAL(x) x
    2634 +#define EX_ST_FP(x,y) x
    2635 #endif
    2636
    2637 #ifndef LOAD
    2638 @@ -77,6 +74,87 @@
    2639 */
    2640
    2641 .text
    2642 +#ifndef EX_RETVAL
    2643 +#define EX_RETVAL(x) x
    2644 +__restore_fp:
    2645 + VISExitHalf
    2646 + retl
    2647 + nop
    2648 +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
    2649 + add %g1, 1, %g1
    2650 + add %g2, %g1, %g2
    2651 + ba,pt %xcc, __restore_fp
    2652 + add %o2, %g2, %o0
    2653 +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
    2654 +ENTRY(U3_retl_o2_plus_g2_fp)
    2655 + ba,pt %xcc, __restore_fp
    2656 + add %o2, %g2, %o0
    2657 +ENDPROC(U3_retl_o2_plus_g2_fp)
    2658 +ENTRY(U3_retl_o2_plus_g2_plus_8_fp)
    2659 + add %g2, 8, %g2
    2660 + ba,pt %xcc, __restore_fp
    2661 + add %o2, %g2, %o0
    2662 +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp)
    2663 +ENTRY(U3_retl_o2)
    2664 + retl
    2665 + mov %o2, %o0
    2666 +ENDPROC(U3_retl_o2)
    2667 +ENTRY(U3_retl_o2_plus_1)
    2668 + retl
    2669 + add %o2, 1, %o0
    2670 +ENDPROC(U3_retl_o2_plus_1)
    2671 +ENTRY(U3_retl_o2_plus_4)
    2672 + retl
    2673 + add %o2, 4, %o0
    2674 +ENDPROC(U3_retl_o2_plus_4)
    2675 +ENTRY(U3_retl_o2_plus_8)
    2676 + retl
    2677 + add %o2, 8, %o0
    2678 +ENDPROC(U3_retl_o2_plus_8)
    2679 +ENTRY(U3_retl_o2_plus_g1_plus_1)
    2680 + add %g1, 1, %g1
    2681 + retl
    2682 + add %o2, %g1, %o0
    2683 +ENDPROC(U3_retl_o2_plus_g1_plus_1)
    2684 +ENTRY(U3_retl_o2_fp)
    2685 + ba,pt %xcc, __restore_fp
    2686 + mov %o2, %o0
    2687 +ENDPROC(U3_retl_o2_fp)
    2688 +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
    2689 + sll %o3, 6, %o3
    2690 + add %o3, 0x80, %o3
    2691 + ba,pt %xcc, __restore_fp
    2692 + add %o2, %o3, %o0
    2693 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
    2694 +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
    2695 + sll %o3, 6, %o3
    2696 + add %o3, 0x40, %o3
    2697 + ba,pt %xcc, __restore_fp
    2698 + add %o2, %o3, %o0
    2699 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
    2700 +ENTRY(U3_retl_o2_plus_GS_plus_0x10)
    2701 + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE
    2702 + retl
    2703 + add %o2, GLOBAL_SPARE, %o0
    2704 +ENDPROC(U3_retl_o2_plus_GS_plus_0x10)
    2705 +ENTRY(U3_retl_o2_plus_GS_plus_0x08)
    2706 + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE
    2707 + retl
    2708 + add %o2, GLOBAL_SPARE, %o0
    2709 +ENDPROC(U3_retl_o2_plus_GS_plus_0x08)
    2710 +ENTRY(U3_retl_o2_and_7_plus_GS)
    2711 + and %o2, 7, %o2
    2712 + retl
    2713 + add %o2, GLOBAL_SPARE, %o2
    2714 +ENDPROC(U3_retl_o2_and_7_plus_GS)
    2715 +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8)
    2716 + add GLOBAL_SPARE, 8, GLOBAL_SPARE
    2717 + and %o2, 7, %o2
    2718 + retl
    2719 + add %o2, GLOBAL_SPARE, %o2
    2720 +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8)
    2721 +#endif
    2722 +
    2723 .align 64
    2724
    2725 /* The cheetah's flexible spine, oversized liver, enlarged heart,
    2726 @@ -126,8 +204,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2727 and %g2, 0x38, %g2
    2728
    2729 1: subcc %g1, 0x1, %g1
    2730 - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
    2731 - EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
    2732 + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1)
    2733 + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1)
    2734 bgu,pt %XCC, 1b
    2735 add %o1, 0x1, %o1
    2736
    2737 @@ -138,20 +216,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2738 be,pt %icc, 3f
    2739 alignaddr %o1, %g0, %o1
    2740
    2741 - EX_LD_FP(LOAD(ldd, %o1, %f4))
    2742 -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
    2743 + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2)
    2744 +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2)
    2745 add %o1, 0x8, %o1
    2746 subcc %g2, 0x8, %g2
    2747 faligndata %f4, %f6, %f0
    2748 - EX_ST_FP(STORE(std, %f0, %o0))
    2749 + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8)
    2750 be,pn %icc, 3f
    2751 add %o0, 0x8, %o0
    2752
    2753 - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
    2754 + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2)
    2755 add %o1, 0x8, %o1
    2756 subcc %g2, 0x8, %g2
    2757 faligndata %f6, %f4, %f2
    2758 - EX_ST_FP(STORE(std, %f2, %o0))
    2759 + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8)
    2760 bne,pt %icc, 1b
    2761 add %o0, 0x8, %o0
    2762
    2763 @@ -161,25 +239,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2764 LOAD(prefetch, %o1 + 0x080, #one_read)
    2765 LOAD(prefetch, %o1 + 0x0c0, #one_read)
    2766 LOAD(prefetch, %o1 + 0x100, #one_read)
    2767 - EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0))
    2768 + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2)
    2769 LOAD(prefetch, %o1 + 0x140, #one_read)
    2770 - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2771 + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2)
    2772 LOAD(prefetch, %o1 + 0x180, #one_read)
    2773 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2774 + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2)
    2775 LOAD(prefetch, %o1 + 0x1c0, #one_read)
    2776 faligndata %f0, %f2, %f16
    2777 - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2778 + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2)
    2779 faligndata %f2, %f4, %f18
    2780 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2781 + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2)
    2782 faligndata %f4, %f6, %f20
    2783 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2784 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2)
    2785 faligndata %f6, %f8, %f22
    2786
    2787 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2788 + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2)
    2789 faligndata %f8, %f10, %f24
    2790 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2791 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2)
    2792 faligndata %f10, %f12, %f26
    2793 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2794 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2)
    2795
    2796 subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE
    2797 add %o1, 0x40, %o1
    2798 @@ -190,26 +268,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2799
    2800 .align 64
    2801 1:
    2802 - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2803 + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2804 faligndata %f12, %f14, %f28
    2805 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2806 + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2807 faligndata %f14, %f0, %f30
    2808 - EX_ST_FP(STORE_BLK(%f16, %o0))
    2809 - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2810 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2811 + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2812 faligndata %f0, %f2, %f16
    2813 add %o0, 0x40, %o0
    2814
    2815 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2816 + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2817 faligndata %f2, %f4, %f18
    2818 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2819 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2820 faligndata %f4, %f6, %f20
    2821 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2822 + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2823 subcc %o3, 0x01, %o3
    2824 faligndata %f6, %f8, %f22
    2825 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2826 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2827
    2828 faligndata %f8, %f10, %f24
    2829 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2830 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2831 LOAD(prefetch, %o1 + 0x1c0, #one_read)
    2832 faligndata %f10, %f12, %f26
    2833 bg,pt %XCC, 1b
    2834 @@ -217,29 +295,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2835
    2836 /* Finally we copy the last full 64-byte block. */
    2837 2:
    2838 - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
    2839 + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2840 faligndata %f12, %f14, %f28
    2841 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
    2842 + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2843 faligndata %f14, %f0, %f30
    2844 - EX_ST_FP(STORE_BLK(%f16, %o0))
    2845 - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
    2846 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
    2847 + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2848 faligndata %f0, %f2, %f16
    2849 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
    2850 + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2851 faligndata %f2, %f4, %f18
    2852 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
    2853 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2854 faligndata %f4, %f6, %f20
    2855 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
    2856 + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2857 faligndata %f6, %f8, %f22
    2858 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
    2859 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2860 faligndata %f8, %f10, %f24
    2861 cmp %g1, 0
    2862 be,pt %XCC, 1f
    2863 add %o0, 0x40, %o0
    2864 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
    2865 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2866 1: faligndata %f10, %f12, %f26
    2867 faligndata %f12, %f14, %f28
    2868 faligndata %f14, %f0, %f30
    2869 - EX_ST_FP(STORE_BLK(%f16, %o0))
    2870 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
    2871 add %o0, 0x40, %o0
    2872 add %o1, 0x40, %o1
    2873 membar #Sync
    2874 @@ -259,20 +337,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2875
    2876 sub %o2, %g2, %o2
    2877 be,a,pt %XCC, 1f
    2878 - EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0))
    2879 + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2)
    2880
    2881 -1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2))
    2882 +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2)
    2883 add %o1, 0x8, %o1
    2884 subcc %g2, 0x8, %g2
    2885 faligndata %f0, %f2, %f8
    2886 - EX_ST_FP(STORE(std, %f8, %o0))
    2887 + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
    2888 be,pn %XCC, 2f
    2889 add %o0, 0x8, %o0
    2890 - EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0))
    2891 + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2)
    2892 add %o1, 0x8, %o1
    2893 subcc %g2, 0x8, %g2
    2894 faligndata %f2, %f0, %f8
    2895 - EX_ST_FP(STORE(std, %f8, %o0))
    2896 + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
    2897 bne,pn %XCC, 1b
    2898 add %o0, 0x8, %o0
    2899
    2900 @@ -292,30 +370,33 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2901 andcc %o2, 0x8, %g0
    2902 be,pt %icc, 1f
    2903 nop
    2904 - EX_LD(LOAD(ldx, %o1, %o5))
    2905 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2906 + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2)
    2907 + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2)
    2908 add %o1, 0x8, %o1
    2909 + sub %o2, 8, %o2
    2910
    2911 1: andcc %o2, 0x4, %g0
    2912 be,pt %icc, 1f
    2913 nop
    2914 - EX_LD(LOAD(lduw, %o1, %o5))
    2915 - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2916 + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2)
    2917 + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2)
    2918 add %o1, 0x4, %o1
    2919 + sub %o2, 4, %o2
    2920
    2921 1: andcc %o2, 0x2, %g0
    2922 be,pt %icc, 1f
    2923 nop
    2924 - EX_LD(LOAD(lduh, %o1, %o5))
    2925 - EX_ST(STORE(sth, %o5, %o1 + %o3))
    2926 + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2)
    2927 + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2)
    2928 add %o1, 0x2, %o1
    2929 + sub %o2, 2, %o2
    2930
    2931 1: andcc %o2, 0x1, %g0
    2932 be,pt %icc, 85f
    2933 nop
    2934 - EX_LD(LOAD(ldub, %o1, %o5))
    2935 + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2)
    2936 ba,pt %xcc, 85f
    2937 - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2938 + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2)
    2939
    2940 .align 64
    2941 70: /* 16 < len <= 64 */
    2942 @@ -326,26 +407,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2943 andn %o2, 0xf, GLOBAL_SPARE
    2944 and %o2, 0xf, %o2
    2945 1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE
    2946 - EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
    2947 - EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
    2948 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2949 + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10)
    2950 + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10)
    2951 + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10)
    2952 add %o1, 0x8, %o1
    2953 - EX_ST(STORE(stx, %g1, %o1 + %o3))
    2954 + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08)
    2955 bgu,pt %XCC, 1b
    2956 add %o1, 0x8, %o1
    2957 73: andcc %o2, 0x8, %g0
    2958 be,pt %XCC, 1f
    2959 nop
    2960 sub %o2, 0x8, %o2
    2961 - EX_LD(LOAD(ldx, %o1, %o5))
    2962 - EX_ST(STORE(stx, %o5, %o1 + %o3))
    2963 + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8)
    2964 + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8)
    2965 add %o1, 0x8, %o1
    2966 1: andcc %o2, 0x4, %g0
    2967 be,pt %XCC, 1f
    2968 nop
    2969 sub %o2, 0x4, %o2
    2970 - EX_LD(LOAD(lduw, %o1, %o5))
    2971 - EX_ST(STORE(stw, %o5, %o1 + %o3))
    2972 + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4)
    2973 + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4)
    2974 add %o1, 0x4, %o1
    2975 1: cmp %o2, 0
    2976 be,pt %XCC, 85f
    2977 @@ -361,8 +442,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2978 sub %o2, %g1, %o2
    2979
    2980 1: subcc %g1, 1, %g1
    2981 - EX_LD(LOAD(ldub, %o1, %o5))
    2982 - EX_ST(STORE(stb, %o5, %o1 + %o3))
    2983 + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1)
    2984 + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1)
    2985 bgu,pt %icc, 1b
    2986 add %o1, 1, %o1
    2987
    2988 @@ -378,16 +459,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    2989
    2990 8: mov 64, %o3
    2991 andn %o1, 0x7, %o1
    2992 - EX_LD(LOAD(ldx, %o1, %g2))
    2993 + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2)
    2994 sub %o3, %g1, %o3
    2995 andn %o2, 0x7, GLOBAL_SPARE
    2996 sllx %g2, %g1, %g2
    2997 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
    2998 +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS)
    2999 subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE
    3000 add %o1, 0x8, %o1
    3001 srlx %g3, %o3, %o5
    3002 or %o5, %g2, %o5
    3003 - EX_ST(STORE(stx, %o5, %o0))
    3004 + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8)
    3005 add %o0, 0x8, %o0
    3006 bgu,pt %icc, 1b
    3007 sllx %g3, %g1, %g2
    3008 @@ -407,8 +488,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    3009
    3010 1:
    3011 subcc %o2, 4, %o2
    3012 - EX_LD(LOAD(lduw, %o1, %g1))
    3013 - EX_ST(STORE(stw, %g1, %o1 + %o3))
    3014 + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4)
    3015 + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4)
    3016 bgu,pt %XCC, 1b
    3017 add %o1, 4, %o1
    3018
    3019 @@ -418,8 +499,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
    3020 .align 32
    3021 90:
    3022 subcc %o2, 1, %o2
    3023 - EX_LD(LOAD(ldub, %o1, %g1))
    3024 - EX_ST(STORE(stb, %g1, %o1 + %o3))
    3025 + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1)
    3026 + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1)
    3027 bgu,pt %XCC, 90b
    3028 add %o1, 1, %o1
    3029 retl
    3030 diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
    3031 index 302c0e60dc2c..4c89b486fa0d 100644
    3032 --- a/arch/sparc/lib/copy_in_user.S
    3033 +++ b/arch/sparc/lib/copy_in_user.S
    3034 @@ -8,18 +8,33 @@
    3035
    3036 #define XCC xcc
    3037
    3038 -#define EX(x,y) \
    3039 +#define EX(x,y,z) \
    3040 98: x,y; \
    3041 .section __ex_table,"a";\
    3042 .align 4; \
    3043 - .word 98b, __retl_one; \
    3044 + .word 98b, z; \
    3045 .text; \
    3046 .align 4;
    3047
    3048 +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8)
    3049 +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4)
    3050 +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1)
    3051 +
    3052 .register %g2,#scratch
    3053 .register %g3,#scratch
    3054
    3055 .text
    3056 +__retl_o4_plus_8:
    3057 + add %o4, %o2, %o4
    3058 + retl
    3059 + add %o4, 8, %o0
    3060 +__retl_o2_plus_4:
    3061 + retl
    3062 + add %o2, 4, %o0
    3063 +__retl_o2_plus_1:
    3064 + retl
    3065 + add %o2, 1, %o0
    3066 +
    3067 .align 32
    3068
    3069 /* Don't try to get too fancy here, just nice and
    3070 @@ -44,8 +59,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    3071 andn %o2, 0x7, %o4
    3072 and %o2, 0x7, %o2
    3073 1: subcc %o4, 0x8, %o4
    3074 - EX(ldxa [%o1] %asi, %o5)
    3075 - EX(stxa %o5, [%o0] %asi)
    3076 + EX_O4(ldxa [%o1] %asi, %o5)
    3077 + EX_O4(stxa %o5, [%o0] %asi)
    3078 add %o1, 0x8, %o1
    3079 bgu,pt %XCC, 1b
    3080 add %o0, 0x8, %o0
    3081 @@ -53,8 +68,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    3082 be,pt %XCC, 1f
    3083 nop
    3084 sub %o2, 0x4, %o2
    3085 - EX(lduwa [%o1] %asi, %o5)
    3086 - EX(stwa %o5, [%o0] %asi)
    3087 + EX_O2_4(lduwa [%o1] %asi, %o5)
    3088 + EX_O2_4(stwa %o5, [%o0] %asi)
    3089 add %o1, 0x4, %o1
    3090 add %o0, 0x4, %o0
    3091 1: cmp %o2, 0
    3092 @@ -70,8 +85,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    3093
    3094 82:
    3095 subcc %o2, 4, %o2
    3096 - EX(lduwa [%o1] %asi, %g1)
    3097 - EX(stwa %g1, [%o0] %asi)
    3098 + EX_O2_4(lduwa [%o1] %asi, %g1)
    3099 + EX_O2_4(stwa %g1, [%o0] %asi)
    3100 add %o1, 4, %o1
    3101 bgu,pt %XCC, 82b
    3102 add %o0, 4, %o0
    3103 @@ -82,8 +97,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */
    3104 .align 32
    3105 90:
    3106 subcc %o2, 1, %o2
    3107 - EX(lduba [%o1] %asi, %g1)
    3108 - EX(stba %g1, [%o0] %asi)
    3109 + EX_O2_1(lduba [%o1] %asi, %g1)
    3110 + EX_O2_1(stba %g1, [%o0] %asi)
    3111 add %o1, 1, %o1
    3112 bgu,pt %XCC, 90b
    3113 add %o0, 1, %o0
    3114 diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c
    3115 deleted file mode 100644
    3116 index ac96ae236709..000000000000
    3117 --- a/arch/sparc/lib/user_fixup.c
    3118 +++ /dev/null
    3119 @@ -1,71 +0,0 @@
    3120 -/* user_fixup.c: Fix up user copy faults.
    3121 - *
    3122 - * Copyright (C) 2004 David S. Miller <davem@redhat.com>
    3123 - */
    3124 -
    3125 -#include <linux/compiler.h>
    3126 -#include <linux/kernel.h>
    3127 -#include <linux/string.h>
    3128 -#include <linux/errno.h>
    3129 -#include <linux/module.h>
    3130 -
    3131 -#include <asm/uaccess.h>
    3132 -
    3133 -/* Calculating the exact fault address when using
    3134 - * block loads and stores can be very complicated.
    3135 - *
    3136 - * Instead of trying to be clever and handling all
    3137 - * of the cases, just fix things up simply here.
    3138 - */
    3139 -
    3140 -static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset)
    3141 -{
    3142 - unsigned long fault_addr = current_thread_info()->fault_address;
    3143 - unsigned long end = start + size;
    3144 -
    3145 - if (fault_addr < start || fault_addr >= end) {
    3146 - *offset = 0;
    3147 - } else {
    3148 - *offset = fault_addr - start;
    3149 - size = end - fault_addr;
    3150 - }
    3151 - return size;
    3152 -}
    3153 -
    3154 -unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size)
    3155 -{
    3156 - unsigned long offset;
    3157 -
    3158 - size = compute_size((unsigned long) from, size, &offset);
    3159 - if (likely(size))
    3160 - memset(to + offset, 0, size);
    3161 -
    3162 - return size;
    3163 -}
    3164 -EXPORT_SYMBOL(copy_from_user_fixup);
    3165 -
    3166 -unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size)
    3167 -{
    3168 - unsigned long offset;
    3169 -
    3170 - return compute_size((unsigned long) to, size, &offset);
    3171 -}
    3172 -EXPORT_SYMBOL(copy_to_user_fixup);
    3173 -
    3174 -unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size)
    3175 -{
    3176 - unsigned long fault_addr = current_thread_info()->fault_address;
    3177 - unsigned long start = (unsigned long) to;
    3178 - unsigned long end = start + size;
    3179 -
    3180 - if (fault_addr >= start && fault_addr < end)
    3181 - return end - fault_addr;
    3182 -
    3183 - start = (unsigned long) from;
    3184 - end = start + size;
    3185 - if (fault_addr >= start && fault_addr < end)
    3186 - return end - fault_addr;
    3187 -
    3188 - return size;
    3189 -}
    3190 -EXPORT_SYMBOL(copy_in_user_fixup);
    3191 diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
    3192 index dbabe5713a15..e15f33715103 100644
    3193 --- a/arch/sparc/mm/fault_64.c
    3194 +++ b/arch/sparc/mm/fault_64.c
    3195 @@ -479,14 +479,14 @@ good_area:
    3196 up_read(&mm->mmap_sem);
    3197
    3198 mm_rss = get_mm_rss(mm);
    3199 -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3200 - mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE));
    3201 +#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3202 + mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE));
    3203 #endif
    3204 if (unlikely(mm_rss >
    3205 mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
    3206 tsb_grow(mm, MM_TSB_BASE, mm_rss);
    3207 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3208 - mm_rss = mm->context.huge_pte_count;
    3209 + mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count;
    3210 if (unlikely(mm_rss >
    3211 mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) {
    3212 if (mm->context.tsb_block[MM_TSB_HUGE].tsb)
    3213 diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
    3214 index 364d093f46c6..da1142401bf4 100644
    3215 --- a/arch/sparc/mm/hugetlbpage.c
    3216 +++ b/arch/sparc/mm/hugetlbpage.c
    3217 @@ -180,7 +180,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
    3218 unsigned long nptes;
    3219
    3220 if (!pte_present(*ptep) && pte_present(entry))
    3221 - mm->context.huge_pte_count++;
    3222 + mm->context.hugetlb_pte_count++;
    3223
    3224 addr &= HPAGE_MASK;
    3225
    3226 @@ -212,7 +212,7 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
    3227
    3228 entry = *ptep;
    3229 if (pte_present(entry))
    3230 - mm->context.huge_pte_count--;
    3231 + mm->context.hugetlb_pte_count--;
    3232
    3233 addr &= HPAGE_MASK;
    3234 nptes = 1 << HUGETLB_PAGE_ORDER;
    3235 diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
    3236 index 3c4b8975fa76..a5331c336b2a 100644
    3237 --- a/arch/sparc/mm/init_64.c
    3238 +++ b/arch/sparc/mm/init_64.c
    3239 @@ -346,7 +346,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
    3240 spin_lock_irqsave(&mm->context.lock, flags);
    3241
    3242 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3243 - if (mm->context.huge_pte_count && is_hugetlb_pte(pte))
    3244 + if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) &&
    3245 + is_hugetlb_pte(pte))
    3246 __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
    3247 address, pte_val(pte));
    3248 else
    3249 diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
    3250 index f81cd9736700..3659d37b4d81 100644
    3251 --- a/arch/sparc/mm/tlb.c
    3252 +++ b/arch/sparc/mm/tlb.c
    3253 @@ -175,9 +175,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
    3254
    3255 if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
    3256 if (pmd_val(pmd) & _PAGE_PMD_HUGE)
    3257 - mm->context.huge_pte_count++;
    3258 + mm->context.thp_pte_count++;
    3259 else
    3260 - mm->context.huge_pte_count--;
    3261 + mm->context.thp_pte_count--;
    3262
    3263 /* Do not try to allocate the TSB hash table if we
    3264 * don't have one already. We have various locks held
    3265 diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
    3266 index a0604a493a36..9cdeca0fa955 100644
    3267 --- a/arch/sparc/mm/tsb.c
    3268 +++ b/arch/sparc/mm/tsb.c
    3269 @@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr)
    3270 return (tag == (vaddr >> 22));
    3271 }
    3272
    3273 +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end)
    3274 +{
    3275 + unsigned long idx;
    3276 +
    3277 + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) {
    3278 + struct tsb *ent = &swapper_tsb[idx];
    3279 + unsigned long match = idx << 13;
    3280 +
    3281 + match |= (ent->tag << 22);
    3282 + if (match >= start && match < end)
    3283 + ent->tag = (1UL << TSB_TAG_INVALID_BIT);
    3284 + }
    3285 +}
    3286 +
    3287 /* TSB flushes need only occur on the processor initiating the address
    3288 * space modification, not on each cpu the address space has run on.
    3289 * Only the TLB flush needs that treatment.
    3290 @@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
    3291 {
    3292 unsigned long v;
    3293
    3294 + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES)
    3295 + return flush_tsb_kernel_range_scan(start, end);
    3296 +
    3297 for (v = start; v < end; v += PAGE_SIZE) {
    3298 unsigned long hash = tsb_hash(v, PAGE_SHIFT,
    3299 KERNEL_TSB_NENTRIES);
    3300 @@ -470,7 +487,7 @@ retry_tsb_alloc:
    3301 int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
    3302 {
    3303 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3304 - unsigned long huge_pte_count;
    3305 + unsigned long total_huge_pte_count;
    3306 #endif
    3307 unsigned int i;
    3308
    3309 @@ -479,12 +496,14 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
    3310 mm->context.sparc64_ctx_val = 0UL;
    3311
    3312 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3313 - /* We reset it to zero because the fork() page copying
    3314 + /* We reset them to zero because the fork() page copying
    3315 * will re-increment the counters as the parent PTEs are
    3316 * copied into the child address space.
    3317 */
    3318 - huge_pte_count = mm->context.huge_pte_count;
    3319 - mm->context.huge_pte_count = 0;
    3320 + total_huge_pte_count = mm->context.hugetlb_pte_count +
    3321 + mm->context.thp_pte_count;
    3322 + mm->context.hugetlb_pte_count = 0;
    3323 + mm->context.thp_pte_count = 0;
    3324 #endif
    3325
    3326 /* copy_mm() copies over the parent's mm_struct before calling
    3327 @@ -500,8 +519,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
    3328 tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm));
    3329
    3330 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
    3331 - if (unlikely(huge_pte_count))
    3332 - tsb_grow(mm, MM_TSB_HUGE, huge_pte_count);
    3333 + if (unlikely(total_huge_pte_count))
    3334 + tsb_grow(mm, MM_TSB_HUGE, total_huge_pte_count);
    3335 #endif
    3336
    3337 if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb))
    3338 diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
    3339 index b4f4733abc6e..5d2fd6cd3189 100644
    3340 --- a/arch/sparc/mm/ultra.S
    3341 +++ b/arch/sparc/mm/ultra.S
    3342 @@ -30,7 +30,7 @@
    3343 .text
    3344 .align 32
    3345 .globl __flush_tlb_mm
    3346 -__flush_tlb_mm: /* 18 insns */
    3347 +__flush_tlb_mm: /* 19 insns */
    3348 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
    3349 ldxa [%o1] ASI_DMMU, %g2
    3350 cmp %g2, %o0
    3351 @@ -81,7 +81,7 @@ __flush_tlb_page: /* 22 insns */
    3352
    3353 .align 32
    3354 .globl __flush_tlb_pending
    3355 -__flush_tlb_pending: /* 26 insns */
    3356 +__flush_tlb_pending: /* 27 insns */
    3357 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
    3358 rdpr %pstate, %g7
    3359 sllx %o1, 3, %o1
    3360 @@ -113,12 +113,14 @@ __flush_tlb_pending: /* 26 insns */
    3361
    3362 .align 32
    3363 .globl __flush_tlb_kernel_range
    3364 -__flush_tlb_kernel_range: /* 16 insns */
    3365 +__flush_tlb_kernel_range: /* 31 insns */
    3366 /* %o0=start, %o1=end */
    3367 cmp %o0, %o1
    3368 be,pn %xcc, 2f
    3369 + sub %o1, %o0, %o3
    3370 + srlx %o3, 18, %o4
    3371 + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow
    3372 sethi %hi(PAGE_SIZE), %o4
    3373 - sub %o1, %o0, %o3
    3374 sub %o3, %o4, %o3
    3375 or %o0, 0x20, %o0 ! Nucleus
    3376 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
    3377 @@ -131,6 +133,41 @@ __flush_tlb_kernel_range: /* 16 insns */
    3378 retl
    3379 nop
    3380 nop
    3381 + nop
    3382 + nop
    3383 + nop
    3384 + nop
    3385 + nop
    3386 + nop
    3387 + nop
    3388 + nop
    3389 + nop
    3390 + nop
    3391 + nop
    3392 + nop
    3393 + nop
    3394 +
    3395 +__spitfire_flush_tlb_kernel_range_slow:
    3396 + mov 63 * 8, %o4
    3397 +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3
    3398 + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */
    3399 + bne,pn %xcc, 2f
    3400 + mov TLB_TAG_ACCESS, %o3
    3401 + stxa %g0, [%o3] ASI_IMMU
    3402 + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS
    3403 + membar #Sync
    3404 +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3
    3405 + andcc %o3, 0x40, %g0
    3406 + bne,pn %xcc, 2f
    3407 + mov TLB_TAG_ACCESS, %o3
    3408 + stxa %g0, [%o3] ASI_DMMU
    3409 + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS
    3410 + membar #Sync
    3411 +2: sub %o4, 8, %o4
    3412 + brgez,pt %o4, 1b
    3413 + nop
    3414 + retl
    3415 + nop
    3416
    3417 __spitfire_flush_tlb_mm_slow:
    3418 rdpr %pstate, %g1
    3419 @@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */
    3420 retl
    3421 wrpr %g7, 0x0, %pstate
    3422
    3423 +__cheetah_flush_tlb_kernel_range: /* 31 insns */
    3424 + /* %o0=start, %o1=end */
    3425 + cmp %o0, %o1
    3426 + be,pn %xcc, 2f
    3427 + sub %o1, %o0, %o3
    3428 + srlx %o3, 18, %o4
    3429 + brnz,pn %o4, 3f
    3430 + sethi %hi(PAGE_SIZE), %o4
    3431 + sub %o3, %o4, %o3
    3432 + or %o0, 0x20, %o0 ! Nucleus
    3433 +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
    3434 + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
    3435 + membar #Sync
    3436 + brnz,pt %o3, 1b
    3437 + sub %o3, %o4, %o3
    3438 +2: sethi %hi(KERNBASE), %o3
    3439 + flush %o3
    3440 + retl
    3441 + nop
    3442 +3: mov 0x80, %o4
    3443 + stxa %g0, [%o4] ASI_DMMU_DEMAP
    3444 + membar #Sync
    3445 + stxa %g0, [%o4] ASI_IMMU_DEMAP
    3446 + membar #Sync
    3447 + retl
    3448 + nop
    3449 + nop
    3450 + nop
    3451 + nop
    3452 + nop
    3453 + nop
    3454 + nop
    3455 + nop
    3456 +
    3457 #ifdef DCACHE_ALIASING_POSSIBLE
    3458 __cheetah_flush_dcache_page: /* 11 insns */
    3459 sethi %hi(PAGE_OFFSET), %g1
    3460 @@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error:
    3461 ret
    3462 restore
    3463
    3464 -__hypervisor_flush_tlb_mm: /* 10 insns */
    3465 +__hypervisor_flush_tlb_mm: /* 19 insns */
    3466 mov %o0, %o2 /* ARG2: mmu context */
    3467 mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3468 mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3469 mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3470 mov HV_FAST_MMU_DEMAP_CTX, %o5
    3471 ta HV_FAST_TRAP
    3472 - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3473 + brnz,pn %o0, 1f
    3474 mov HV_FAST_MMU_DEMAP_CTX, %o1
    3475 retl
    3476 nop
    3477 +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5
    3478 + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0
    3479 + nop
    3480 + nop
    3481 + nop
    3482 + nop
    3483 + nop
    3484 + nop
    3485 + nop
    3486
    3487 -__hypervisor_flush_tlb_page: /* 11 insns */
    3488 +__hypervisor_flush_tlb_page: /* 22 insns */
    3489 /* %o0 = context, %o1 = vaddr */
    3490 mov %o0, %g2
    3491 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */
    3492 @@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */
    3493 srlx %o0, PAGE_SHIFT, %o0
    3494 sllx %o0, PAGE_SHIFT, %o0
    3495 ta HV_MMU_UNMAP_ADDR_TRAP
    3496 - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3497 + brnz,pn %o0, 1f
    3498 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3499 retl
    3500 nop
    3501 +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3502 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3503 + nop
    3504 + nop
    3505 + nop
    3506 + nop
    3507 + nop
    3508 + nop
    3509 + nop
    3510 + nop
    3511 + nop
    3512
    3513 -__hypervisor_flush_tlb_pending: /* 16 insns */
    3514 +__hypervisor_flush_tlb_pending: /* 27 insns */
    3515 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
    3516 sllx %o1, 3, %g1
    3517 mov %o2, %g2
    3518 @@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */
    3519 srlx %o0, PAGE_SHIFT, %o0
    3520 sllx %o0, PAGE_SHIFT, %o0
    3521 ta HV_MMU_UNMAP_ADDR_TRAP
    3522 - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3523 + brnz,pn %o0, 1f
    3524 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3525 brnz,pt %g1, 1b
    3526 nop
    3527 retl
    3528 nop
    3529 +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3530 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3531 + nop
    3532 + nop
    3533 + nop
    3534 + nop
    3535 + nop
    3536 + nop
    3537 + nop
    3538 + nop
    3539 + nop
    3540
    3541 -__hypervisor_flush_tlb_kernel_range: /* 16 insns */
    3542 +__hypervisor_flush_tlb_kernel_range: /* 31 insns */
    3543 /* %o0=start, %o1=end */
    3544 cmp %o0, %o1
    3545 be,pn %xcc, 2f
    3546 - sethi %hi(PAGE_SIZE), %g3
    3547 - mov %o0, %g1
    3548 - sub %o1, %g1, %g2
    3549 + sub %o1, %o0, %g2
    3550 + srlx %g2, 18, %g3
    3551 + brnz,pn %g3, 4f
    3552 + mov %o0, %g1
    3553 + sethi %hi(PAGE_SIZE), %g3
    3554 sub %g2, %g3, %g2
    3555 1: add %g1, %g2, %o0 /* ARG0: virtual address */
    3556 mov 0, %o1 /* ARG1: mmu context */
    3557 mov HV_MMU_ALL, %o2 /* ARG2: flags */
    3558 ta HV_MMU_UNMAP_ADDR_TRAP
    3559 - brnz,pn %o0, __hypervisor_tlb_tl0_error
    3560 + brnz,pn %o0, 3f
    3561 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
    3562 brnz,pt %g2, 1b
    3563 sub %g2, %g3, %g2
    3564 2: retl
    3565 nop
    3566 +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2
    3567 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
    3568 + nop
    3569 +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3570 + mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3571 + mov 0, %o2 /* ARG2: mmu context == nucleus */
    3572 + mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3573 + mov HV_FAST_MMU_DEMAP_CTX, %o5
    3574 + ta HV_FAST_TRAP
    3575 + brnz,pn %o0, 3b
    3576 + mov HV_FAST_MMU_DEMAP_CTX, %o1
    3577 + retl
    3578 + nop
    3579
    3580 #ifdef DCACHE_ALIASING_POSSIBLE
    3581 /* XXX Niagara and friends have an 8K cache, so no aliasing is
    3582 @@ -394,43 +511,6 @@ tlb_patch_one:
    3583 retl
    3584 nop
    3585
    3586 - .globl cheetah_patch_cachetlbops
    3587 -cheetah_patch_cachetlbops:
    3588 - save %sp, -128, %sp
    3589 -
    3590 - sethi %hi(__flush_tlb_mm), %o0
    3591 - or %o0, %lo(__flush_tlb_mm), %o0
    3592 - sethi %hi(__cheetah_flush_tlb_mm), %o1
    3593 - or %o1, %lo(__cheetah_flush_tlb_mm), %o1
    3594 - call tlb_patch_one
    3595 - mov 19, %o2
    3596 -
    3597 - sethi %hi(__flush_tlb_page), %o0
    3598 - or %o0, %lo(__flush_tlb_page), %o0
    3599 - sethi %hi(__cheetah_flush_tlb_page), %o1
    3600 - or %o1, %lo(__cheetah_flush_tlb_page), %o1
    3601 - call tlb_patch_one
    3602 - mov 22, %o2
    3603 -
    3604 - sethi %hi(__flush_tlb_pending), %o0
    3605 - or %o0, %lo(__flush_tlb_pending), %o0
    3606 - sethi %hi(__cheetah_flush_tlb_pending), %o1
    3607 - or %o1, %lo(__cheetah_flush_tlb_pending), %o1
    3608 - call tlb_patch_one
    3609 - mov 27, %o2
    3610 -
    3611 -#ifdef DCACHE_ALIASING_POSSIBLE
    3612 - sethi %hi(__flush_dcache_page), %o0
    3613 - or %o0, %lo(__flush_dcache_page), %o0
    3614 - sethi %hi(__cheetah_flush_dcache_page), %o1
    3615 - or %o1, %lo(__cheetah_flush_dcache_page), %o1
    3616 - call tlb_patch_one
    3617 - mov 11, %o2
    3618 -#endif /* DCACHE_ALIASING_POSSIBLE */
    3619 -
    3620 - ret
    3621 - restore
    3622 -
    3623 #ifdef CONFIG_SMP
    3624 /* These are all called by the slaves of a cross call, at
    3625 * trap level 1, with interrupts fully disabled.
    3626 @@ -447,7 +527,7 @@ cheetah_patch_cachetlbops:
    3627 */
    3628 .align 32
    3629 .globl xcall_flush_tlb_mm
    3630 -xcall_flush_tlb_mm: /* 21 insns */
    3631 +xcall_flush_tlb_mm: /* 24 insns */
    3632 mov PRIMARY_CONTEXT, %g2
    3633 ldxa [%g2] ASI_DMMU, %g3
    3634 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
    3635 @@ -469,9 +549,12 @@ xcall_flush_tlb_mm: /* 21 insns */
    3636 nop
    3637 nop
    3638 nop
    3639 + nop
    3640 + nop
    3641 + nop
    3642
    3643 .globl xcall_flush_tlb_page
    3644 -xcall_flush_tlb_page: /* 17 insns */
    3645 +xcall_flush_tlb_page: /* 20 insns */
    3646 /* %g5=context, %g1=vaddr */
    3647 mov PRIMARY_CONTEXT, %g4
    3648 ldxa [%g4] ASI_DMMU, %g2
    3649 @@ -490,15 +573,20 @@ xcall_flush_tlb_page: /* 17 insns */
    3650 retry
    3651 nop
    3652 nop
    3653 + nop
    3654 + nop
    3655 + nop
    3656
    3657 .globl xcall_flush_tlb_kernel_range
    3658 -xcall_flush_tlb_kernel_range: /* 25 insns */
    3659 +xcall_flush_tlb_kernel_range: /* 44 insns */
    3660 sethi %hi(PAGE_SIZE - 1), %g2
    3661 or %g2, %lo(PAGE_SIZE - 1), %g2
    3662 andn %g1, %g2, %g1
    3663 andn %g7, %g2, %g7
    3664 sub %g7, %g1, %g3
    3665 - add %g2, 1, %g2
    3666 + srlx %g3, 18, %g2
    3667 + brnz,pn %g2, 2f
    3668 + add %g2, 1, %g2
    3669 sub %g3, %g2, %g3
    3670 or %g1, 0x20, %g1 ! Nucleus
    3671 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
    3672 @@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */
    3673 brnz,pt %g3, 1b
    3674 sub %g3, %g2, %g3
    3675 retry
    3676 - nop
    3677 - nop
    3678 +2: mov 63 * 8, %g1
    3679 +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2
    3680 + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */
    3681 + bne,pn %xcc, 2f
    3682 + mov TLB_TAG_ACCESS, %g2
    3683 + stxa %g0, [%g2] ASI_IMMU
    3684 + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS
    3685 + membar #Sync
    3686 +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2
    3687 + andcc %g2, 0x40, %g0
    3688 + bne,pn %xcc, 2f
    3689 + mov TLB_TAG_ACCESS, %g2
    3690 + stxa %g0, [%g2] ASI_DMMU
    3691 + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS
    3692 + membar #Sync
    3693 +2: sub %g1, 8, %g1
    3694 + brgez,pt %g1, 1b
    3695 + nop
    3696 + retry
    3697 nop
    3698 nop
    3699 nop
    3700 @@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4:
    3701
    3702 retry
    3703
    3704 +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */
    3705 + sethi %hi(PAGE_SIZE - 1), %g2
    3706 + or %g2, %lo(PAGE_SIZE - 1), %g2
    3707 + andn %g1, %g2, %g1
    3708 + andn %g7, %g2, %g7
    3709 + sub %g7, %g1, %g3
    3710 + srlx %g3, 18, %g2
    3711 + brnz,pn %g2, 2f
    3712 + add %g2, 1, %g2
    3713 + sub %g3, %g2, %g3
    3714 + or %g1, 0x20, %g1 ! Nucleus
    3715 +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
    3716 + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
    3717 + membar #Sync
    3718 + brnz,pt %g3, 1b
    3719 + sub %g3, %g2, %g3
    3720 + retry
    3721 +2: mov 0x80, %g2
    3722 + stxa %g0, [%g2] ASI_DMMU_DEMAP
    3723 + membar #Sync
    3724 + stxa %g0, [%g2] ASI_IMMU_DEMAP
    3725 + membar #Sync
    3726 + retry
    3727 + nop
    3728 + nop
    3729 + nop
    3730 + nop
    3731 + nop
    3732 + nop
    3733 + nop
    3734 + nop
    3735 + nop
    3736 + nop
    3737 + nop
    3738 + nop
    3739 + nop
    3740 + nop
    3741 + nop
    3742 + nop
    3743 + nop
    3744 + nop
    3745 + nop
    3746 + nop
    3747 + nop
    3748 + nop
    3749 +
    3750 #ifdef DCACHE_ALIASING_POSSIBLE
    3751 .align 32
    3752 .globl xcall_flush_dcache_page_cheetah
    3753 @@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error:
    3754 ba,a,pt %xcc, rtrap
    3755
    3756 .globl __hypervisor_xcall_flush_tlb_mm
    3757 -__hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3758 +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
    3759 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
    3760 mov %o0, %g2
    3761 mov %o1, %g3
    3762 @@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3763 mov HV_FAST_MMU_DEMAP_CTX, %o5
    3764 ta HV_FAST_TRAP
    3765 mov HV_FAST_MMU_DEMAP_CTX, %g6
    3766 - brnz,pn %o0, __hypervisor_tlb_xcall_error
    3767 + brnz,pn %o0, 1f
    3768 mov %o0, %g5
    3769 mov %g2, %o0
    3770 mov %g3, %o1
    3771 @@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
    3772 mov %g7, %o5
    3773 membar #Sync
    3774 retry
    3775 +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3776 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3777 + nop
    3778
    3779 .globl __hypervisor_xcall_flush_tlb_page
    3780 -__hypervisor_xcall_flush_tlb_page: /* 17 insns */
    3781 +__hypervisor_xcall_flush_tlb_page: /* 20 insns */
    3782 /* %g5=ctx, %g1=vaddr */
    3783 mov %o0, %g2
    3784 mov %o1, %g3
    3785 @@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */
    3786 sllx %o0, PAGE_SHIFT, %o0
    3787 ta HV_MMU_UNMAP_ADDR_TRAP
    3788 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
    3789 - brnz,a,pn %o0, __hypervisor_tlb_xcall_error
    3790 + brnz,a,pn %o0, 1f
    3791 mov %o0, %g5
    3792 mov %g2, %o0
    3793 mov %g3, %o1
    3794 mov %g4, %o2
    3795 membar #Sync
    3796 retry
    3797 +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3798 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3799 + nop
    3800
    3801 .globl __hypervisor_xcall_flush_tlb_kernel_range
    3802 -__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
    3803 +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
    3804 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
    3805 sethi %hi(PAGE_SIZE - 1), %g2
    3806 or %g2, %lo(PAGE_SIZE - 1), %g2
    3807 andn %g1, %g2, %g1
    3808 andn %g7, %g2, %g7
    3809 sub %g7, %g1, %g3
    3810 + srlx %g3, 18, %g7
    3811 add %g2, 1, %g2
    3812 sub %g3, %g2, %g3
    3813 mov %o0, %g2
    3814 mov %o1, %g4
    3815 - mov %o2, %g7
    3816 + brnz,pn %g7, 2f
    3817 + mov %o2, %g7
    3818 1: add %g1, %g3, %o0 /* ARG0: virtual address */
    3819 mov 0, %o1 /* ARG1: mmu context */
    3820 mov HV_MMU_ALL, %o2 /* ARG2: flags */
    3821 ta HV_MMU_UNMAP_ADDR_TRAP
    3822 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
    3823 - brnz,pn %o0, __hypervisor_tlb_xcall_error
    3824 + brnz,pn %o0, 1f
    3825 mov %o0, %g5
    3826 sethi %hi(PAGE_SIZE), %o2
    3827 brnz,pt %g3, 1b
    3828 sub %g3, %o2, %g3
    3829 - mov %g2, %o0
    3830 +5: mov %g2, %o0
    3831 mov %g4, %o1
    3832 mov %g7, %o2
    3833 membar #Sync
    3834 retry
    3835 +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4
    3836 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
    3837 + nop
    3838 +2: mov %o3, %g1
    3839 + mov %o5, %g3
    3840 + mov 0, %o0 /* ARG0: CPU lists unimplemented */
    3841 + mov 0, %o1 /* ARG1: CPU lists unimplemented */
    3842 + mov 0, %o2 /* ARG2: mmu context == nucleus */
    3843 + mov HV_MMU_ALL, %o3 /* ARG3: flags */
    3844 + mov HV_FAST_MMU_DEMAP_CTX, %o5
    3845 + ta HV_FAST_TRAP
    3846 + mov %g1, %o3
    3847 + brz,pt %o0, 5b
    3848 + mov %g3, %o5
    3849 + mov HV_FAST_MMU_DEMAP_CTX, %g6
    3850 + ba,pt %xcc, 1b
    3851 + clr %g5
    3852
    3853 /* These just get rescheduled to PIL vectors. */
    3854 .globl xcall_call_function
    3855 @@ -809,6 +985,58 @@ xcall_kgdb_capture:
    3856
    3857 #endif /* CONFIG_SMP */
    3858
    3859 + .globl cheetah_patch_cachetlbops
    3860 +cheetah_patch_cachetlbops:
    3861 + save %sp, -128, %sp
    3862 +
    3863 + sethi %hi(__flush_tlb_mm), %o0
    3864 + or %o0, %lo(__flush_tlb_mm), %o0
    3865 + sethi %hi(__cheetah_flush_tlb_mm), %o1
    3866 + or %o1, %lo(__cheetah_flush_tlb_mm), %o1
    3867 + call tlb_patch_one
    3868 + mov 19, %o2
    3869 +
    3870 + sethi %hi(__flush_tlb_page), %o0
    3871 + or %o0, %lo(__flush_tlb_page), %o0
    3872 + sethi %hi(__cheetah_flush_tlb_page), %o1
    3873 + or %o1, %lo(__cheetah_flush_tlb_page), %o1
    3874 + call tlb_patch_one
    3875 + mov 22, %o2
    3876 +
    3877 + sethi %hi(__flush_tlb_pending), %o0
    3878 + or %o0, %lo(__flush_tlb_pending), %o0
    3879 + sethi %hi(__cheetah_flush_tlb_pending), %o1
    3880 + or %o1, %lo(__cheetah_flush_tlb_pending), %o1
    3881 + call tlb_patch_one
    3882 + mov 27, %o2
    3883 +
    3884 + sethi %hi(__flush_tlb_kernel_range), %o0
    3885 + or %o0, %lo(__flush_tlb_kernel_range), %o0
    3886 + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1
    3887 + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
    3888 + call tlb_patch_one
    3889 + mov 31, %o2
    3890 +
    3891 +#ifdef DCACHE_ALIASING_POSSIBLE
    3892 + sethi %hi(__flush_dcache_page), %o0
    3893 + or %o0, %lo(__flush_dcache_page), %o0
    3894 + sethi %hi(__cheetah_flush_dcache_page), %o1
    3895 + or %o1, %lo(__cheetah_flush_dcache_page), %o1
    3896 + call tlb_patch_one
    3897 + mov 11, %o2
    3898 +#endif /* DCACHE_ALIASING_POSSIBLE */
    3899 +
    3900 +#ifdef CONFIG_SMP
    3901 + sethi %hi(xcall_flush_tlb_kernel_range), %o0
    3902 + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
    3903 + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
    3904 + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
    3905 + call tlb_patch_one
    3906 + mov 44, %o2
    3907 +#endif /* CONFIG_SMP */
    3908 +
    3909 + ret
    3910 + restore
    3911
    3912 .globl hypervisor_patch_cachetlbops
    3913 hypervisor_patch_cachetlbops:
    3914 @@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops:
    3915 sethi %hi(__hypervisor_flush_tlb_mm), %o1
    3916 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1
    3917 call tlb_patch_one
    3918 - mov 10, %o2
    3919 + mov 19, %o2
    3920
    3921 sethi %hi(__flush_tlb_page), %o0
    3922 or %o0, %lo(__flush_tlb_page), %o0
    3923 sethi %hi(__hypervisor_flush_tlb_page), %o1
    3924 or %o1, %lo(__hypervisor_flush_tlb_page), %o1
    3925 call tlb_patch_one
    3926 - mov 11, %o2
    3927 + mov 22, %o2
    3928
    3929 sethi %hi(__flush_tlb_pending), %o0
    3930 or %o0, %lo(__flush_tlb_pending), %o0
    3931 sethi %hi(__hypervisor_flush_tlb_pending), %o1
    3932 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1
    3933 call tlb_patch_one
    3934 - mov 16, %o2
    3935 + mov 27, %o2
    3936
    3937 sethi %hi(__flush_tlb_kernel_range), %o0
    3938 or %o0, %lo(__flush_tlb_kernel_range), %o0
    3939 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
    3940 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
    3941 call tlb_patch_one
    3942 - mov 16, %o2
    3943 + mov 31, %o2
    3944
    3945 #ifdef DCACHE_ALIASING_POSSIBLE
    3946 sethi %hi(__flush_dcache_page), %o0
    3947 @@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops:
    3948 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1
    3949 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
    3950 call tlb_patch_one
    3951 - mov 21, %o2
    3952 + mov 24, %o2
    3953
    3954 sethi %hi(xcall_flush_tlb_page), %o0
    3955 or %o0, %lo(xcall_flush_tlb_page), %o0
    3956 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1
    3957 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
    3958 call tlb_patch_one
    3959 - mov 17, %o2
    3960 + mov 20, %o2
    3961
    3962 sethi %hi(xcall_flush_tlb_kernel_range), %o0
    3963 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
    3964 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
    3965 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
    3966 call tlb_patch_one
    3967 - mov 25, %o2
    3968 + mov 44, %o2
    3969 #endif /* CONFIG_SMP */
    3970
    3971 ret
    3972 diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
    3973 index c32f5d32f811..b56c9c581359 100644
    3974 --- a/drivers/net/ethernet/broadcom/bgmac.c
    3975 +++ b/drivers/net/ethernet/broadcom/bgmac.c
    3976 @@ -314,6 +314,10 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
    3977 u32 ctl;
    3978
    3979 ctl = bgmac_read(bgmac, ring->mmio_base + BGMAC_DMA_RX_CTL);
    3980 +
    3981 + /* preserve ONLY bits 16-17 from current hardware value */
    3982 + ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
    3983 +
    3984 if (bgmac->core->id.rev >= 4) {
    3985 ctl &= ~BGMAC_DMA_RX_BL_MASK;
    3986 ctl |= BGMAC_DMA_RX_BL_128 << BGMAC_DMA_RX_BL_SHIFT;
    3987 @@ -324,7 +328,6 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
    3988 ctl &= ~BGMAC_DMA_RX_PT_MASK;
    3989 ctl |= BGMAC_DMA_RX_PT_1 << BGMAC_DMA_RX_PT_SHIFT;
    3990 }
    3991 - ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
    3992 ctl |= BGMAC_DMA_RX_ENABLE;
    3993 ctl |= BGMAC_DMA_RX_PARITY_DISABLE;
    3994 ctl |= BGMAC_DMA_RX_OVERFLOW_CONT;
    3995 diff --git a/drivers/tty/serial/sunhv.c b/drivers/tty/serial/sunhv.c
    3996 index ca0d3802f2af..4e603d060e80 100644
    3997 --- a/drivers/tty/serial/sunhv.c
    3998 +++ b/drivers/tty/serial/sunhv.c
    3999 @@ -490,12 +490,6 @@ static void sunhv_console_write_bychar(struct console *con, const char *s, unsig
    4000 locked = spin_trylock_irqsave(&port->lock, flags);
    4001 else
    4002 spin_lock_irqsave(&port->lock, flags);
    4003 - if (port->sysrq) {
    4004 - locked = 0;
    4005 - } else if (oops_in_progress) {
    4006 - locked = spin_trylock(&port->lock);
    4007 - } else
    4008 - spin_lock(&port->lock);
    4009
    4010 for (i = 0; i < n; i++) {
    4011 if (*s == '\n')
    4012 diff --git a/drivers/tty/tty_ldisc.c b/drivers/tty/tty_ldisc.c
    4013 index 629e3c865072..9bee25cfa0be 100644
    4014 --- a/drivers/tty/tty_ldisc.c
    4015 +++ b/drivers/tty/tty_ldisc.c
    4016 @@ -417,6 +417,10 @@ EXPORT_SYMBOL_GPL(tty_ldisc_flush);
    4017 * they are not on hot paths so a little discipline won't do
    4018 * any harm.
    4019 *
    4020 + * The line discipline-related tty_struct fields are reset to
    4021 + * prevent the ldisc driver from re-using stale information for
    4022 + * the new ldisc instance.
    4023 + *
    4024 * Locking: takes termios_rwsem
    4025 */
    4026
    4027 @@ -425,6 +429,9 @@ static void tty_set_termios_ldisc(struct tty_struct *tty, int num)
    4028 down_write(&tty->termios_rwsem);
    4029 tty->termios.c_line = num;
    4030 up_write(&tty->termios_rwsem);
    4031 +
    4032 + tty->disc_data = NULL;
    4033 + tty->receive_room = 0;
    4034 }
    4035
    4036 /**
    4037 diff --git a/include/linux/filter.h b/include/linux/filter.h
    4038 index 5110d4211866..ccb98b459c59 100644
    4039 --- a/include/linux/filter.h
    4040 +++ b/include/linux/filter.h
    4041 @@ -421,7 +421,11 @@ static inline void bpf_prog_unlock_ro(struct bpf_prog *fp)
    4042 }
    4043 #endif /* CONFIG_DEBUG_SET_MODULE_RONX */
    4044
    4045 -int sk_filter(struct sock *sk, struct sk_buff *skb);
    4046 +int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);
    4047 +static inline int sk_filter(struct sock *sk, struct sk_buff *skb)
    4048 +{
    4049 + return sk_filter_trim_cap(sk, skb, 1);
    4050 +}
    4051
    4052 int bpf_prog_select_runtime(struct bpf_prog *fp);
    4053 void bpf_prog_free(struct bpf_prog *fp);
    4054 diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
    4055 index ff788b665277..9c2c044153f6 100644
    4056 --- a/include/net/ip6_tunnel.h
    4057 +++ b/include/net/ip6_tunnel.h
    4058 @@ -86,6 +86,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb,
    4059 struct net_device_stats *stats = &dev->stats;
    4060 int pkt_len, err;
    4061
    4062 + memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
    4063 pkt_len = skb->len - skb_inner_network_offset(skb);
    4064 err = ip6_local_out(dev_net(skb_dst(skb)->dev), sk, skb);
    4065
    4066 diff --git a/include/net/tcp.h b/include/net/tcp.h
    4067 index 9c3ab544d3a8..e9d7a8ef9a6d 100644
    4068 --- a/include/net/tcp.h
    4069 +++ b/include/net/tcp.h
    4070 @@ -1156,6 +1156,7 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp)
    4071 }
    4072
    4073 bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
    4074 +int tcp_filter(struct sock *sk, struct sk_buff *skb);
    4075
    4076 #undef STATE_TRACE
    4077
    4078 diff --git a/net/core/dev.c b/net/core/dev.c
    4079 index b3fa4b86ab4c..9ca749c81b6c 100644
    4080 --- a/net/core/dev.c
    4081 +++ b/net/core/dev.c
    4082 @@ -2462,7 +2462,7 @@ int skb_checksum_help(struct sk_buff *skb)
    4083 goto out;
    4084 }
    4085
    4086 - *(__sum16 *)(skb->data + offset) = csum_fold(csum);
    4087 + *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
    4088 out_set_summed:
    4089 skb->ip_summed = CHECKSUM_NONE;
    4090 out:
    4091 diff --git a/net/core/filter.c b/net/core/filter.c
    4092 index 75e9b2b2336d..e94355452166 100644
    4093 --- a/net/core/filter.c
    4094 +++ b/net/core/filter.c
    4095 @@ -52,9 +52,10 @@
    4096 #include <net/dst.h>
    4097
    4098 /**
    4099 - * sk_filter - run a packet through a socket filter
    4100 + * sk_filter_trim_cap - run a packet through a socket filter
    4101 * @sk: sock associated with &sk_buff
    4102 * @skb: buffer to filter
    4103 + * @cap: limit on how short the eBPF program may trim the packet
    4104 *
    4105 * Run the eBPF program and then cut skb->data to correct size returned by
    4106 * the program. If pkt_len is 0 we toss packet. If skb->len is smaller
    4107 @@ -63,7 +64,7 @@
    4108 * be accepted or -EPERM if the packet should be tossed.
    4109 *
    4110 */
    4111 -int sk_filter(struct sock *sk, struct sk_buff *skb)
    4112 +int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap)
    4113 {
    4114 int err;
    4115 struct sk_filter *filter;
    4116 @@ -84,14 +85,13 @@ int sk_filter(struct sock *sk, struct sk_buff *skb)
    4117 filter = rcu_dereference(sk->sk_filter);
    4118 if (filter) {
    4119 unsigned int pkt_len = bpf_prog_run_save_cb(filter->prog, skb);
    4120 -
    4121 - err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM;
    4122 + err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM;
    4123 }
    4124 rcu_read_unlock();
    4125
    4126 return err;
    4127 }
    4128 -EXPORT_SYMBOL(sk_filter);
    4129 +EXPORT_SYMBOL(sk_filter_trim_cap);
    4130
    4131 static u64 __skb_get_pay_offset(u64 ctx, u64 a, u64 x, u64 r4, u64 r5)
    4132 {
    4133 diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
    4134 index 4ab6ead3d8ee..9aba9e93c0a2 100644
    4135 --- a/net/core/flow_dissector.c
    4136 +++ b/net/core/flow_dissector.c
    4137 @@ -131,7 +131,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
    4138 struct flow_dissector_key_tags *key_tags;
    4139 struct flow_dissector_key_keyid *key_keyid;
    4140 u8 ip_proto = 0;
    4141 - bool ret = false;
    4142 + bool ret;
    4143
    4144 if (!data) {
    4145 data = skb->data;
    4146 @@ -492,12 +492,17 @@ ip_proto_again:
    4147 out_good:
    4148 ret = true;
    4149
    4150 -out_bad:
    4151 + key_control->thoff = (u16)nhoff;
    4152 +out:
    4153 key_basic->n_proto = proto;
    4154 key_basic->ip_proto = ip_proto;
    4155 - key_control->thoff = (u16)nhoff;
    4156
    4157 return ret;
    4158 +
    4159 +out_bad:
    4160 + ret = false;
    4161 + key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
    4162 + goto out;
    4163 }
    4164 EXPORT_SYMBOL(__skb_flow_dissect);
    4165
    4166 diff --git a/net/core/sock.c b/net/core/sock.c
    4167 index 0d91f7dca751..88f017854509 100644
    4168 --- a/net/core/sock.c
    4169 +++ b/net/core/sock.c
    4170 @@ -1562,6 +1562,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
    4171 }
    4172
    4173 newsk->sk_err = 0;
    4174 + newsk->sk_err_soft = 0;
    4175 newsk->sk_priority = 0;
    4176 newsk->sk_incoming_cpu = raw_smp_processor_id();
    4177 atomic64_set(&newsk->sk_cookie, 0);
    4178 diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
    4179 index 8be8f27bfacc..861e1fa25d5e 100644
    4180 --- a/net/dccp/ipv4.c
    4181 +++ b/net/dccp/ipv4.c
    4182 @@ -235,7 +235,7 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
    4183 {
    4184 const struct iphdr *iph = (struct iphdr *)skb->data;
    4185 const u8 offset = iph->ihl << 2;
    4186 - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
    4187 + const struct dccp_hdr *dh;
    4188 struct dccp_sock *dp;
    4189 struct inet_sock *inet;
    4190 const int type = icmp_hdr(skb)->type;
    4191 @@ -245,11 +245,13 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
    4192 int err;
    4193 struct net *net = dev_net(skb->dev);
    4194
    4195 - if (skb->len < offset + sizeof(*dh) ||
    4196 - skb->len < offset + __dccp_basic_hdr_len(dh)) {
    4197 - ICMP_INC_STATS_BH(net, ICMP_MIB_INERRORS);
    4198 - return;
    4199 - }
    4200 + /* Only need dccph_dport & dccph_sport which are the first
    4201 + * 4 bytes in dccp header.
    4202 + * Our caller (icmp_socket_deliver()) already pulled 8 bytes for us.
    4203 + */
    4204 + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
    4205 + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
    4206 + dh = (struct dccp_hdr *)(skb->data + offset);
    4207
    4208 sk = __inet_lookup_established(net, &dccp_hashinfo,
    4209 iph->daddr, dh->dccph_dport,
    4210 diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
    4211 index b8608b71a66d..27c4e81efa24 100644
    4212 --- a/net/dccp/ipv6.c
    4213 +++ b/net/dccp/ipv6.c
    4214 @@ -70,7 +70,7 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
    4215 u8 type, u8 code, int offset, __be32 info)
    4216 {
    4217 const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
    4218 - const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
    4219 + const struct dccp_hdr *dh;
    4220 struct dccp_sock *dp;
    4221 struct ipv6_pinfo *np;
    4222 struct sock *sk;
    4223 @@ -78,12 +78,13 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
    4224 __u64 seq;
    4225 struct net *net = dev_net(skb->dev);
    4226
    4227 - if (skb->len < offset + sizeof(*dh) ||
    4228 - skb->len < offset + __dccp_basic_hdr_len(dh)) {
    4229 - ICMP6_INC_STATS_BH(net, __in6_dev_get(skb->dev),
    4230 - ICMP6_MIB_INERRORS);
    4231 - return;
    4232 - }
    4233 + /* Only need dccph_dport & dccph_sport which are the first
    4234 + * 4 bytes in dccp header.
    4235 + * Our caller (icmpv6_notify()) already pulled 8 bytes for us.
    4236 + */
    4237 + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
    4238 + BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
    4239 + dh = (struct dccp_hdr *)(skb->data + offset);
    4240
    4241 sk = __inet6_lookup_established(net, &dccp_hashinfo,
    4242 &hdr->daddr, dh->dccph_dport,
    4243 @@ -947,6 +948,7 @@ static const struct inet_connection_sock_af_ops dccp_ipv6_mapped = {
    4244 .getsockopt = ipv6_getsockopt,
    4245 .addr2sockaddr = inet6_csk_addr2sockaddr,
    4246 .sockaddr_len = sizeof(struct sockaddr_in6),
    4247 + .bind_conflict = inet6_csk_bind_conflict,
    4248 #ifdef CONFIG_COMPAT
    4249 .compat_setsockopt = compat_ipv6_setsockopt,
    4250 .compat_getsockopt = compat_ipv6_getsockopt,
    4251 diff --git a/net/dccp/proto.c b/net/dccp/proto.c
    4252 index 41e65804ddf5..9fe25bf63296 100644
    4253 --- a/net/dccp/proto.c
    4254 +++ b/net/dccp/proto.c
    4255 @@ -1009,6 +1009,10 @@ void dccp_close(struct sock *sk, long timeout)
    4256 __kfree_skb(skb);
    4257 }
    4258
    4259 + /* If socket has been already reset kill it. */
    4260 + if (sk->sk_state == DCCP_CLOSED)
    4261 + goto adjudge_to_death;
    4262 +
    4263 if (data_was_unread) {
    4264 /* Unread data was tossed, send an appropriate Reset Code */
    4265 DCCP_WARN("ABORT with %u bytes unread\n", data_was_unread);
    4266 diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
    4267 index e5a3ff210fec..7c52afb98c42 100644
    4268 --- a/net/ipv4/fib_trie.c
    4269 +++ b/net/ipv4/fib_trie.c
    4270 @@ -2456,22 +2456,19 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
    4271 struct key_vector *l, **tp = &iter->tnode;
    4272 t_key key;
    4273
    4274 - /* use cache location of next-to-find key */
    4275 + /* use cached location of previously found key */
    4276 if (iter->pos > 0 && pos >= iter->pos) {
    4277 - pos -= iter->pos;
    4278 key = iter->key;
    4279 } else {
    4280 - iter->pos = 0;
    4281 + iter->pos = 1;
    4282 key = 0;
    4283 }
    4284
    4285 - while ((l = leaf_walk_rcu(tp, key)) != NULL) {
    4286 + pos -= iter->pos;
    4287 +
    4288 + while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) {
    4289 key = l->key + 1;
    4290 iter->pos++;
    4291 -
    4292 - if (--pos <= 0)
    4293 - break;
    4294 -
    4295 l = NULL;
    4296
    4297 /* handle unlikely case of a key wrap */
    4298 @@ -2480,7 +2477,7 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
    4299 }
    4300
    4301 if (l)
    4302 - iter->key = key; /* remember it */
    4303 + iter->key = l->key; /* remember it */
    4304 else
    4305 iter->pos = 0; /* forget it */
    4306
    4307 @@ -2508,7 +2505,7 @@ static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos)
    4308 return fib_route_get_idx(iter, *pos);
    4309
    4310 iter->pos = 0;
    4311 - iter->key = 0;
    4312 + iter->key = KEY_MAX;
    4313
    4314 return SEQ_START_TOKEN;
    4315 }
    4316 @@ -2517,7 +2514,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
    4317 {
    4318 struct fib_route_iter *iter = seq->private;
    4319 struct key_vector *l = NULL;
    4320 - t_key key = iter->key;
    4321 + t_key key = iter->key + 1;
    4322
    4323 ++*pos;
    4324
    4325 @@ -2526,7 +2523,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
    4326 l = leaf_walk_rcu(&iter->tnode, key);
    4327
    4328 if (l) {
    4329 - iter->key = l->key + 1;
    4330 + iter->key = l->key;
    4331 iter->pos++;
    4332 } else {
    4333 iter->pos = 0;
    4334 diff --git a/net/ipv4/route.c b/net/ipv4/route.c
    4335 index 8533a75a9328..7ceb8a574a50 100644
    4336 --- a/net/ipv4/route.c
    4337 +++ b/net/ipv4/route.c
    4338 @@ -747,7 +747,9 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
    4339 goto reject_redirect;
    4340 }
    4341
    4342 - n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
    4343 + n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
    4344 + if (!n)
    4345 + n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
    4346 if (!IS_ERR(n)) {
    4347 if (!(n->nud_state & NUD_VALID)) {
    4348 neigh_event_send(n, NULL);
    4349 diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
    4350 index 036a76ba2ac2..69daa81736f6 100644
    4351 --- a/net/ipv4/tcp.c
    4352 +++ b/net/ipv4/tcp.c
    4353 @@ -1212,7 +1212,7 @@ new_segment:
    4354
    4355 if (!skb_can_coalesce(skb, i, pfrag->page,
    4356 pfrag->offset)) {
    4357 - if (i == sysctl_max_skb_frags || !sg) {
    4358 + if (i >= sysctl_max_skb_frags || !sg) {
    4359 tcp_mark_push(tp, skb);
    4360 goto new_segment;
    4361 }
    4362 diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c
    4363 index 7e538f71f5fb..55d7da1d2ce9 100644
    4364 --- a/net/ipv4/tcp_dctcp.c
    4365 +++ b/net/ipv4/tcp_dctcp.c
    4366 @@ -56,6 +56,7 @@ struct dctcp {
    4367 u32 next_seq;
    4368 u32 ce_state;
    4369 u32 delayed_ack_reserved;
    4370 + u32 loss_cwnd;
    4371 };
    4372
    4373 static unsigned int dctcp_shift_g __read_mostly = 4; /* g = 1/2^4 */
    4374 @@ -96,6 +97,7 @@ static void dctcp_init(struct sock *sk)
    4375 ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
    4376
    4377 ca->delayed_ack_reserved = 0;
    4378 + ca->loss_cwnd = 0;
    4379 ca->ce_state = 0;
    4380
    4381 dctcp_reset(tp, ca);
    4382 @@ -111,9 +113,10 @@ static void dctcp_init(struct sock *sk)
    4383
    4384 static u32 dctcp_ssthresh(struct sock *sk)
    4385 {
    4386 - const struct dctcp *ca = inet_csk_ca(sk);
    4387 + struct dctcp *ca = inet_csk_ca(sk);
    4388 struct tcp_sock *tp = tcp_sk(sk);
    4389
    4390 + ca->loss_cwnd = tp->snd_cwnd;
    4391 return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
    4392 }
    4393
    4394 @@ -308,12 +311,20 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr,
    4395 return 0;
    4396 }
    4397
    4398 +static u32 dctcp_cwnd_undo(struct sock *sk)
    4399 +{
    4400 + const struct dctcp *ca = inet_csk_ca(sk);
    4401 +
    4402 + return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
    4403 +}
    4404 +
    4405 static struct tcp_congestion_ops dctcp __read_mostly = {
    4406 .init = dctcp_init,
    4407 .in_ack_event = dctcp_update_alpha,
    4408 .cwnd_event = dctcp_cwnd_event,
    4409 .ssthresh = dctcp_ssthresh,
    4410 .cong_avoid = tcp_reno_cong_avoid,
    4411 + .undo_cwnd = dctcp_cwnd_undo,
    4412 .set_state = dctcp_state,
    4413 .get_info = dctcp_get_info,
    4414 .flags = TCP_CONG_NEEDS_ECN,
    4415 diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
    4416 index b5853cac3269..b58a38eea059 100644
    4417 --- a/net/ipv4/tcp_ipv4.c
    4418 +++ b/net/ipv4/tcp_ipv4.c
    4419 @@ -1533,6 +1533,21 @@ bool tcp_prequeue(struct sock *sk, struct sk_buff *skb)
    4420 }
    4421 EXPORT_SYMBOL(tcp_prequeue);
    4422
    4423 +int tcp_filter(struct sock *sk, struct sk_buff *skb)
    4424 +{
    4425 + struct tcphdr *th = (struct tcphdr *)skb->data;
    4426 + unsigned int eaten = skb->len;
    4427 + int err;
    4428 +
    4429 + err = sk_filter_trim_cap(sk, skb, th->doff * 4);
    4430 + if (!err) {
    4431 + eaten -= skb->len;
    4432 + TCP_SKB_CB(skb)->end_seq -= eaten;
    4433 + }
    4434 + return err;
    4435 +}
    4436 +EXPORT_SYMBOL(tcp_filter);
    4437 +
    4438 /*
    4439 * From tcp_input.c
    4440 */
    4441 @@ -1638,8 +1653,10 @@ process:
    4442
    4443 nf_reset(skb);
    4444
    4445 - if (sk_filter(sk, skb))
    4446 + if (tcp_filter(sk, skb))
    4447 goto discard_and_relse;
    4448 + th = (const struct tcphdr *)skb->data;
    4449 + iph = ip_hdr(skb);
    4450
    4451 skb->dev = NULL;
    4452
    4453 diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
    4454 index fbd521fdae53..5f581616bf6a 100644
    4455 --- a/net/ipv6/tcp_ipv6.c
    4456 +++ b/net/ipv6/tcp_ipv6.c
    4457 @@ -1214,7 +1214,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
    4458 if (skb->protocol == htons(ETH_P_IP))
    4459 return tcp_v4_do_rcv(sk, skb);
    4460
    4461 - if (sk_filter(sk, skb))
    4462 + if (tcp_filter(sk, skb))
    4463 goto discard;
    4464
    4465 /*
    4466 @@ -1438,8 +1438,10 @@ process:
    4467 if (tcp_v6_inbound_md5_hash(sk, skb))
    4468 goto discard_and_relse;
    4469
    4470 - if (sk_filter(sk, skb))
    4471 + if (tcp_filter(sk, skb))
    4472 goto discard_and_relse;
    4473 + th = (const struct tcphdr *)skb->data;
    4474 + hdr = ipv6_hdr(skb);
    4475
    4476 skb->dev = NULL;
    4477
    4478 diff --git a/net/sctp/socket.c b/net/sctp/socket.c
    4479 index 402817be3873..b5fd4ab56156 100644
    4480 --- a/net/sctp/socket.c
    4481 +++ b/net/sctp/socket.c
    4482 @@ -1212,9 +1212,12 @@ static int __sctp_connect(struct sock *sk,
    4483
    4484 timeo = sock_sndtimeo(sk, f_flags & O_NONBLOCK);
    4485
    4486 - err = sctp_wait_for_connect(asoc, &timeo);
    4487 - if ((err == 0 || err == -EINPROGRESS) && assoc_id)
    4488 + if (assoc_id)
    4489 *assoc_id = asoc->assoc_id;
    4490 + err = sctp_wait_for_connect(asoc, &timeo);
    4491 + /* Note: the asoc may be freed after the return of
    4492 + * sctp_wait_for_connect.
    4493 + */
    4494
    4495 /* Don't free association on exit. */
    4496 asoc = NULL;
    4497 diff --git a/net/socket.c b/net/socket.c
    4498 index 263b334ec5e4..0090225eeb1e 100644
    4499 --- a/net/socket.c
    4500 +++ b/net/socket.c
    4501 @@ -2041,6 +2041,8 @@ int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen,
    4502 if (err)
    4503 break;
    4504 ++datagrams;
    4505 + if (msg_data_left(&msg_sys))
    4506 + break;
    4507 }
    4508
    4509 fput_light(sock->file, fput_needed);