Annotation of /trunk/kernel-alx/patches-5.4/0303-5.4.204-all-fixes.patch
Parent Directory | Revision Log
Revision 3637 -
(hide annotations)
(download)
Mon Oct 24 12:40:44 2022 UTC (23 months ago) by niro
File size: 154160 byte(s)
Mon Oct 24 12:40:44 2022 UTC (23 months ago) by niro
File size: 154160 byte(s)
-add missing
1 | niro | 3637 | diff --git a/Makefile b/Makefile |
2 | index 86f518e49c333..b5d1718ae70c6 100644 | ||
3 | --- a/Makefile | ||
4 | +++ b/Makefile | ||
5 | @@ -1,7 +1,7 @@ | ||
6 | # SPDX-License-Identifier: GPL-2.0 | ||
7 | VERSION = 5 | ||
8 | PATCHLEVEL = 4 | ||
9 | -SUBLEVEL = 203 | ||
10 | +SUBLEVEL = 204 | ||
11 | EXTRAVERSION = | ||
12 | NAME = Kleptomaniac Octopus | ||
13 | |||
14 | diff --git a/arch/arm/xen/p2m.c b/arch/arm/xen/p2m.c | ||
15 | index acb464547a54f..4a1991a103ea0 100644 | ||
16 | --- a/arch/arm/xen/p2m.c | ||
17 | +++ b/arch/arm/xen/p2m.c | ||
18 | @@ -62,11 +62,12 @@ out: | ||
19 | |||
20 | unsigned long __pfn_to_mfn(unsigned long pfn) | ||
21 | { | ||
22 | - struct rb_node *n = phys_to_mach.rb_node; | ||
23 | + struct rb_node *n; | ||
24 | struct xen_p2m_entry *entry; | ||
25 | unsigned long irqflags; | ||
26 | |||
27 | read_lock_irqsave(&p2m_lock, irqflags); | ||
28 | + n = phys_to_mach.rb_node; | ||
29 | while (n) { | ||
30 | entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); | ||
31 | if (entry->pfn <= pfn && | ||
32 | @@ -153,10 +154,11 @@ bool __set_phys_to_machine_multi(unsigned long pfn, | ||
33 | int rc; | ||
34 | unsigned long irqflags; | ||
35 | struct xen_p2m_entry *p2m_entry; | ||
36 | - struct rb_node *n = phys_to_mach.rb_node; | ||
37 | + struct rb_node *n; | ||
38 | |||
39 | if (mfn == INVALID_P2M_ENTRY) { | ||
40 | write_lock_irqsave(&p2m_lock, irqflags); | ||
41 | + n = phys_to_mach.rb_node; | ||
42 | while (n) { | ||
43 | p2m_entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); | ||
44 | if (p2m_entry->pfn <= pfn && | ||
45 | diff --git a/arch/powerpc/include/asm/bpf_perf_event.h b/arch/powerpc/include/asm/bpf_perf_event.h | ||
46 | new file mode 100644 | ||
47 | index 0000000000000..e8a7b4ffb58c2 | ||
48 | --- /dev/null | ||
49 | +++ b/arch/powerpc/include/asm/bpf_perf_event.h | ||
50 | @@ -0,0 +1,9 @@ | ||
51 | +/* SPDX-License-Identifier: GPL-2.0 */ | ||
52 | +#ifndef _ASM_POWERPC_BPF_PERF_EVENT_H | ||
53 | +#define _ASM_POWERPC_BPF_PERF_EVENT_H | ||
54 | + | ||
55 | +#include <asm/ptrace.h> | ||
56 | + | ||
57 | +typedef struct user_pt_regs bpf_user_pt_regs_t; | ||
58 | + | ||
59 | +#endif /* _ASM_POWERPC_BPF_PERF_EVENT_H */ | ||
60 | diff --git a/arch/powerpc/include/uapi/asm/bpf_perf_event.h b/arch/powerpc/include/uapi/asm/bpf_perf_event.h | ||
61 | deleted file mode 100644 | ||
62 | index 5e1e648aeec4c..0000000000000 | ||
63 | --- a/arch/powerpc/include/uapi/asm/bpf_perf_event.h | ||
64 | +++ /dev/null | ||
65 | @@ -1,9 +0,0 @@ | ||
66 | -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ | ||
67 | -#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ | ||
68 | -#define _UAPI__ASM_BPF_PERF_EVENT_H__ | ||
69 | - | ||
70 | -#include <asm/ptrace.h> | ||
71 | - | ||
72 | -typedef struct user_pt_regs bpf_user_pt_regs_t; | ||
73 | - | ||
74 | -#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ | ||
75 | diff --git a/arch/powerpc/kernel/prom_init_check.sh b/arch/powerpc/kernel/prom_init_check.sh | ||
76 | index b183ab9c5107c..dfa5f729f774d 100644 | ||
77 | --- a/arch/powerpc/kernel/prom_init_check.sh | ||
78 | +++ b/arch/powerpc/kernel/prom_init_check.sh | ||
79 | @@ -13,7 +13,7 @@ | ||
80 | # If you really need to reference something from prom_init.o add | ||
81 | # it to the list below: | ||
82 | |||
83 | -grep "^CONFIG_KASAN=y$" .config >/dev/null | ||
84 | +grep "^CONFIG_KASAN=y$" ${KCONFIG_CONFIG} >/dev/null | ||
85 | if [ $? -eq 0 ] | ||
86 | then | ||
87 | MEM_FUNCS="__memcpy __memset" | ||
88 | diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig | ||
89 | index 0023b78391f14..d7c7c7b80aed1 100644 | ||
90 | --- a/arch/s390/Kconfig | ||
91 | +++ b/arch/s390/Kconfig | ||
92 | @@ -541,7 +541,6 @@ config KEXEC | ||
93 | config KEXEC_FILE | ||
94 | bool "kexec file based system call" | ||
95 | select KEXEC_CORE | ||
96 | - select BUILD_BIN2C | ||
97 | depends on CRYPTO | ||
98 | depends on CRYPTO_SHA256 | ||
99 | depends on CRYPTO_SHA256_S390 | ||
100 | diff --git a/arch/s390/crypto/arch_random.c b/arch/s390/crypto/arch_random.c | ||
101 | index 4cbb4b6d85a83..1f2d40993c4d2 100644 | ||
102 | --- a/arch/s390/crypto/arch_random.c | ||
103 | +++ b/arch/s390/crypto/arch_random.c | ||
104 | @@ -2,126 +2,17 @@ | ||
105 | /* | ||
106 | * s390 arch random implementation. | ||
107 | * | ||
108 | - * Copyright IBM Corp. 2017, 2018 | ||
109 | + * Copyright IBM Corp. 2017, 2020 | ||
110 | * Author(s): Harald Freudenberger | ||
111 | - * | ||
112 | - * The s390_arch_random_generate() function may be called from random.c | ||
113 | - * in interrupt context. So this implementation does the best to be very | ||
114 | - * fast. There is a buffer of random data which is asynchronously checked | ||
115 | - * and filled by a workqueue thread. | ||
116 | - * If there are enough bytes in the buffer the s390_arch_random_generate() | ||
117 | - * just delivers these bytes. Otherwise false is returned until the | ||
118 | - * worker thread refills the buffer. | ||
119 | - * The worker fills the rng buffer by pulling fresh entropy from the | ||
120 | - * high quality (but slow) true hardware random generator. This entropy | ||
121 | - * is then spread over the buffer with an pseudo random generator PRNG. | ||
122 | - * As the arch_get_random_seed_long() fetches 8 bytes and the calling | ||
123 | - * function add_interrupt_randomness() counts this as 1 bit entropy the | ||
124 | - * distribution needs to make sure there is in fact 1 bit entropy contained | ||
125 | - * in 8 bytes of the buffer. The current values pull 32 byte entropy | ||
126 | - * and scatter this into a 2048 byte buffer. So 8 byte in the buffer | ||
127 | - * will contain 1 bit of entropy. | ||
128 | - * The worker thread is rescheduled based on the charge level of the | ||
129 | - * buffer but at least with 500 ms delay to avoid too much CPU consumption. | ||
130 | - * So the max. amount of rng data delivered via arch_get_random_seed is | ||
131 | - * limited to 4k bytes per second. | ||
132 | */ | ||
133 | |||
134 | #include <linux/kernel.h> | ||
135 | #include <linux/atomic.h> | ||
136 | #include <linux/random.h> | ||
137 | -#include <linux/slab.h> | ||
138 | #include <linux/static_key.h> | ||
139 | -#include <linux/workqueue.h> | ||
140 | #include <asm/cpacf.h> | ||
141 | |||
142 | DEFINE_STATIC_KEY_FALSE(s390_arch_random_available); | ||
143 | |||
144 | atomic64_t s390_arch_random_counter = ATOMIC64_INIT(0); | ||
145 | EXPORT_SYMBOL(s390_arch_random_counter); | ||
146 | - | ||
147 | -#define ARCH_REFILL_TICKS (HZ/2) | ||
148 | -#define ARCH_PRNG_SEED_SIZE 32 | ||
149 | -#define ARCH_RNG_BUF_SIZE 2048 | ||
150 | - | ||
151 | -static DEFINE_SPINLOCK(arch_rng_lock); | ||
152 | -static u8 *arch_rng_buf; | ||
153 | -static unsigned int arch_rng_buf_idx; | ||
154 | - | ||
155 | -static void arch_rng_refill_buffer(struct work_struct *); | ||
156 | -static DECLARE_DELAYED_WORK(arch_rng_work, arch_rng_refill_buffer); | ||
157 | - | ||
158 | -bool s390_arch_random_generate(u8 *buf, unsigned int nbytes) | ||
159 | -{ | ||
160 | - /* max hunk is ARCH_RNG_BUF_SIZE */ | ||
161 | - if (nbytes > ARCH_RNG_BUF_SIZE) | ||
162 | - return false; | ||
163 | - | ||
164 | - /* lock rng buffer */ | ||
165 | - if (!spin_trylock(&arch_rng_lock)) | ||
166 | - return false; | ||
167 | - | ||
168 | - /* try to resolve the requested amount of bytes from the buffer */ | ||
169 | - arch_rng_buf_idx -= nbytes; | ||
170 | - if (arch_rng_buf_idx < ARCH_RNG_BUF_SIZE) { | ||
171 | - memcpy(buf, arch_rng_buf + arch_rng_buf_idx, nbytes); | ||
172 | - atomic64_add(nbytes, &s390_arch_random_counter); | ||
173 | - spin_unlock(&arch_rng_lock); | ||
174 | - return true; | ||
175 | - } | ||
176 | - | ||
177 | - /* not enough bytes in rng buffer, refill is done asynchronously */ | ||
178 | - spin_unlock(&arch_rng_lock); | ||
179 | - | ||
180 | - return false; | ||
181 | -} | ||
182 | -EXPORT_SYMBOL(s390_arch_random_generate); | ||
183 | - | ||
184 | -static void arch_rng_refill_buffer(struct work_struct *unused) | ||
185 | -{ | ||
186 | - unsigned int delay = ARCH_REFILL_TICKS; | ||
187 | - | ||
188 | - spin_lock(&arch_rng_lock); | ||
189 | - if (arch_rng_buf_idx > ARCH_RNG_BUF_SIZE) { | ||
190 | - /* buffer is exhausted and needs refill */ | ||
191 | - u8 seed[ARCH_PRNG_SEED_SIZE]; | ||
192 | - u8 prng_wa[240]; | ||
193 | - /* fetch ARCH_PRNG_SEED_SIZE bytes of entropy */ | ||
194 | - cpacf_trng(NULL, 0, seed, sizeof(seed)); | ||
195 | - /* blow this entropy up to ARCH_RNG_BUF_SIZE with PRNG */ | ||
196 | - memset(prng_wa, 0, sizeof(prng_wa)); | ||
197 | - cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED, | ||
198 | - &prng_wa, NULL, 0, seed, sizeof(seed)); | ||
199 | - cpacf_prno(CPACF_PRNO_SHA512_DRNG_GEN, | ||
200 | - &prng_wa, arch_rng_buf, ARCH_RNG_BUF_SIZE, NULL, 0); | ||
201 | - arch_rng_buf_idx = ARCH_RNG_BUF_SIZE; | ||
202 | - } | ||
203 | - delay += (ARCH_REFILL_TICKS * arch_rng_buf_idx) / ARCH_RNG_BUF_SIZE; | ||
204 | - spin_unlock(&arch_rng_lock); | ||
205 | - | ||
206 | - /* kick next check */ | ||
207 | - queue_delayed_work(system_long_wq, &arch_rng_work, delay); | ||
208 | -} | ||
209 | - | ||
210 | -static int __init s390_arch_random_init(void) | ||
211 | -{ | ||
212 | - /* all the needed PRNO subfunctions available ? */ | ||
213 | - if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG) && | ||
214 | - cpacf_query_func(CPACF_PRNO, CPACF_PRNO_SHA512_DRNG_GEN)) { | ||
215 | - | ||
216 | - /* alloc arch random working buffer */ | ||
217 | - arch_rng_buf = kmalloc(ARCH_RNG_BUF_SIZE, GFP_KERNEL); | ||
218 | - if (!arch_rng_buf) | ||
219 | - return -ENOMEM; | ||
220 | - | ||
221 | - /* kick worker queue job to fill the random buffer */ | ||
222 | - queue_delayed_work(system_long_wq, | ||
223 | - &arch_rng_work, ARCH_REFILL_TICKS); | ||
224 | - | ||
225 | - /* enable arch random to the outside world */ | ||
226 | - static_branch_enable(&s390_arch_random_available); | ||
227 | - } | ||
228 | - | ||
229 | - return 0; | ||
230 | -} | ||
231 | -arch_initcall(s390_arch_random_init); | ||
232 | diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h | ||
233 | index 9a6835137a164..2c6e1c6ecbe78 100644 | ||
234 | --- a/arch/s390/include/asm/archrandom.h | ||
235 | +++ b/arch/s390/include/asm/archrandom.h | ||
236 | @@ -2,7 +2,7 @@ | ||
237 | /* | ||
238 | * Kernel interface for the s390 arch_random_* functions | ||
239 | * | ||
240 | - * Copyright IBM Corp. 2017 | ||
241 | + * Copyright IBM Corp. 2017, 2020 | ||
242 | * | ||
243 | * Author: Harald Freudenberger <freude@de.ibm.com> | ||
244 | * | ||
245 | @@ -15,34 +15,37 @@ | ||
246 | |||
247 | #include <linux/static_key.h> | ||
248 | #include <linux/atomic.h> | ||
249 | +#include <asm/cpacf.h> | ||
250 | |||
251 | DECLARE_STATIC_KEY_FALSE(s390_arch_random_available); | ||
252 | extern atomic64_t s390_arch_random_counter; | ||
253 | |||
254 | -bool s390_arch_random_generate(u8 *buf, unsigned int nbytes); | ||
255 | - | ||
256 | -static inline bool arch_get_random_long(unsigned long *v) | ||
257 | +static inline bool __must_check arch_get_random_long(unsigned long *v) | ||
258 | { | ||
259 | return false; | ||
260 | } | ||
261 | |||
262 | -static inline bool arch_get_random_int(unsigned int *v) | ||
263 | +static inline bool __must_check arch_get_random_int(unsigned int *v) | ||
264 | { | ||
265 | return false; | ||
266 | } | ||
267 | |||
268 | -static inline bool arch_get_random_seed_long(unsigned long *v) | ||
269 | +static inline bool __must_check arch_get_random_seed_long(unsigned long *v) | ||
270 | { | ||
271 | if (static_branch_likely(&s390_arch_random_available)) { | ||
272 | - return s390_arch_random_generate((u8 *)v, sizeof(*v)); | ||
273 | + cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v)); | ||
274 | + atomic64_add(sizeof(*v), &s390_arch_random_counter); | ||
275 | + return true; | ||
276 | } | ||
277 | return false; | ||
278 | } | ||
279 | |||
280 | -static inline bool arch_get_random_seed_int(unsigned int *v) | ||
281 | +static inline bool __must_check arch_get_random_seed_int(unsigned int *v) | ||
282 | { | ||
283 | if (static_branch_likely(&s390_arch_random_available)) { | ||
284 | - return s390_arch_random_generate((u8 *)v, sizeof(*v)); | ||
285 | + cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v)); | ||
286 | + atomic64_add(sizeof(*v), &s390_arch_random_counter); | ||
287 | + return true; | ||
288 | } | ||
289 | return false; | ||
290 | } | ||
291 | diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c | ||
292 | index 9a0316a067a11..dbc2a718d232d 100644 | ||
293 | --- a/arch/s390/kernel/setup.c | ||
294 | +++ b/arch/s390/kernel/setup.c | ||
295 | @@ -1005,6 +1005,11 @@ static void __init setup_randomness(void) | ||
296 | if (stsi(vmms, 3, 2, 2) == 0 && vmms->count) | ||
297 | add_device_randomness(&vmms->vm, sizeof(vmms->vm[0]) * vmms->count); | ||
298 | memblock_free((unsigned long) vmms, PAGE_SIZE); | ||
299 | + | ||
300 | +#ifdef CONFIG_ARCH_RANDOM | ||
301 | + if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG)) | ||
302 | + static_branch_enable(&s390_arch_random_available); | ||
303 | +#endif | ||
304 | } | ||
305 | |||
306 | /* | ||
307 | diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c | ||
308 | index 3731066f2c1ca..d0538c03f0332 100644 | ||
309 | --- a/drivers/block/xen-blkfront.c | ||
310 | +++ b/drivers/block/xen-blkfront.c | ||
311 | @@ -151,6 +151,10 @@ static unsigned int xen_blkif_max_ring_order; | ||
312 | module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444); | ||
313 | MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring"); | ||
314 | |||
315 | +static bool __read_mostly xen_blkif_trusted = true; | ||
316 | +module_param_named(trusted, xen_blkif_trusted, bool, 0644); | ||
317 | +MODULE_PARM_DESC(trusted, "Is the backend trusted"); | ||
318 | + | ||
319 | #define BLK_RING_SIZE(info) \ | ||
320 | __CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages) | ||
321 | |||
322 | @@ -211,6 +215,7 @@ struct blkfront_info | ||
323 | unsigned int feature_discard:1; | ||
324 | unsigned int feature_secdiscard:1; | ||
325 | unsigned int feature_persistent:1; | ||
326 | + unsigned int bounce:1; | ||
327 | unsigned int discard_granularity; | ||
328 | unsigned int discard_alignment; | ||
329 | /* Number of 4KB segments handled */ | ||
330 | @@ -300,8 +305,8 @@ static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num) | ||
331 | if (!gnt_list_entry) | ||
332 | goto out_of_memory; | ||
333 | |||
334 | - if (info->feature_persistent) { | ||
335 | - granted_page = alloc_page(GFP_NOIO); | ||
336 | + if (info->bounce) { | ||
337 | + granted_page = alloc_page(GFP_NOIO | __GFP_ZERO); | ||
338 | if (!granted_page) { | ||
339 | kfree(gnt_list_entry); | ||
340 | goto out_of_memory; | ||
341 | @@ -320,7 +325,7 @@ out_of_memory: | ||
342 | list_for_each_entry_safe(gnt_list_entry, n, | ||
343 | &rinfo->grants, node) { | ||
344 | list_del(&gnt_list_entry->node); | ||
345 | - if (info->feature_persistent) | ||
346 | + if (info->bounce) | ||
347 | __free_page(gnt_list_entry->page); | ||
348 | kfree(gnt_list_entry); | ||
349 | i--; | ||
350 | @@ -366,7 +371,7 @@ static struct grant *get_grant(grant_ref_t *gref_head, | ||
351 | /* Assign a gref to this page */ | ||
352 | gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head); | ||
353 | BUG_ON(gnt_list_entry->gref == -ENOSPC); | ||
354 | - if (info->feature_persistent) | ||
355 | + if (info->bounce) | ||
356 | grant_foreign_access(gnt_list_entry, info); | ||
357 | else { | ||
358 | /* Grant access to the GFN passed by the caller */ | ||
359 | @@ -390,7 +395,7 @@ static struct grant *get_indirect_grant(grant_ref_t *gref_head, | ||
360 | /* Assign a gref to this page */ | ||
361 | gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head); | ||
362 | BUG_ON(gnt_list_entry->gref == -ENOSPC); | ||
363 | - if (!info->feature_persistent) { | ||
364 | + if (!info->bounce) { | ||
365 | struct page *indirect_page; | ||
366 | |||
367 | /* Fetch a pre-allocated page to use for indirect grefs */ | ||
368 | @@ -705,7 +710,7 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri | ||
369 | .grant_idx = 0, | ||
370 | .segments = NULL, | ||
371 | .rinfo = rinfo, | ||
372 | - .need_copy = rq_data_dir(req) && info->feature_persistent, | ||
373 | + .need_copy = rq_data_dir(req) && info->bounce, | ||
374 | }; | ||
375 | |||
376 | /* | ||
377 | @@ -1026,11 +1031,12 @@ static void xlvbd_flush(struct blkfront_info *info) | ||
378 | { | ||
379 | blk_queue_write_cache(info->rq, info->feature_flush ? true : false, | ||
380 | info->feature_fua ? true : false); | ||
381 | - pr_info("blkfront: %s: %s %s %s %s %s\n", | ||
382 | + pr_info("blkfront: %s: %s %s %s %s %s %s %s\n", | ||
383 | info->gd->disk_name, flush_info(info), | ||
384 | "persistent grants:", info->feature_persistent ? | ||
385 | "enabled;" : "disabled;", "indirect descriptors:", | ||
386 | - info->max_indirect_segments ? "enabled;" : "disabled;"); | ||
387 | + info->max_indirect_segments ? "enabled;" : "disabled;", | ||
388 | + "bounce buffer:", info->bounce ? "enabled" : "disabled;"); | ||
389 | } | ||
390 | |||
391 | static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset) | ||
392 | @@ -1265,7 +1271,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo) | ||
393 | if (!list_empty(&rinfo->indirect_pages)) { | ||
394 | struct page *indirect_page, *n; | ||
395 | |||
396 | - BUG_ON(info->feature_persistent); | ||
397 | + BUG_ON(info->bounce); | ||
398 | list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) { | ||
399 | list_del(&indirect_page->lru); | ||
400 | __free_page(indirect_page); | ||
401 | @@ -1282,7 +1288,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo) | ||
402 | 0, 0UL); | ||
403 | rinfo->persistent_gnts_c--; | ||
404 | } | ||
405 | - if (info->feature_persistent) | ||
406 | + if (info->bounce) | ||
407 | __free_page(persistent_gnt->page); | ||
408 | kfree(persistent_gnt); | ||
409 | } | ||
410 | @@ -1303,7 +1309,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo) | ||
411 | for (j = 0; j < segs; j++) { | ||
412 | persistent_gnt = rinfo->shadow[i].grants_used[j]; | ||
413 | gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL); | ||
414 | - if (info->feature_persistent) | ||
415 | + if (info->bounce) | ||
416 | __free_page(persistent_gnt->page); | ||
417 | kfree(persistent_gnt); | ||
418 | } | ||
419 | @@ -1493,7 +1499,7 @@ static int blkif_completion(unsigned long *id, | ||
420 | data.s = s; | ||
421 | num_sg = s->num_sg; | ||
422 | |||
423 | - if (bret->operation == BLKIF_OP_READ && info->feature_persistent) { | ||
424 | + if (bret->operation == BLKIF_OP_READ && info->bounce) { | ||
425 | for_each_sg(s->sg, sg, num_sg, i) { | ||
426 | BUG_ON(sg->offset + sg->length > PAGE_SIZE); | ||
427 | |||
428 | @@ -1552,7 +1558,7 @@ static int blkif_completion(unsigned long *id, | ||
429 | * Add the used indirect page back to the list of | ||
430 | * available pages for indirect grefs. | ||
431 | */ | ||
432 | - if (!info->feature_persistent) { | ||
433 | + if (!info->bounce) { | ||
434 | indirect_page = s->indirect_grants[i]->page; | ||
435 | list_add(&indirect_page->lru, &rinfo->indirect_pages); | ||
436 | } | ||
437 | @@ -1744,7 +1750,7 @@ static int setup_blkring(struct xenbus_device *dev, | ||
438 | for (i = 0; i < info->nr_ring_pages; i++) | ||
439 | rinfo->ring_ref[i] = GRANT_INVALID_REF; | ||
440 | |||
441 | - sring = alloc_pages_exact(ring_size, GFP_NOIO); | ||
442 | + sring = alloc_pages_exact(ring_size, GFP_NOIO | __GFP_ZERO); | ||
443 | if (!sring) { | ||
444 | xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring"); | ||
445 | return -ENOMEM; | ||
446 | @@ -1847,6 +1853,10 @@ static int talk_to_blkback(struct xenbus_device *dev, | ||
447 | if (!info) | ||
448 | return -ENODEV; | ||
449 | |||
450 | + /* Check if backend is trusted. */ | ||
451 | + info->bounce = !xen_blkif_trusted || | ||
452 | + !xenbus_read_unsigned(dev->nodename, "trusted", 1); | ||
453 | + | ||
454 | max_page_order = xenbus_read_unsigned(info->xbdev->otherend, | ||
455 | "max-ring-page-order", 0); | ||
456 | ring_page_order = min(xen_blkif_max_ring_order, max_page_order); | ||
457 | @@ -2273,17 +2283,18 @@ static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo) | ||
458 | if (err) | ||
459 | goto out_of_memory; | ||
460 | |||
461 | - if (!info->feature_persistent && info->max_indirect_segments) { | ||
462 | + if (!info->bounce && info->max_indirect_segments) { | ||
463 | /* | ||
464 | - * We are using indirect descriptors but not persistent | ||
465 | - * grants, we need to allocate a set of pages that can be | ||
466 | + * We are using indirect descriptors but don't have a bounce | ||
467 | + * buffer, we need to allocate a set of pages that can be | ||
468 | * used for mapping indirect grefs | ||
469 | */ | ||
470 | int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info); | ||
471 | |||
472 | BUG_ON(!list_empty(&rinfo->indirect_pages)); | ||
473 | for (i = 0; i < num; i++) { | ||
474 | - struct page *indirect_page = alloc_page(GFP_KERNEL); | ||
475 | + struct page *indirect_page = alloc_page(GFP_KERNEL | | ||
476 | + __GFP_ZERO); | ||
477 | if (!indirect_page) | ||
478 | goto out_of_memory; | ||
479 | list_add(&indirect_page->lru, &rinfo->indirect_pages); | ||
480 | @@ -2375,6 +2386,8 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) | ||
481 | info->feature_persistent = | ||
482 | !!xenbus_read_unsigned(info->xbdev->otherend, | ||
483 | "feature-persistent", 0); | ||
484 | + if (info->feature_persistent) | ||
485 | + info->bounce = true; | ||
486 | |||
487 | indirect_segments = xenbus_read_unsigned(info->xbdev->otherend, | ||
488 | "feature-max-indirect-segments", 0); | ||
489 | @@ -2750,6 +2763,13 @@ static void blkfront_delay_work(struct work_struct *work) | ||
490 | struct blkfront_info *info; | ||
491 | bool need_schedule_work = false; | ||
492 | |||
493 | + /* | ||
494 | + * Note that when using bounce buffers but not persistent grants | ||
495 | + * there's no need to run blkfront_delay_work because grants are | ||
496 | + * revoked in blkif_completion or else an error is reported and the | ||
497 | + * connection is closed. | ||
498 | + */ | ||
499 | + | ||
500 | mutex_lock(&blkfront_mutex); | ||
501 | |||
502 | list_for_each_entry(info, &info_list, info_list) { | ||
503 | diff --git a/drivers/clocksource/timer-ixp4xx.c b/drivers/clocksource/timer-ixp4xx.c | ||
504 | index 9396745e1c179..ad904bbbac6fd 100644 | ||
505 | --- a/drivers/clocksource/timer-ixp4xx.c | ||
506 | +++ b/drivers/clocksource/timer-ixp4xx.c | ||
507 | @@ -258,7 +258,6 @@ void __init ixp4xx_timer_setup(resource_size_t timerbase, | ||
508 | } | ||
509 | ixp4xx_timer_register(base, timer_irq, timer_freq); | ||
510 | } | ||
511 | -EXPORT_SYMBOL_GPL(ixp4xx_timer_setup); | ||
512 | |||
513 | #ifdef CONFIG_OF | ||
514 | static __init int ixp4xx_of_timer_init(struct device_node *np) | ||
515 | diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c | ||
516 | index c4873bb791f88..94e1727d29998 100644 | ||
517 | --- a/drivers/devfreq/event/exynos-ppmu.c | ||
518 | +++ b/drivers/devfreq/event/exynos-ppmu.c | ||
519 | @@ -514,15 +514,19 @@ static int of_get_devfreq_events(struct device_node *np, | ||
520 | |||
521 | count = of_get_child_count(events_np); | ||
522 | desc = devm_kcalloc(dev, count, sizeof(*desc), GFP_KERNEL); | ||
523 | - if (!desc) | ||
524 | + if (!desc) { | ||
525 | + of_node_put(events_np); | ||
526 | return -ENOMEM; | ||
527 | + } | ||
528 | info->num_events = count; | ||
529 | |||
530 | of_id = of_match_device(exynos_ppmu_id_match, dev); | ||
531 | if (of_id) | ||
532 | info->ppmu_type = (enum exynos_ppmu_type)of_id->data; | ||
533 | - else | ||
534 | + else { | ||
535 | + of_node_put(events_np); | ||
536 | return -EINVAL; | ||
537 | + } | ||
538 | |||
539 | j = 0; | ||
540 | for_each_child_of_node(events_np, node) { | ||
541 | diff --git a/drivers/hwmon/ibmaem.c b/drivers/hwmon/ibmaem.c | ||
542 | index d05ab713566df..00a4ffe010393 100644 | ||
543 | --- a/drivers/hwmon/ibmaem.c | ||
544 | +++ b/drivers/hwmon/ibmaem.c | ||
545 | @@ -550,7 +550,7 @@ static int aem_init_aem1_inst(struct aem_ipmi_data *probe, u8 module_handle) | ||
546 | |||
547 | res = platform_device_add(data->pdev); | ||
548 | if (res) | ||
549 | - goto ipmi_err; | ||
550 | + goto dev_add_err; | ||
551 | |||
552 | platform_set_drvdata(data->pdev, data); | ||
553 | |||
554 | @@ -598,7 +598,9 @@ hwmon_reg_err: | ||
555 | ipmi_destroy_user(data->ipmi.user); | ||
556 | ipmi_err: | ||
557 | platform_set_drvdata(data->pdev, NULL); | ||
558 | - platform_device_unregister(data->pdev); | ||
559 | + platform_device_del(data->pdev); | ||
560 | +dev_add_err: | ||
561 | + platform_device_put(data->pdev); | ||
562 | dev_err: | ||
563 | ida_simple_remove(&aem_ida, data->id); | ||
564 | id_err: | ||
565 | @@ -690,7 +692,7 @@ static int aem_init_aem2_inst(struct aem_ipmi_data *probe, | ||
566 | |||
567 | res = platform_device_add(data->pdev); | ||
568 | if (res) | ||
569 | - goto ipmi_err; | ||
570 | + goto dev_add_err; | ||
571 | |||
572 | platform_set_drvdata(data->pdev, data); | ||
573 | |||
574 | @@ -738,7 +740,9 @@ hwmon_reg_err: | ||
575 | ipmi_destroy_user(data->ipmi.user); | ||
576 | ipmi_err: | ||
577 | platform_set_drvdata(data->pdev, NULL); | ||
578 | - platform_device_unregister(data->pdev); | ||
579 | + platform_device_del(data->pdev); | ||
580 | +dev_add_err: | ||
581 | + platform_device_put(data->pdev); | ||
582 | dev_err: | ||
583 | ida_simple_remove(&aem_ida, data->id); | ||
584 | id_err: | ||
585 | diff --git a/drivers/infiniband/hw/qedr/qedr.h b/drivers/infiniband/hw/qedr/qedr.h | ||
586 | index ed56df319d2df..f4b60a2d8aa91 100644 | ||
587 | --- a/drivers/infiniband/hw/qedr/qedr.h | ||
588 | +++ b/drivers/infiniband/hw/qedr/qedr.h | ||
589 | @@ -416,6 +416,7 @@ struct qedr_qp { | ||
590 | u32 sq_psn; | ||
591 | u32 qkey; | ||
592 | u32 dest_qp_num; | ||
593 | + u8 timeout; | ||
594 | |||
595 | /* Relevant to qps created from kernel space only (ULPs) */ | ||
596 | u8 prev_wqe_size; | ||
597 | diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c | ||
598 | index 717f174647647..798a96c2278e8 100644 | ||
599 | --- a/drivers/infiniband/hw/qedr/verbs.c | ||
600 | +++ b/drivers/infiniband/hw/qedr/verbs.c | ||
601 | @@ -2259,6 +2259,8 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, | ||
602 | 1 << max_t(int, attr->timeout - 8, 0); | ||
603 | else | ||
604 | qp_params.ack_timeout = 0; | ||
605 | + | ||
606 | + qp->timeout = attr->timeout; | ||
607 | } | ||
608 | |||
609 | if (attr_mask & IB_QP_RETRY_CNT) { | ||
610 | @@ -2418,7 +2420,7 @@ int qedr_query_qp(struct ib_qp *ibqp, | ||
611 | rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); | ||
612 | rdma_ah_set_port_num(&qp_attr->ah_attr, 1); | ||
613 | rdma_ah_set_sl(&qp_attr->ah_attr, 0); | ||
614 | - qp_attr->timeout = params.timeout; | ||
615 | + qp_attr->timeout = qp->timeout; | ||
616 | qp_attr->rnr_retry = params.rnr_retry; | ||
617 | qp_attr->retry_cnt = params.retry_cnt; | ||
618 | qp_attr->min_rnr_timer = params.min_rnr_nak_timer; | ||
619 | diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c | ||
620 | index 5e73cc6ad0ce4..42151c9fc6e56 100644 | ||
621 | --- a/drivers/md/dm-raid.c | ||
622 | +++ b/drivers/md/dm-raid.c | ||
623 | @@ -998,12 +998,13 @@ static int validate_region_size(struct raid_set *rs, unsigned long region_size) | ||
624 | static int validate_raid_redundancy(struct raid_set *rs) | ||
625 | { | ||
626 | unsigned int i, rebuild_cnt = 0; | ||
627 | - unsigned int rebuilds_per_group = 0, copies; | ||
628 | + unsigned int rebuilds_per_group = 0, copies, raid_disks; | ||
629 | unsigned int group_size, last_group_start; | ||
630 | |||
631 | - for (i = 0; i < rs->md.raid_disks; i++) | ||
632 | - if (!test_bit(In_sync, &rs->dev[i].rdev.flags) || | ||
633 | - !rs->dev[i].rdev.sb_page) | ||
634 | + for (i = 0; i < rs->raid_disks; i++) | ||
635 | + if (!test_bit(FirstUse, &rs->dev[i].rdev.flags) && | ||
636 | + ((!test_bit(In_sync, &rs->dev[i].rdev.flags) || | ||
637 | + !rs->dev[i].rdev.sb_page))) | ||
638 | rebuild_cnt++; | ||
639 | |||
640 | switch (rs->md.level) { | ||
641 | @@ -1043,8 +1044,9 @@ static int validate_raid_redundancy(struct raid_set *rs) | ||
642 | * A A B B C | ||
643 | * C D D E E | ||
644 | */ | ||
645 | + raid_disks = min(rs->raid_disks, rs->md.raid_disks); | ||
646 | if (__is_raid10_near(rs->md.new_layout)) { | ||
647 | - for (i = 0; i < rs->md.raid_disks; i++) { | ||
648 | + for (i = 0; i < raid_disks; i++) { | ||
649 | if (!(i % copies)) | ||
650 | rebuilds_per_group = 0; | ||
651 | if ((!rs->dev[i].rdev.sb_page || | ||
652 | @@ -1067,10 +1069,10 @@ static int validate_raid_redundancy(struct raid_set *rs) | ||
653 | * results in the need to treat the last (potentially larger) | ||
654 | * set differently. | ||
655 | */ | ||
656 | - group_size = (rs->md.raid_disks / copies); | ||
657 | - last_group_start = (rs->md.raid_disks / group_size) - 1; | ||
658 | + group_size = (raid_disks / copies); | ||
659 | + last_group_start = (raid_disks / group_size) - 1; | ||
660 | last_group_start *= group_size; | ||
661 | - for (i = 0; i < rs->md.raid_disks; i++) { | ||
662 | + for (i = 0; i < raid_disks; i++) { | ||
663 | if (!(i % copies) && !(i > last_group_start)) | ||
664 | rebuilds_per_group = 0; | ||
665 | if ((!rs->dev[i].rdev.sb_page || | ||
666 | @@ -1585,7 +1587,7 @@ static sector_t __rdev_sectors(struct raid_set *rs) | ||
667 | { | ||
668 | int i; | ||
669 | |||
670 | - for (i = 0; i < rs->md.raid_disks; i++) { | ||
671 | + for (i = 0; i < rs->raid_disks; i++) { | ||
672 | struct md_rdev *rdev = &rs->dev[i].rdev; | ||
673 | |||
674 | if (!test_bit(Journal, &rdev->flags) && | ||
675 | @@ -3746,13 +3748,13 @@ static int raid_iterate_devices(struct dm_target *ti, | ||
676 | unsigned int i; | ||
677 | int r = 0; | ||
678 | |||
679 | - for (i = 0; !r && i < rs->md.raid_disks; i++) | ||
680 | - if (rs->dev[i].data_dev) | ||
681 | - r = fn(ti, | ||
682 | - rs->dev[i].data_dev, | ||
683 | - 0, /* No offset on data devs */ | ||
684 | - rs->md.dev_sectors, | ||
685 | - data); | ||
686 | + for (i = 0; !r && i < rs->raid_disks; i++) { | ||
687 | + if (rs->dev[i].data_dev) { | ||
688 | + r = fn(ti, rs->dev[i].data_dev, | ||
689 | + 0, /* No offset on data devs */ | ||
690 | + rs->md.dev_sectors, data); | ||
691 | + } | ||
692 | + } | ||
693 | |||
694 | return r; | ||
695 | } | ||
696 | diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c | ||
697 | index c40327ad6ad53..474cf6abefea5 100644 | ||
698 | --- a/drivers/md/raid5.c | ||
699 | +++ b/drivers/md/raid5.c | ||
700 | @@ -7722,6 +7722,7 @@ static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) | ||
701 | */ | ||
702 | if (rdev->saved_raid_disk >= 0 && | ||
703 | rdev->saved_raid_disk >= first && | ||
704 | + rdev->saved_raid_disk <= last && | ||
705 | conf->disks[rdev->saved_raid_disk].rdev == NULL) | ||
706 | first = rdev->saved_raid_disk; | ||
707 | |||
708 | diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c | ||
709 | index 41e58bd2e6069..31ed7616e84e7 100644 | ||
710 | --- a/drivers/net/bonding/bond_3ad.c | ||
711 | +++ b/drivers/net/bonding/bond_3ad.c | ||
712 | @@ -2218,7 +2218,8 @@ void bond_3ad_unbind_slave(struct slave *slave) | ||
713 | temp_aggregator->num_of_ports--; | ||
714 | if (__agg_active_ports(temp_aggregator) == 0) { | ||
715 | select_new_active_agg = temp_aggregator->is_active; | ||
716 | - ad_clear_agg(temp_aggregator); | ||
717 | + if (temp_aggregator->num_of_ports == 0) | ||
718 | + ad_clear_agg(temp_aggregator); | ||
719 | if (select_new_active_agg) { | ||
720 | slave_info(bond->dev, slave->dev, "Removing an active aggregator\n"); | ||
721 | /* select new active aggregator */ | ||
722 | diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c | ||
723 | index a4af185fa466d..8bee935c8f90f 100644 | ||
724 | --- a/drivers/net/bonding/bond_alb.c | ||
725 | +++ b/drivers/net/bonding/bond_alb.c | ||
726 | @@ -1276,12 +1276,12 @@ int bond_alb_initialize(struct bonding *bond, int rlb_enabled) | ||
727 | return res; | ||
728 | |||
729 | if (rlb_enabled) { | ||
730 | - bond->alb_info.rlb_enabled = 1; | ||
731 | res = rlb_initialize(bond); | ||
732 | if (res) { | ||
733 | tlb_deinitialize(bond); | ||
734 | return res; | ||
735 | } | ||
736 | + bond->alb_info.rlb_enabled = 1; | ||
737 | } else { | ||
738 | bond->alb_info.rlb_enabled = 0; | ||
739 | } | ||
740 | diff --git a/drivers/net/caif/caif_virtio.c b/drivers/net/caif/caif_virtio.c | ||
741 | index eb426822ad066..7f2c551e5d690 100644 | ||
742 | --- a/drivers/net/caif/caif_virtio.c | ||
743 | +++ b/drivers/net/caif/caif_virtio.c | ||
744 | @@ -723,13 +723,21 @@ static int cfv_probe(struct virtio_device *vdev) | ||
745 | /* Carrier is off until netdevice is opened */ | ||
746 | netif_carrier_off(netdev); | ||
747 | |||
748 | + /* serialize netdev register + virtio_device_ready() with ndo_open() */ | ||
749 | + rtnl_lock(); | ||
750 | + | ||
751 | /* register Netdev */ | ||
752 | - err = register_netdev(netdev); | ||
753 | + err = register_netdevice(netdev); | ||
754 | if (err) { | ||
755 | + rtnl_unlock(); | ||
756 | dev_err(&vdev->dev, "Unable to register netdev (%d)\n", err); | ||
757 | goto err; | ||
758 | } | ||
759 | |||
760 | + virtio_device_ready(vdev); | ||
761 | + | ||
762 | + rtnl_unlock(); | ||
763 | + | ||
764 | debugfs_init(cfv); | ||
765 | |||
766 | return 0; | ||
767 | diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c | ||
768 | index 0ee1c0a7b165b..282e9e2a50d9c 100644 | ||
769 | --- a/drivers/net/dsa/bcm_sf2.c | ||
770 | +++ b/drivers/net/dsa/bcm_sf2.c | ||
771 | @@ -602,6 +602,11 @@ force_link: | ||
772 | reg |= LINK_STS; | ||
773 | if (state->duplex == DUPLEX_FULL) | ||
774 | reg |= DUPLX_MODE; | ||
775 | + if (state->pause & MLO_PAUSE_TXRX_MASK) { | ||
776 | + if (state->pause & MLO_PAUSE_TX) | ||
777 | + reg |= TXFLOW_CNTL; | ||
778 | + reg |= RXFLOW_CNTL; | ||
779 | + } | ||
780 | |||
781 | core_writel(priv, reg, offset); | ||
782 | } | ||
783 | diff --git a/drivers/net/tun.c b/drivers/net/tun.c | ||
784 | index d9993884a97d9..dd02fcc972774 100644 | ||
785 | --- a/drivers/net/tun.c | ||
786 | +++ b/drivers/net/tun.c | ||
787 | @@ -327,6 +327,12 @@ static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, | ||
788 | } | ||
789 | } | ||
790 | |||
791 | +static void tun_napi_enable(struct tun_file *tfile) | ||
792 | +{ | ||
793 | + if (tfile->napi_enabled) | ||
794 | + napi_enable(&tfile->napi); | ||
795 | +} | ||
796 | + | ||
797 | static void tun_napi_disable(struct tun_file *tfile) | ||
798 | { | ||
799 | if (tfile->napi_enabled) | ||
800 | @@ -690,7 +696,8 @@ static void __tun_detach(struct tun_file *tfile, bool clean) | ||
801 | tun = rtnl_dereference(tfile->tun); | ||
802 | |||
803 | if (tun && clean) { | ||
804 | - tun_napi_disable(tfile); | ||
805 | + if (!tfile->detached) | ||
806 | + tun_napi_disable(tfile); | ||
807 | tun_napi_del(tfile); | ||
808 | } | ||
809 | |||
810 | @@ -709,8 +716,10 @@ static void __tun_detach(struct tun_file *tfile, bool clean) | ||
811 | if (clean) { | ||
812 | RCU_INIT_POINTER(tfile->tun, NULL); | ||
813 | sock_put(&tfile->sk); | ||
814 | - } else | ||
815 | + } else { | ||
816 | tun_disable_queue(tun, tfile); | ||
817 | + tun_napi_disable(tfile); | ||
818 | + } | ||
819 | |||
820 | synchronize_net(); | ||
821 | tun_flow_delete_by_queue(tun, tun->numqueues + 1); | ||
822 | @@ -783,6 +792,7 @@ static void tun_detach_all(struct net_device *dev) | ||
823 | sock_put(&tfile->sk); | ||
824 | } | ||
825 | list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { | ||
826 | + tun_napi_del(tfile); | ||
827 | tun_enable_queue(tfile); | ||
828 | tun_queue_purge(tfile); | ||
829 | xdp_rxq_info_unreg(&tfile->xdp_rxq); | ||
830 | @@ -863,6 +873,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file, | ||
831 | |||
832 | if (tfile->detached) { | ||
833 | tun_enable_queue(tfile); | ||
834 | + tun_napi_enable(tfile); | ||
835 | } else { | ||
836 | sock_hold(&tfile->sk); | ||
837 | tun_napi_init(tun, tfile, napi, napi_frags); | ||
838 | diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c | ||
839 | index a06c3924e0396..ea9c8361bf464 100644 | ||
840 | --- a/drivers/net/usb/ax88179_178a.c | ||
841 | +++ b/drivers/net/usb/ax88179_178a.c | ||
842 | @@ -1365,6 +1365,42 @@ static int ax88179_rx_fixup(struct usbnet *dev, struct sk_buff *skb) | ||
843 | * are bundled into this buffer and where we can find an array of | ||
844 | * per-packet metadata (which contains elements encoded into u16). | ||
845 | */ | ||
846 | + | ||
847 | + /* SKB contents for current firmware: | ||
848 | + * <packet 1> <padding> | ||
849 | + * ... | ||
850 | + * <packet N> <padding> | ||
851 | + * <per-packet metadata entry 1> <dummy header> | ||
852 | + * ... | ||
853 | + * <per-packet metadata entry N> <dummy header> | ||
854 | + * <padding2> <rx_hdr> | ||
855 | + * | ||
856 | + * where: | ||
857 | + * <packet N> contains pkt_len bytes: | ||
858 | + * 2 bytes of IP alignment pseudo header | ||
859 | + * packet received | ||
860 | + * <per-packet metadata entry N> contains 4 bytes: | ||
861 | + * pkt_len and fields AX_RXHDR_* | ||
862 | + * <padding> 0-7 bytes to terminate at | ||
863 | + * 8 bytes boundary (64-bit). | ||
864 | + * <padding2> 4 bytes to make rx_hdr terminate at | ||
865 | + * 8 bytes boundary (64-bit) | ||
866 | + * <dummy-header> contains 4 bytes: | ||
867 | + * pkt_len=0 and AX_RXHDR_DROP_ERR | ||
868 | + * <rx-hdr> contains 4 bytes: | ||
869 | + * pkt_cnt and hdr_off (offset of | ||
870 | + * <per-packet metadata entry 1>) | ||
871 | + * | ||
872 | + * pkt_cnt is number of entrys in the per-packet metadata. | ||
873 | + * In current firmware there is 2 entrys per packet. | ||
874 | + * The first points to the packet and the | ||
875 | + * second is a dummy header. | ||
876 | + * This was done probably to align fields in 64-bit and | ||
877 | + * maintain compatibility with old firmware. | ||
878 | + * This code assumes that <dummy header> and <padding2> are | ||
879 | + * optional. | ||
880 | + */ | ||
881 | + | ||
882 | if (skb->len < 4) | ||
883 | return 0; | ||
884 | skb_trim(skb, skb->len - 4); | ||
885 | @@ -1378,51 +1414,66 @@ static int ax88179_rx_fixup(struct usbnet *dev, struct sk_buff *skb) | ||
886 | /* Make sure that the bounds of the metadata array are inside the SKB | ||
887 | * (and in front of the counter at the end). | ||
888 | */ | ||
889 | - if (pkt_cnt * 2 + hdr_off > skb->len) | ||
890 | + if (pkt_cnt * 4 + hdr_off > skb->len) | ||
891 | return 0; | ||
892 | pkt_hdr = (u32 *)(skb->data + hdr_off); | ||
893 | |||
894 | /* Packets must not overlap the metadata array */ | ||
895 | skb_trim(skb, hdr_off); | ||
896 | |||
897 | - for (; ; pkt_cnt--, pkt_hdr++) { | ||
898 | + for (; pkt_cnt > 0; pkt_cnt--, pkt_hdr++) { | ||
899 | + u16 pkt_len_plus_padd; | ||
900 | u16 pkt_len; | ||
901 | |||
902 | le32_to_cpus(pkt_hdr); | ||
903 | pkt_len = (*pkt_hdr >> 16) & 0x1fff; | ||
904 | + pkt_len_plus_padd = (pkt_len + 7) & 0xfff8; | ||
905 | |||
906 | - if (pkt_len > skb->len) | ||
907 | + /* Skip dummy header used for alignment | ||
908 | + */ | ||
909 | + if (pkt_len == 0) | ||
910 | + continue; | ||
911 | + | ||
912 | + if (pkt_len_plus_padd > skb->len) | ||
913 | return 0; | ||
914 | |||
915 | /* Check CRC or runt packet */ | ||
916 | - if (((*pkt_hdr & (AX_RXHDR_CRC_ERR | AX_RXHDR_DROP_ERR)) == 0) && | ||
917 | - pkt_len >= 2 + ETH_HLEN) { | ||
918 | - bool last = (pkt_cnt == 0); | ||
919 | - | ||
920 | - if (last) { | ||
921 | - ax_skb = skb; | ||
922 | - } else { | ||
923 | - ax_skb = skb_clone(skb, GFP_ATOMIC); | ||
924 | - if (!ax_skb) | ||
925 | - return 0; | ||
926 | - } | ||
927 | - ax_skb->len = pkt_len; | ||
928 | - /* Skip IP alignment pseudo header */ | ||
929 | - skb_pull(ax_skb, 2); | ||
930 | - skb_set_tail_pointer(ax_skb, ax_skb->len); | ||
931 | - ax_skb->truesize = pkt_len + sizeof(struct sk_buff); | ||
932 | - ax88179_rx_checksum(ax_skb, pkt_hdr); | ||
933 | + if ((*pkt_hdr & (AX_RXHDR_CRC_ERR | AX_RXHDR_DROP_ERR)) || | ||
934 | + pkt_len < 2 + ETH_HLEN) { | ||
935 | + dev->net->stats.rx_errors++; | ||
936 | + skb_pull(skb, pkt_len_plus_padd); | ||
937 | + continue; | ||
938 | + } | ||
939 | |||
940 | - if (last) | ||
941 | - return 1; | ||
942 | + /* last packet */ | ||
943 | + if (pkt_len_plus_padd == skb->len) { | ||
944 | + skb_trim(skb, pkt_len); | ||
945 | |||
946 | - usbnet_skb_return(dev, ax_skb); | ||
947 | + /* Skip IP alignment pseudo header */ | ||
948 | + skb_pull(skb, 2); | ||
949 | + | ||
950 | + skb->truesize = SKB_TRUESIZE(pkt_len_plus_padd); | ||
951 | + ax88179_rx_checksum(skb, pkt_hdr); | ||
952 | + return 1; | ||
953 | } | ||
954 | |||
955 | - /* Trim this packet away from the SKB */ | ||
956 | - if (!skb_pull(skb, (pkt_len + 7) & 0xFFF8)) | ||
957 | + ax_skb = skb_clone(skb, GFP_ATOMIC); | ||
958 | + if (!ax_skb) | ||
959 | return 0; | ||
960 | + skb_trim(ax_skb, pkt_len); | ||
961 | + | ||
962 | + /* Skip IP alignment pseudo header */ | ||
963 | + skb_pull(ax_skb, 2); | ||
964 | + | ||
965 | + skb->truesize = pkt_len_plus_padd + | ||
966 | + SKB_DATA_ALIGN(sizeof(struct sk_buff)); | ||
967 | + ax88179_rx_checksum(ax_skb, pkt_hdr); | ||
968 | + usbnet_skb_return(dev, ax_skb); | ||
969 | + | ||
970 | + skb_pull(skb, pkt_len_plus_padd); | ||
971 | } | ||
972 | + | ||
973 | + return 0; | ||
974 | } | ||
975 | |||
976 | static struct sk_buff * | ||
977 | diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c | ||
978 | index 566ea48fd6078..8ef0a013874c5 100644 | ||
979 | --- a/drivers/net/usb/qmi_wwan.c | ||
980 | +++ b/drivers/net/usb/qmi_wwan.c | ||
981 | @@ -1315,6 +1315,8 @@ static const struct usb_device_id products[] = { | ||
982 | {QMI_QUIRK_SET_DTR(0x1bc7, 0x1031, 3)}, /* Telit LE910C1-EUX */ | ||
983 | {QMI_QUIRK_SET_DTR(0x1bc7, 0x1040, 2)}, /* Telit LE922A */ | ||
984 | {QMI_QUIRK_SET_DTR(0x1bc7, 0x1050, 2)}, /* Telit FN980 */ | ||
985 | + {QMI_QUIRK_SET_DTR(0x1bc7, 0x1060, 2)}, /* Telit LN920 */ | ||
986 | + {QMI_QUIRK_SET_DTR(0x1bc7, 0x1070, 2)}, /* Telit FN990 */ | ||
987 | {QMI_FIXED_INTF(0x1bc7, 0x1100, 3)}, /* Telit ME910 */ | ||
988 | {QMI_FIXED_INTF(0x1bc7, 0x1101, 3)}, /* Telit ME910 dual modem */ | ||
989 | {QMI_FIXED_INTF(0x1bc7, 0x1200, 5)}, /* Telit LE920 */ | ||
990 | diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c | ||
991 | index b8b9df82f51ef..997936c3a30fd 100644 | ||
992 | --- a/drivers/net/usb/usbnet.c | ||
993 | +++ b/drivers/net/usb/usbnet.c | ||
994 | @@ -1987,7 +1987,7 @@ static int __usbnet_read_cmd(struct usbnet *dev, u8 cmd, u8 reqtype, | ||
995 | cmd, reqtype, value, index, size); | ||
996 | |||
997 | if (size) { | ||
998 | - buf = kmalloc(size, GFP_KERNEL); | ||
999 | + buf = kmalloc(size, GFP_NOIO); | ||
1000 | if (!buf) | ||
1001 | goto out; | ||
1002 | } | ||
1003 | @@ -2019,7 +2019,7 @@ static int __usbnet_write_cmd(struct usbnet *dev, u8 cmd, u8 reqtype, | ||
1004 | cmd, reqtype, value, index, size); | ||
1005 | |||
1006 | if (data) { | ||
1007 | - buf = kmemdup(data, size, GFP_KERNEL); | ||
1008 | + buf = kmemdup(data, size, GFP_NOIO); | ||
1009 | if (!buf) | ||
1010 | goto out; | ||
1011 | } else { | ||
1012 | diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c | ||
1013 | index f4c03518d7d23..e14842fbe3d62 100644 | ||
1014 | --- a/drivers/net/virtio_net.c | ||
1015 | +++ b/drivers/net/virtio_net.c | ||
1016 | @@ -3219,14 +3219,20 @@ static int virtnet_probe(struct virtio_device *vdev) | ||
1017 | } | ||
1018 | } | ||
1019 | |||
1020 | - err = register_netdev(dev); | ||
1021 | + /* serialize netdev register + virtio_device_ready() with ndo_open() */ | ||
1022 | + rtnl_lock(); | ||
1023 | + | ||
1024 | + err = register_netdevice(dev); | ||
1025 | if (err) { | ||
1026 | pr_debug("virtio_net: registering device failed\n"); | ||
1027 | + rtnl_unlock(); | ||
1028 | goto free_failover; | ||
1029 | } | ||
1030 | |||
1031 | virtio_device_ready(vdev); | ||
1032 | |||
1033 | + rtnl_unlock(); | ||
1034 | + | ||
1035 | err = virtnet_cpu_notif_add(vi); | ||
1036 | if (err) { | ||
1037 | pr_debug("virtio_net: registering cpu notifier failed\n"); | ||
1038 | diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c | ||
1039 | index 94dd6edd18006..810fa9968be7d 100644 | ||
1040 | --- a/drivers/net/xen-netfront.c | ||
1041 | +++ b/drivers/net/xen-netfront.c | ||
1042 | @@ -63,6 +63,10 @@ module_param_named(max_queues, xennet_max_queues, uint, 0644); | ||
1043 | MODULE_PARM_DESC(max_queues, | ||
1044 | "Maximum number of queues per virtual interface"); | ||
1045 | |||
1046 | +static bool __read_mostly xennet_trusted = true; | ||
1047 | +module_param_named(trusted, xennet_trusted, bool, 0644); | ||
1048 | +MODULE_PARM_DESC(trusted, "Is the backend trusted"); | ||
1049 | + | ||
1050 | #define XENNET_TIMEOUT (5 * HZ) | ||
1051 | |||
1052 | static const struct ethtool_ops xennet_ethtool_ops; | ||
1053 | @@ -163,6 +167,9 @@ struct netfront_info { | ||
1054 | /* Is device behaving sane? */ | ||
1055 | bool broken; | ||
1056 | |||
1057 | + /* Should skbs be bounced into a zeroed buffer? */ | ||
1058 | + bool bounce; | ||
1059 | + | ||
1060 | atomic_t rx_gso_checksum_fixup; | ||
1061 | }; | ||
1062 | |||
1063 | @@ -261,7 +268,7 @@ static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) | ||
1064 | if (unlikely(!skb)) | ||
1065 | return NULL; | ||
1066 | |||
1067 | - page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); | ||
1068 | + page = alloc_page(GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO); | ||
1069 | if (!page) { | ||
1070 | kfree_skb(skb); | ||
1071 | return NULL; | ||
1072 | @@ -590,6 +597,34 @@ static void xennet_mark_tx_pending(struct netfront_queue *queue) | ||
1073 | queue->tx_link[i] = TX_PENDING; | ||
1074 | } | ||
1075 | |||
1076 | +struct sk_buff *bounce_skb(const struct sk_buff *skb) | ||
1077 | +{ | ||
1078 | + unsigned int headerlen = skb_headroom(skb); | ||
1079 | + /* Align size to allocate full pages and avoid contiguous data leaks */ | ||
1080 | + unsigned int size = ALIGN(skb_end_offset(skb) + skb->data_len, | ||
1081 | + XEN_PAGE_SIZE); | ||
1082 | + struct sk_buff *n = alloc_skb(size, GFP_ATOMIC | __GFP_ZERO); | ||
1083 | + | ||
1084 | + if (!n) | ||
1085 | + return NULL; | ||
1086 | + | ||
1087 | + if (!IS_ALIGNED((uintptr_t)n->head, XEN_PAGE_SIZE)) { | ||
1088 | + WARN_ONCE(1, "misaligned skb allocated\n"); | ||
1089 | + kfree_skb(n); | ||
1090 | + return NULL; | ||
1091 | + } | ||
1092 | + | ||
1093 | + /* Set the data pointer */ | ||
1094 | + skb_reserve(n, headerlen); | ||
1095 | + /* Set the tail pointer and length */ | ||
1096 | + skb_put(n, skb->len); | ||
1097 | + | ||
1098 | + BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); | ||
1099 | + | ||
1100 | + skb_copy_header(n, skb); | ||
1101 | + return n; | ||
1102 | +} | ||
1103 | + | ||
1104 | #define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1) | ||
1105 | |||
1106 | static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev) | ||
1107 | @@ -642,9 +677,13 @@ static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev | ||
1108 | |||
1109 | /* The first req should be at least ETH_HLEN size or the packet will be | ||
1110 | * dropped by netback. | ||
1111 | + * | ||
1112 | + * If the backend is not trusted bounce all data to zeroed pages to | ||
1113 | + * avoid exposing contiguous data on the granted page not belonging to | ||
1114 | + * the skb. | ||
1115 | */ | ||
1116 | - if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) { | ||
1117 | - nskb = skb_copy(skb, GFP_ATOMIC); | ||
1118 | + if (np->bounce || unlikely(PAGE_SIZE - offset < ETH_HLEN)) { | ||
1119 | + nskb = bounce_skb(skb); | ||
1120 | if (!nskb) | ||
1121 | goto drop; | ||
1122 | dev_consume_skb_any(skb); | ||
1123 | @@ -1950,6 +1989,10 @@ static int talk_to_netback(struct xenbus_device *dev, | ||
1124 | |||
1125 | info->netdev->irq = 0; | ||
1126 | |||
1127 | + /* Check if backend is trusted. */ | ||
1128 | + info->bounce = !xennet_trusted || | ||
1129 | + !xenbus_read_unsigned(dev->nodename, "trusted", 1); | ||
1130 | + | ||
1131 | /* Check if backend supports multiple queues */ | ||
1132 | max_queues = xenbus_read_unsigned(info->xbdev->otherend, | ||
1133 | "multi-queue-max-queues", 1); | ||
1134 | @@ -2103,6 +2146,9 @@ static int xennet_connect(struct net_device *dev) | ||
1135 | err = talk_to_netback(np->xbdev, np); | ||
1136 | if (err) | ||
1137 | return err; | ||
1138 | + if (np->bounce) | ||
1139 | + dev_info(&np->xbdev->dev, | ||
1140 | + "bouncing transmitted data to zeroed pages\n"); | ||
1141 | |||
1142 | /* talk_to_netback() sets the correct number of queues */ | ||
1143 | num_queues = dev->real_num_tx_queues; | ||
1144 | diff --git a/drivers/nfc/nfcmrvl/i2c.c b/drivers/nfc/nfcmrvl/i2c.c | ||
1145 | index 0f22379887ca8..919b4d2f5d8b5 100644 | ||
1146 | --- a/drivers/nfc/nfcmrvl/i2c.c | ||
1147 | +++ b/drivers/nfc/nfcmrvl/i2c.c | ||
1148 | @@ -186,9 +186,9 @@ static int nfcmrvl_i2c_parse_dt(struct device_node *node, | ||
1149 | pdata->irq_polarity = IRQF_TRIGGER_RISING; | ||
1150 | |||
1151 | ret = irq_of_parse_and_map(node, 0); | ||
1152 | - if (ret < 0) { | ||
1153 | - pr_err("Unable to get irq, error: %d\n", ret); | ||
1154 | - return ret; | ||
1155 | + if (!ret) { | ||
1156 | + pr_err("Unable to get irq\n"); | ||
1157 | + return -EINVAL; | ||
1158 | } | ||
1159 | pdata->irq = ret; | ||
1160 | |||
1161 | diff --git a/drivers/nfc/nfcmrvl/spi.c b/drivers/nfc/nfcmrvl/spi.c | ||
1162 | index 8e0ddb4347704..1f4120e3314b2 100644 | ||
1163 | --- a/drivers/nfc/nfcmrvl/spi.c | ||
1164 | +++ b/drivers/nfc/nfcmrvl/spi.c | ||
1165 | @@ -129,9 +129,9 @@ static int nfcmrvl_spi_parse_dt(struct device_node *node, | ||
1166 | } | ||
1167 | |||
1168 | ret = irq_of_parse_and_map(node, 0); | ||
1169 | - if (ret < 0) { | ||
1170 | - pr_err("Unable to get irq, error: %d\n", ret); | ||
1171 | - return ret; | ||
1172 | + if (!ret) { | ||
1173 | + pr_err("Unable to get irq\n"); | ||
1174 | + return -EINVAL; | ||
1175 | } | ||
1176 | pdata->irq = ret; | ||
1177 | |||
1178 | diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c | ||
1179 | index 9f60e4dc5a908..3943a30053b3b 100644 | ||
1180 | --- a/drivers/nfc/nxp-nci/i2c.c | ||
1181 | +++ b/drivers/nfc/nxp-nci/i2c.c | ||
1182 | @@ -162,6 +162,9 @@ static int nxp_nci_i2c_nci_read(struct nxp_nci_i2c_phy *phy, | ||
1183 | |||
1184 | skb_put_data(*skb, (void *)&header, NCI_CTRL_HDR_SIZE); | ||
1185 | |||
1186 | + if (!header.plen) | ||
1187 | + return 0; | ||
1188 | + | ||
1189 | r = i2c_master_recv(client, skb_put(*skb, header.plen), header.plen); | ||
1190 | if (r != header.plen) { | ||
1191 | nfc_err(&client->dev, | ||
1192 | diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c | ||
1193 | index 5e5c6aafc070b..56f189c3129af 100644 | ||
1194 | --- a/drivers/nvdimm/bus.c | ||
1195 | +++ b/drivers/nvdimm/bus.c | ||
1196 | @@ -187,8 +187,8 @@ static int nvdimm_clear_badblocks_region(struct device *dev, void *data) | ||
1197 | ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1; | ||
1198 | |||
1199 | /* make sure we are in the region */ | ||
1200 | - if (ctx->phys < nd_region->ndr_start | ||
1201 | - || (ctx->phys + ctx->cleared) > ndr_end) | ||
1202 | + if (ctx->phys < nd_region->ndr_start || | ||
1203 | + (ctx->phys + ctx->cleared - 1) > ndr_end) | ||
1204 | return 0; | ||
1205 | |||
1206 | sector = (ctx->phys - nd_region->ndr_start) / 512; | ||
1207 | diff --git a/drivers/xen/gntdev-common.h b/drivers/xen/gntdev-common.h | ||
1208 | index 2f8b949c3eeb1..fab6f5a54d5b2 100644 | ||
1209 | --- a/drivers/xen/gntdev-common.h | ||
1210 | +++ b/drivers/xen/gntdev-common.h | ||
1211 | @@ -15,6 +15,8 @@ | ||
1212 | #include <linux/mman.h> | ||
1213 | #include <linux/mmu_notifier.h> | ||
1214 | #include <linux/types.h> | ||
1215 | +#include <xen/interface/event_channel.h> | ||
1216 | +#include <xen/grant_table.h> | ||
1217 | |||
1218 | struct gntdev_dmabuf_priv; | ||
1219 | |||
1220 | @@ -61,6 +63,7 @@ struct gntdev_grant_map { | ||
1221 | struct gnttab_unmap_grant_ref *unmap_ops; | ||
1222 | struct gnttab_map_grant_ref *kmap_ops; | ||
1223 | struct gnttab_unmap_grant_ref *kunmap_ops; | ||
1224 | + bool *being_removed; | ||
1225 | struct page **pages; | ||
1226 | unsigned long pages_vm_start; | ||
1227 | |||
1228 | @@ -78,6 +81,11 @@ struct gntdev_grant_map { | ||
1229 | /* Needed to avoid allocation in gnttab_dma_free_pages(). */ | ||
1230 | xen_pfn_t *frames; | ||
1231 | #endif | ||
1232 | + | ||
1233 | + /* Number of live grants */ | ||
1234 | + atomic_t live_grants; | ||
1235 | + /* Needed to avoid allocation in __unmap_grant_pages */ | ||
1236 | + struct gntab_unmap_queue_data unmap_data; | ||
1237 | }; | ||
1238 | |||
1239 | struct gntdev_grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count, | ||
1240 | diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c | ||
1241 | index e953ea34b3e43..f464793477650 100644 | ||
1242 | --- a/drivers/xen/gntdev.c | ||
1243 | +++ b/drivers/xen/gntdev.c | ||
1244 | @@ -35,6 +35,7 @@ | ||
1245 | #include <linux/slab.h> | ||
1246 | #include <linux/highmem.h> | ||
1247 | #include <linux/refcount.h> | ||
1248 | +#include <linux/workqueue.h> | ||
1249 | |||
1250 | #include <xen/xen.h> | ||
1251 | #include <xen/grant_table.h> | ||
1252 | @@ -62,11 +63,12 @@ MODULE_PARM_DESC(limit, "Maximum number of grants that may be mapped by " | ||
1253 | |||
1254 | static atomic_t pages_mapped = ATOMIC_INIT(0); | ||
1255 | |||
1256 | +/* True in PV mode, false otherwise */ | ||
1257 | static int use_ptemod; | ||
1258 | #define populate_freeable_maps use_ptemod | ||
1259 | |||
1260 | -static int unmap_grant_pages(struct gntdev_grant_map *map, | ||
1261 | - int offset, int pages); | ||
1262 | +static void unmap_grant_pages(struct gntdev_grant_map *map, | ||
1263 | + int offset, int pages); | ||
1264 | |||
1265 | static struct miscdevice gntdev_miscdev; | ||
1266 | |||
1267 | @@ -123,6 +125,7 @@ static void gntdev_free_map(struct gntdev_grant_map *map) | ||
1268 | kfree(map->unmap_ops); | ||
1269 | kfree(map->kmap_ops); | ||
1270 | kfree(map->kunmap_ops); | ||
1271 | + kfree(map->being_removed); | ||
1272 | kfree(map); | ||
1273 | } | ||
1274 | |||
1275 | @@ -142,12 +145,15 @@ struct gntdev_grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count, | ||
1276 | add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL); | ||
1277 | add->kunmap_ops = kcalloc(count, sizeof(add->kunmap_ops[0]), GFP_KERNEL); | ||
1278 | add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL); | ||
1279 | + add->being_removed = | ||
1280 | + kcalloc(count, sizeof(add->being_removed[0]), GFP_KERNEL); | ||
1281 | if (NULL == add->grants || | ||
1282 | NULL == add->map_ops || | ||
1283 | NULL == add->unmap_ops || | ||
1284 | NULL == add->kmap_ops || | ||
1285 | NULL == add->kunmap_ops || | ||
1286 | - NULL == add->pages) | ||
1287 | + NULL == add->pages || | ||
1288 | + NULL == add->being_removed) | ||
1289 | goto err; | ||
1290 | |||
1291 | #ifdef CONFIG_XEN_GRANT_DMA_ALLOC | ||
1292 | @@ -243,6 +249,35 @@ void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map) | ||
1293 | return; | ||
1294 | |||
1295 | atomic_sub(map->count, &pages_mapped); | ||
1296 | + if (map->pages && !use_ptemod) { | ||
1297 | + /* | ||
1298 | + * Increment the reference count. This ensures that the | ||
1299 | + * subsequent call to unmap_grant_pages() will not wind up | ||
1300 | + * re-entering itself. It *can* wind up calling | ||
1301 | + * gntdev_put_map() recursively, but such calls will be with a | ||
1302 | + * reference count greater than 1, so they will return before | ||
1303 | + * this code is reached. The recursion depth is thus limited to | ||
1304 | + * 1. Do NOT use refcount_inc() here, as it will detect that | ||
1305 | + * the reference count is zero and WARN(). | ||
1306 | + */ | ||
1307 | + refcount_set(&map->users, 1); | ||
1308 | + | ||
1309 | + /* | ||
1310 | + * Unmap the grants. This may or may not be asynchronous, so it | ||
1311 | + * is possible that the reference count is 1 on return, but it | ||
1312 | + * could also be greater than 1. | ||
1313 | + */ | ||
1314 | + unmap_grant_pages(map, 0, map->count); | ||
1315 | + | ||
1316 | + /* Check if the memory now needs to be freed */ | ||
1317 | + if (!refcount_dec_and_test(&map->users)) | ||
1318 | + return; | ||
1319 | + | ||
1320 | + /* | ||
1321 | + * All pages have been returned to the hypervisor, so free the | ||
1322 | + * map. | ||
1323 | + */ | ||
1324 | + } | ||
1325 | |||
1326 | if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) { | ||
1327 | notify_remote_via_evtchn(map->notify.event); | ||
1328 | @@ -298,6 +333,7 @@ static int set_grant_ptes_as_special(pte_t *pte, unsigned long addr, void *data) | ||
1329 | |||
1330 | int gntdev_map_grant_pages(struct gntdev_grant_map *map) | ||
1331 | { | ||
1332 | + size_t alloced = 0; | ||
1333 | int i, err = 0; | ||
1334 | |||
1335 | if (!use_ptemod) { | ||
1336 | @@ -346,87 +382,109 @@ int gntdev_map_grant_pages(struct gntdev_grant_map *map) | ||
1337 | map->pages, map->count); | ||
1338 | |||
1339 | for (i = 0; i < map->count; i++) { | ||
1340 | - if (map->map_ops[i].status == GNTST_okay) | ||
1341 | + if (map->map_ops[i].status == GNTST_okay) { | ||
1342 | map->unmap_ops[i].handle = map->map_ops[i].handle; | ||
1343 | - else if (!err) | ||
1344 | + if (!use_ptemod) | ||
1345 | + alloced++; | ||
1346 | + } else if (!err) | ||
1347 | err = -EINVAL; | ||
1348 | |||
1349 | if (map->flags & GNTMAP_device_map) | ||
1350 | map->unmap_ops[i].dev_bus_addr = map->map_ops[i].dev_bus_addr; | ||
1351 | |||
1352 | if (use_ptemod) { | ||
1353 | - if (map->kmap_ops[i].status == GNTST_okay) | ||
1354 | + if (map->kmap_ops[i].status == GNTST_okay) { | ||
1355 | + if (map->map_ops[i].status == GNTST_okay) | ||
1356 | + alloced++; | ||
1357 | map->kunmap_ops[i].handle = map->kmap_ops[i].handle; | ||
1358 | - else if (!err) | ||
1359 | + } else if (!err) | ||
1360 | err = -EINVAL; | ||
1361 | } | ||
1362 | } | ||
1363 | + atomic_add(alloced, &map->live_grants); | ||
1364 | return err; | ||
1365 | } | ||
1366 | |||
1367 | -static int __unmap_grant_pages(struct gntdev_grant_map *map, int offset, | ||
1368 | - int pages) | ||
1369 | +static void __unmap_grant_pages_done(int result, | ||
1370 | + struct gntab_unmap_queue_data *data) | ||
1371 | { | ||
1372 | - int i, err = 0; | ||
1373 | - struct gntab_unmap_queue_data unmap_data; | ||
1374 | + unsigned int i; | ||
1375 | + struct gntdev_grant_map *map = data->data; | ||
1376 | + unsigned int offset = data->unmap_ops - map->unmap_ops; | ||
1377 | |||
1378 | + for (i = 0; i < data->count; i++) { | ||
1379 | + WARN_ON(map->unmap_ops[offset+i].status); | ||
1380 | + pr_debug("unmap handle=%d st=%d\n", | ||
1381 | + map->unmap_ops[offset+i].handle, | ||
1382 | + map->unmap_ops[offset+i].status); | ||
1383 | + map->unmap_ops[offset+i].handle = -1; | ||
1384 | + } | ||
1385 | + /* | ||
1386 | + * Decrease the live-grant counter. This must happen after the loop to | ||
1387 | + * prevent premature reuse of the grants by gnttab_mmap(). | ||
1388 | + */ | ||
1389 | + atomic_sub(data->count, &map->live_grants); | ||
1390 | + | ||
1391 | + /* Release reference taken by __unmap_grant_pages */ | ||
1392 | + gntdev_put_map(NULL, map); | ||
1393 | +} | ||
1394 | + | ||
1395 | +static void __unmap_grant_pages(struct gntdev_grant_map *map, int offset, | ||
1396 | + int pages) | ||
1397 | +{ | ||
1398 | if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { | ||
1399 | int pgno = (map->notify.addr >> PAGE_SHIFT); | ||
1400 | + | ||
1401 | if (pgno >= offset && pgno < offset + pages) { | ||
1402 | /* No need for kmap, pages are in lowmem */ | ||
1403 | uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno])); | ||
1404 | + | ||
1405 | tmp[map->notify.addr & (PAGE_SIZE-1)] = 0; | ||
1406 | map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE; | ||
1407 | } | ||
1408 | } | ||
1409 | |||
1410 | - unmap_data.unmap_ops = map->unmap_ops + offset; | ||
1411 | - unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL; | ||
1412 | - unmap_data.pages = map->pages + offset; | ||
1413 | - unmap_data.count = pages; | ||
1414 | + map->unmap_data.unmap_ops = map->unmap_ops + offset; | ||
1415 | + map->unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL; | ||
1416 | + map->unmap_data.pages = map->pages + offset; | ||
1417 | + map->unmap_data.count = pages; | ||
1418 | + map->unmap_data.done = __unmap_grant_pages_done; | ||
1419 | + map->unmap_data.data = map; | ||
1420 | + refcount_inc(&map->users); /* to keep map alive during async call below */ | ||
1421 | |||
1422 | - err = gnttab_unmap_refs_sync(&unmap_data); | ||
1423 | - if (err) | ||
1424 | - return err; | ||
1425 | - | ||
1426 | - for (i = 0; i < pages; i++) { | ||
1427 | - if (map->unmap_ops[offset+i].status) | ||
1428 | - err = -EINVAL; | ||
1429 | - pr_debug("unmap handle=%d st=%d\n", | ||
1430 | - map->unmap_ops[offset+i].handle, | ||
1431 | - map->unmap_ops[offset+i].status); | ||
1432 | - map->unmap_ops[offset+i].handle = -1; | ||
1433 | - } | ||
1434 | - return err; | ||
1435 | + gnttab_unmap_refs_async(&map->unmap_data); | ||
1436 | } | ||
1437 | |||
1438 | -static int unmap_grant_pages(struct gntdev_grant_map *map, int offset, | ||
1439 | - int pages) | ||
1440 | +static void unmap_grant_pages(struct gntdev_grant_map *map, int offset, | ||
1441 | + int pages) | ||
1442 | { | ||
1443 | - int range, err = 0; | ||
1444 | + int range; | ||
1445 | + | ||
1446 | + if (atomic_read(&map->live_grants) == 0) | ||
1447 | + return; /* Nothing to do */ | ||
1448 | |||
1449 | pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages); | ||
1450 | |||
1451 | /* It is possible the requested range will have a "hole" where we | ||
1452 | * already unmapped some of the grants. Only unmap valid ranges. | ||
1453 | */ | ||
1454 | - while (pages && !err) { | ||
1455 | - while (pages && map->unmap_ops[offset].handle == -1) { | ||
1456 | + while (pages) { | ||
1457 | + while (pages && map->being_removed[offset]) { | ||
1458 | offset++; | ||
1459 | pages--; | ||
1460 | } | ||
1461 | range = 0; | ||
1462 | while (range < pages) { | ||
1463 | - if (map->unmap_ops[offset+range].handle == -1) | ||
1464 | + if (map->being_removed[offset + range]) | ||
1465 | break; | ||
1466 | + map->being_removed[offset + range] = true; | ||
1467 | range++; | ||
1468 | } | ||
1469 | - err = __unmap_grant_pages(map, offset, range); | ||
1470 | + if (range) | ||
1471 | + __unmap_grant_pages(map, offset, range); | ||
1472 | offset += range; | ||
1473 | pages -= range; | ||
1474 | } | ||
1475 | - | ||
1476 | - return err; | ||
1477 | } | ||
1478 | |||
1479 | /* ------------------------------------------------------------------ */ | ||
1480 | @@ -496,7 +554,6 @@ static int unmap_if_in_range(struct gntdev_grant_map *map, | ||
1481 | bool blockable) | ||
1482 | { | ||
1483 | unsigned long mstart, mend; | ||
1484 | - int err; | ||
1485 | |||
1486 | if (!in_range(map, start, end)) | ||
1487 | return 0; | ||
1488 | @@ -510,10 +567,9 @@ static int unmap_if_in_range(struct gntdev_grant_map *map, | ||
1489 | map->index, map->count, | ||
1490 | map->vma->vm_start, map->vma->vm_end, | ||
1491 | start, end, mstart, mend); | ||
1492 | - err = unmap_grant_pages(map, | ||
1493 | + unmap_grant_pages(map, | ||
1494 | (mstart - map->vma->vm_start) >> PAGE_SHIFT, | ||
1495 | (mend - mstart) >> PAGE_SHIFT); | ||
1496 | - WARN_ON(err); | ||
1497 | |||
1498 | return 0; | ||
1499 | } | ||
1500 | @@ -554,7 +610,6 @@ static void mn_release(struct mmu_notifier *mn, | ||
1501 | { | ||
1502 | struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); | ||
1503 | struct gntdev_grant_map *map; | ||
1504 | - int err; | ||
1505 | |||
1506 | mutex_lock(&priv->lock); | ||
1507 | list_for_each_entry(map, &priv->maps, next) { | ||
1508 | @@ -563,8 +618,7 @@ static void mn_release(struct mmu_notifier *mn, | ||
1509 | pr_debug("map %d+%d (%lx %lx)\n", | ||
1510 | map->index, map->count, | ||
1511 | map->vma->vm_start, map->vma->vm_end); | ||
1512 | - err = unmap_grant_pages(map, /* offset */ 0, map->count); | ||
1513 | - WARN_ON(err); | ||
1514 | + unmap_grant_pages(map, /* offset */ 0, map->count); | ||
1515 | } | ||
1516 | list_for_each_entry(map, &priv->freeable_maps, next) { | ||
1517 | if (!map->vma) | ||
1518 | @@ -572,8 +626,7 @@ static void mn_release(struct mmu_notifier *mn, | ||
1519 | pr_debug("map %d+%d (%lx %lx)\n", | ||
1520 | map->index, map->count, | ||
1521 | map->vma->vm_start, map->vma->vm_end); | ||
1522 | - err = unmap_grant_pages(map, /* offset */ 0, map->count); | ||
1523 | - WARN_ON(err); | ||
1524 | + unmap_grant_pages(map, /* offset */ 0, map->count); | ||
1525 | } | ||
1526 | mutex_unlock(&priv->lock); | ||
1527 | } | ||
1528 | @@ -1102,6 +1155,10 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) | ||
1529 | goto unlock_out; | ||
1530 | } | ||
1531 | |||
1532 | + if (atomic_read(&map->live_grants)) { | ||
1533 | + err = -EAGAIN; | ||
1534 | + goto unlock_out; | ||
1535 | + } | ||
1536 | refcount_inc(&map->users); | ||
1537 | |||
1538 | vma->vm_ops = &gntdev_vmops; | ||
1539 | diff --git a/include/linux/dim.h b/include/linux/dim.h | ||
1540 | index 9fa4b3f88c397..2571da63877c5 100644 | ||
1541 | --- a/include/linux/dim.h | ||
1542 | +++ b/include/linux/dim.h | ||
1543 | @@ -17,7 +17,7 @@ | ||
1544 | * We consider 10% difference as significant. | ||
1545 | */ | ||
1546 | #define IS_SIGNIFICANT_DIFF(val, ref) \ | ||
1547 | - (((100UL * abs((val) - (ref))) / (ref)) > 10) | ||
1548 | + ((ref) && (((100UL * abs((val) - (ref))) / (ref)) > 10)) | ||
1549 | |||
1550 | /** | ||
1551 | * Calculate the gap between two values. | ||
1552 | diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c | ||
1553 | index e29553e4f4ee3..a0123760fb2c7 100644 | ||
1554 | --- a/net/ipv6/addrconf.c | ||
1555 | +++ b/net/ipv6/addrconf.c | ||
1556 | @@ -1102,10 +1102,6 @@ ipv6_add_addr(struct inet6_dev *idev, struct ifa6_config *cfg, | ||
1557 | goto out; | ||
1558 | } | ||
1559 | |||
1560 | - if (net->ipv6.devconf_all->disable_policy || | ||
1561 | - idev->cnf.disable_policy) | ||
1562 | - f6i->dst_nopolicy = true; | ||
1563 | - | ||
1564 | neigh_parms_data_state_setall(idev->nd_parms); | ||
1565 | |||
1566 | ifa->addr = *cfg->pfx; | ||
1567 | diff --git a/net/ipv6/route.c b/net/ipv6/route.c | ||
1568 | index 619d9dffa9e44..00732ee6bbd8a 100644 | ||
1569 | --- a/net/ipv6/route.c | ||
1570 | +++ b/net/ipv6/route.c | ||
1571 | @@ -4483,8 +4483,15 @@ struct fib6_info *addrconf_f6i_alloc(struct net *net, | ||
1572 | } | ||
1573 | |||
1574 | f6i = ip6_route_info_create(&cfg, gfp_flags, NULL); | ||
1575 | - if (!IS_ERR(f6i)) | ||
1576 | + if (!IS_ERR(f6i)) { | ||
1577 | f6i->dst_nocount = true; | ||
1578 | + | ||
1579 | + if (!anycast && | ||
1580 | + (net->ipv6.devconf_all->disable_policy || | ||
1581 | + idev->cnf.disable_policy)) | ||
1582 | + f6i->dst_nopolicy = true; | ||
1583 | + } | ||
1584 | + | ||
1585 | return f6i; | ||
1586 | } | ||
1587 | |||
1588 | diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c | ||
1589 | index a4cad71c42047..65394abf4736b 100644 | ||
1590 | --- a/net/ipv6/seg6_hmac.c | ||
1591 | +++ b/net/ipv6/seg6_hmac.c | ||
1592 | @@ -410,7 +410,6 @@ int __net_init seg6_hmac_net_init(struct net *net) | ||
1593 | |||
1594 | return 0; | ||
1595 | } | ||
1596 | -EXPORT_SYMBOL(seg6_hmac_net_init); | ||
1597 | |||
1598 | void seg6_hmac_exit(void) | ||
1599 | { | ||
1600 | diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c | ||
1601 | index 16e75a996b749..117d374695fe6 100644 | ||
1602 | --- a/net/ipv6/sit.c | ||
1603 | +++ b/net/ipv6/sit.c | ||
1604 | @@ -310,9 +310,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t, | ||
1605 | kcalloc(cmax, sizeof(*kp), GFP_KERNEL | __GFP_NOWARN) : | ||
1606 | NULL; | ||
1607 | |||
1608 | - rcu_read_lock(); | ||
1609 | - | ||
1610 | - ca = t->prl_count < cmax ? t->prl_count : cmax; | ||
1611 | + ca = min(t->prl_count, cmax); | ||
1612 | |||
1613 | if (!kp) { | ||
1614 | /* We don't try hard to allocate much memory for | ||
1615 | @@ -327,7 +325,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t, | ||
1616 | } | ||
1617 | } | ||
1618 | |||
1619 | - c = 0; | ||
1620 | + rcu_read_lock(); | ||
1621 | for_each_prl_rcu(t->prl) { | ||
1622 | if (c >= cmax) | ||
1623 | break; | ||
1624 | @@ -339,7 +337,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t, | ||
1625 | if (kprl.addr != htonl(INADDR_ANY)) | ||
1626 | break; | ||
1627 | } | ||
1628 | -out: | ||
1629 | + | ||
1630 | rcu_read_unlock(); | ||
1631 | |||
1632 | len = sizeof(*kp) * c; | ||
1633 | @@ -348,7 +346,7 @@ out: | ||
1634 | ret = -EFAULT; | ||
1635 | |||
1636 | kfree(kp); | ||
1637 | - | ||
1638 | +out: | ||
1639 | return ret; | ||
1640 | } | ||
1641 | |||
1642 | diff --git a/net/netfilter/nft_set_hash.c b/net/netfilter/nft_set_hash.c | ||
1643 | index 9de0eb20e9544..e7eb56b4b89e5 100644 | ||
1644 | --- a/net/netfilter/nft_set_hash.c | ||
1645 | +++ b/net/netfilter/nft_set_hash.c | ||
1646 | @@ -142,6 +142,7 @@ static bool nft_rhash_update(struct nft_set *set, const u32 *key, | ||
1647 | /* Another cpu may race to insert the element with the same key */ | ||
1648 | if (prev) { | ||
1649 | nft_set_elem_destroy(set, he, true); | ||
1650 | + atomic_dec(&set->nelems); | ||
1651 | he = prev; | ||
1652 | } | ||
1653 | |||
1654 | @@ -151,6 +152,7 @@ out: | ||
1655 | |||
1656 | err2: | ||
1657 | nft_set_elem_destroy(set, he, true); | ||
1658 | + atomic_dec(&set->nelems); | ||
1659 | err1: | ||
1660 | return false; | ||
1661 | } | ||
1662 | diff --git a/net/rose/rose_timer.c b/net/rose/rose_timer.c | ||
1663 | index b3138fc2e552e..f06ddbed3fed6 100644 | ||
1664 | --- a/net/rose/rose_timer.c | ||
1665 | +++ b/net/rose/rose_timer.c | ||
1666 | @@ -31,89 +31,89 @@ static void rose_idletimer_expiry(struct timer_list *); | ||
1667 | |||
1668 | void rose_start_heartbeat(struct sock *sk) | ||
1669 | { | ||
1670 | - del_timer(&sk->sk_timer); | ||
1671 | + sk_stop_timer(sk, &sk->sk_timer); | ||
1672 | |||
1673 | sk->sk_timer.function = rose_heartbeat_expiry; | ||
1674 | sk->sk_timer.expires = jiffies + 5 * HZ; | ||
1675 | |||
1676 | - add_timer(&sk->sk_timer); | ||
1677 | + sk_reset_timer(sk, &sk->sk_timer, sk->sk_timer.expires); | ||
1678 | } | ||
1679 | |||
1680 | void rose_start_t1timer(struct sock *sk) | ||
1681 | { | ||
1682 | struct rose_sock *rose = rose_sk(sk); | ||
1683 | |||
1684 | - del_timer(&rose->timer); | ||
1685 | + sk_stop_timer(sk, &rose->timer); | ||
1686 | |||
1687 | rose->timer.function = rose_timer_expiry; | ||
1688 | rose->timer.expires = jiffies + rose->t1; | ||
1689 | |||
1690 | - add_timer(&rose->timer); | ||
1691 | + sk_reset_timer(sk, &rose->timer, rose->timer.expires); | ||
1692 | } | ||
1693 | |||
1694 | void rose_start_t2timer(struct sock *sk) | ||
1695 | { | ||
1696 | struct rose_sock *rose = rose_sk(sk); | ||
1697 | |||
1698 | - del_timer(&rose->timer); | ||
1699 | + sk_stop_timer(sk, &rose->timer); | ||
1700 | |||
1701 | rose->timer.function = rose_timer_expiry; | ||
1702 | rose->timer.expires = jiffies + rose->t2; | ||
1703 | |||
1704 | - add_timer(&rose->timer); | ||
1705 | + sk_reset_timer(sk, &rose->timer, rose->timer.expires); | ||
1706 | } | ||
1707 | |||
1708 | void rose_start_t3timer(struct sock *sk) | ||
1709 | { | ||
1710 | struct rose_sock *rose = rose_sk(sk); | ||
1711 | |||
1712 | - del_timer(&rose->timer); | ||
1713 | + sk_stop_timer(sk, &rose->timer); | ||
1714 | |||
1715 | rose->timer.function = rose_timer_expiry; | ||
1716 | rose->timer.expires = jiffies + rose->t3; | ||
1717 | |||
1718 | - add_timer(&rose->timer); | ||
1719 | + sk_reset_timer(sk, &rose->timer, rose->timer.expires); | ||
1720 | } | ||
1721 | |||
1722 | void rose_start_hbtimer(struct sock *sk) | ||
1723 | { | ||
1724 | struct rose_sock *rose = rose_sk(sk); | ||
1725 | |||
1726 | - del_timer(&rose->timer); | ||
1727 | + sk_stop_timer(sk, &rose->timer); | ||
1728 | |||
1729 | rose->timer.function = rose_timer_expiry; | ||
1730 | rose->timer.expires = jiffies + rose->hb; | ||
1731 | |||
1732 | - add_timer(&rose->timer); | ||
1733 | + sk_reset_timer(sk, &rose->timer, rose->timer.expires); | ||
1734 | } | ||
1735 | |||
1736 | void rose_start_idletimer(struct sock *sk) | ||
1737 | { | ||
1738 | struct rose_sock *rose = rose_sk(sk); | ||
1739 | |||
1740 | - del_timer(&rose->idletimer); | ||
1741 | + sk_stop_timer(sk, &rose->idletimer); | ||
1742 | |||
1743 | if (rose->idle > 0) { | ||
1744 | rose->idletimer.function = rose_idletimer_expiry; | ||
1745 | rose->idletimer.expires = jiffies + rose->idle; | ||
1746 | |||
1747 | - add_timer(&rose->idletimer); | ||
1748 | + sk_reset_timer(sk, &rose->idletimer, rose->idletimer.expires); | ||
1749 | } | ||
1750 | } | ||
1751 | |||
1752 | void rose_stop_heartbeat(struct sock *sk) | ||
1753 | { | ||
1754 | - del_timer(&sk->sk_timer); | ||
1755 | + sk_stop_timer(sk, &sk->sk_timer); | ||
1756 | } | ||
1757 | |||
1758 | void rose_stop_timer(struct sock *sk) | ||
1759 | { | ||
1760 | - del_timer(&rose_sk(sk)->timer); | ||
1761 | + sk_stop_timer(sk, &rose_sk(sk)->timer); | ||
1762 | } | ||
1763 | |||
1764 | void rose_stop_idletimer(struct sock *sk) | ||
1765 | { | ||
1766 | - del_timer(&rose_sk(sk)->idletimer); | ||
1767 | + sk_stop_timer(sk, &rose_sk(sk)->idletimer); | ||
1768 | } | ||
1769 | |||
1770 | static void rose_heartbeat_expiry(struct timer_list *t) | ||
1771 | @@ -130,6 +130,7 @@ static void rose_heartbeat_expiry(struct timer_list *t) | ||
1772 | (sk->sk_state == TCP_LISTEN && sock_flag(sk, SOCK_DEAD))) { | ||
1773 | bh_unlock_sock(sk); | ||
1774 | rose_destroy_socket(sk); | ||
1775 | + sock_put(sk); | ||
1776 | return; | ||
1777 | } | ||
1778 | break; | ||
1779 | @@ -152,6 +153,7 @@ static void rose_heartbeat_expiry(struct timer_list *t) | ||
1780 | |||
1781 | rose_start_heartbeat(sk); | ||
1782 | bh_unlock_sock(sk); | ||
1783 | + sock_put(sk); | ||
1784 | } | ||
1785 | |||
1786 | static void rose_timer_expiry(struct timer_list *t) | ||
1787 | @@ -181,6 +183,7 @@ static void rose_timer_expiry(struct timer_list *t) | ||
1788 | break; | ||
1789 | } | ||
1790 | bh_unlock_sock(sk); | ||
1791 | + sock_put(sk); | ||
1792 | } | ||
1793 | |||
1794 | static void rose_idletimer_expiry(struct timer_list *t) | ||
1795 | @@ -205,4 +208,5 @@ static void rose_idletimer_expiry(struct timer_list *t) | ||
1796 | sock_set_flag(sk, SOCK_DEAD); | ||
1797 | } | ||
1798 | bh_unlock_sock(sk); | ||
1799 | + sock_put(sk); | ||
1800 | } | ||
1801 | diff --git a/net/sched/act_api.c b/net/sched/act_api.c | ||
1802 | index ab277ee950322..db1c0139f99c6 100644 | ||
1803 | --- a/net/sched/act_api.c | ||
1804 | +++ b/net/sched/act_api.c | ||
1805 | @@ -287,7 +287,8 @@ static int tcf_idr_release_unsafe(struct tc_action *p) | ||
1806 | } | ||
1807 | |||
1808 | static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, | ||
1809 | - const struct tc_action_ops *ops) | ||
1810 | + const struct tc_action_ops *ops, | ||
1811 | + struct netlink_ext_ack *extack) | ||
1812 | { | ||
1813 | struct nlattr *nest; | ||
1814 | int n_i = 0; | ||
1815 | @@ -303,20 +304,25 @@ static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, | ||
1816 | if (nla_put_string(skb, TCA_KIND, ops->kind)) | ||
1817 | goto nla_put_failure; | ||
1818 | |||
1819 | + ret = 0; | ||
1820 | mutex_lock(&idrinfo->lock); | ||
1821 | idr_for_each_entry_ul(idr, p, tmp, id) { | ||
1822 | if (IS_ERR(p)) | ||
1823 | continue; | ||
1824 | ret = tcf_idr_release_unsafe(p); | ||
1825 | - if (ret == ACT_P_DELETED) { | ||
1826 | + if (ret == ACT_P_DELETED) | ||
1827 | module_put(ops->owner); | ||
1828 | - n_i++; | ||
1829 | - } else if (ret < 0) { | ||
1830 | - mutex_unlock(&idrinfo->lock); | ||
1831 | - goto nla_put_failure; | ||
1832 | - } | ||
1833 | + else if (ret < 0) | ||
1834 | + break; | ||
1835 | + n_i++; | ||
1836 | } | ||
1837 | mutex_unlock(&idrinfo->lock); | ||
1838 | + if (ret < 0) { | ||
1839 | + if (n_i) | ||
1840 | + NL_SET_ERR_MSG(extack, "Unable to flush all TC actions"); | ||
1841 | + else | ||
1842 | + goto nla_put_failure; | ||
1843 | + } | ||
1844 | |||
1845 | ret = nla_put_u32(skb, TCA_FCNT, n_i); | ||
1846 | if (ret) | ||
1847 | @@ -337,7 +343,7 @@ int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb, | ||
1848 | struct tcf_idrinfo *idrinfo = tn->idrinfo; | ||
1849 | |||
1850 | if (type == RTM_DELACTION) { | ||
1851 | - return tcf_del_walker(idrinfo, skb, ops); | ||
1852 | + return tcf_del_walker(idrinfo, skb, ops, extack); | ||
1853 | } else if (type == RTM_GETACTION) { | ||
1854 | return tcf_dump_walker(idrinfo, skb, cb); | ||
1855 | } else { | ||
1856 | diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c | ||
1857 | index cb8740d156336..daa4165f11794 100644 | ||
1858 | --- a/net/sunrpc/xdr.c | ||
1859 | +++ b/net/sunrpc/xdr.c | ||
1860 | @@ -608,7 +608,7 @@ static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, | ||
1861 | */ | ||
1862 | xdr->p = (void *)p + frag2bytes; | ||
1863 | space_left = xdr->buf->buflen - xdr->buf->len; | ||
1864 | - if (space_left - nbytes >= PAGE_SIZE) | ||
1865 | + if (space_left - frag1bytes >= PAGE_SIZE) | ||
1866 | xdr->end = (void *)p + PAGE_SIZE; | ||
1867 | else | ||
1868 | xdr->end = (void *)p + space_left - frag1bytes; | ||
1869 | diff --git a/tools/testing/selftests/net/udpgso_bench.sh b/tools/testing/selftests/net/udpgso_bench.sh | ||
1870 | index 80b5d352702e5..dc932fd653634 100755 | ||
1871 | --- a/tools/testing/selftests/net/udpgso_bench.sh | ||
1872 | +++ b/tools/testing/selftests/net/udpgso_bench.sh | ||
1873 | @@ -120,7 +120,7 @@ run_all() { | ||
1874 | run_udp "${ipv4_args}" | ||
1875 | |||
1876 | echo "ipv6" | ||
1877 | - run_tcp "${ipv4_args}" | ||
1878 | + run_tcp "${ipv6_args}" | ||
1879 | run_udp "${ipv6_args}" | ||
1880 | } | ||
1881 | |||
1882 | diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile | ||
1883 | index 2af9d39a97168..215e1067f0376 100644 | ||
1884 | --- a/tools/testing/selftests/rseq/Makefile | ||
1885 | +++ b/tools/testing/selftests/rseq/Makefile | ||
1886 | @@ -6,7 +6,7 @@ endif | ||
1887 | |||
1888 | CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L$(OUTPUT) -Wl,-rpath=./ \ | ||
1889 | $(CLANG_FLAGS) | ||
1890 | -LDLIBS += -lpthread | ||
1891 | +LDLIBS += -lpthread -ldl | ||
1892 | |||
1893 | # Own dependencies because we only want to build against 1st prerequisite, but | ||
1894 | # still track changes to header files and depend on shared object. | ||
1895 | diff --git a/tools/testing/selftests/rseq/basic_percpu_ops_test.c b/tools/testing/selftests/rseq/basic_percpu_ops_test.c | ||
1896 | index eb3f6db36d369..517756afc2a4e 100644 | ||
1897 | --- a/tools/testing/selftests/rseq/basic_percpu_ops_test.c | ||
1898 | +++ b/tools/testing/selftests/rseq/basic_percpu_ops_test.c | ||
1899 | @@ -9,10 +9,9 @@ | ||
1900 | #include <string.h> | ||
1901 | #include <stddef.h> | ||
1902 | |||
1903 | +#include "../kselftest.h" | ||
1904 | #include "rseq.h" | ||
1905 | |||
1906 | -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) | ||
1907 | - | ||
1908 | struct percpu_lock_entry { | ||
1909 | intptr_t v; | ||
1910 | } __attribute__((aligned(128))); | ||
1911 | @@ -168,7 +167,7 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, | ||
1912 | for (;;) { | ||
1913 | struct percpu_list_node *head; | ||
1914 | intptr_t *targetptr, expectnot, *load; | ||
1915 | - off_t offset; | ||
1916 | + long offset; | ||
1917 | int ret, cpu; | ||
1918 | |||
1919 | cpu = rseq_cpu_start(); | ||
1920 | diff --git a/tools/testing/selftests/rseq/compiler.h b/tools/testing/selftests/rseq/compiler.h | ||
1921 | new file mode 100644 | ||
1922 | index 0000000000000..876eb6a7f75be | ||
1923 | --- /dev/null | ||
1924 | +++ b/tools/testing/selftests/rseq/compiler.h | ||
1925 | @@ -0,0 +1,30 @@ | ||
1926 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ | ||
1927 | +/* | ||
1928 | + * rseq/compiler.h | ||
1929 | + * | ||
1930 | + * Work-around asm goto compiler bugs. | ||
1931 | + * | ||
1932 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | ||
1933 | + */ | ||
1934 | + | ||
1935 | +#ifndef RSEQ_COMPILER_H | ||
1936 | +#define RSEQ_COMPILER_H | ||
1937 | + | ||
1938 | +/* | ||
1939 | + * gcc prior to 4.8.2 miscompiles asm goto. | ||
1940 | + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 | ||
1941 | + * | ||
1942 | + * gcc prior to 8.1.0 miscompiles asm goto at O1. | ||
1943 | + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103908 | ||
1944 | + * | ||
1945 | + * clang prior to version 13.0.1 miscompiles asm goto at O2. | ||
1946 | + * https://github.com/llvm/llvm-project/issues/52735 | ||
1947 | + * | ||
1948 | + * Work around these issues by adding a volatile inline asm with | ||
1949 | + * memory clobber in the fallthrough after the asm goto and at each | ||
1950 | + * label target. Emit this for all compilers in case other similar | ||
1951 | + * issues are found in the future. | ||
1952 | + */ | ||
1953 | +#define rseq_after_asm_goto() asm volatile ("" : : : "memory") | ||
1954 | + | ||
1955 | +#endif /* RSEQ_COMPILER_H_ */ | ||
1956 | diff --git a/tools/testing/selftests/rseq/param_test.c b/tools/testing/selftests/rseq/param_test.c | ||
1957 | index e8a657a5f48a0..1f399aaf1efe2 100644 | ||
1958 | --- a/tools/testing/selftests/rseq/param_test.c | ||
1959 | +++ b/tools/testing/selftests/rseq/param_test.c | ||
1960 | @@ -159,7 +159,7 @@ unsigned int yield_mod_cnt, nr_abort; | ||
1961 | " cbnz " INJECT_ASM_REG ", 222b\n" \ | ||
1962 | "333:\n" | ||
1963 | |||
1964 | -#elif __PPC__ | ||
1965 | +#elif defined(__PPC__) | ||
1966 | |||
1967 | #define RSEQ_INJECT_INPUT \ | ||
1968 | , [loop_cnt_1]"m"(loop_cnt[1]) \ | ||
1969 | @@ -366,9 +366,7 @@ void *test_percpu_spinlock_thread(void *arg) | ||
1970 | abort(); | ||
1971 | reps = thread_data->reps; | ||
1972 | for (i = 0; i < reps; i++) { | ||
1973 | - int cpu = rseq_cpu_start(); | ||
1974 | - | ||
1975 | - cpu = rseq_this_cpu_lock(&data->lock); | ||
1976 | + int cpu = rseq_this_cpu_lock(&data->lock); | ||
1977 | data->c[cpu].count++; | ||
1978 | rseq_percpu_unlock(&data->lock, cpu); | ||
1979 | #ifndef BENCHMARK | ||
1980 | @@ -549,7 +547,7 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, | ||
1981 | for (;;) { | ||
1982 | struct percpu_list_node *head; | ||
1983 | intptr_t *targetptr, expectnot, *load; | ||
1984 | - off_t offset; | ||
1985 | + long offset; | ||
1986 | int ret; | ||
1987 | |||
1988 | cpu = rseq_cpu_start(); | ||
1989 | diff --git a/tools/testing/selftests/rseq/rseq-abi.h b/tools/testing/selftests/rseq/rseq-abi.h | ||
1990 | new file mode 100644 | ||
1991 | index 0000000000000..a8c44d9af71fb | ||
1992 | --- /dev/null | ||
1993 | +++ b/tools/testing/selftests/rseq/rseq-abi.h | ||
1994 | @@ -0,0 +1,151 @@ | ||
1995 | +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ | ||
1996 | +#ifndef _RSEQ_ABI_H | ||
1997 | +#define _RSEQ_ABI_H | ||
1998 | + | ||
1999 | +/* | ||
2000 | + * rseq-abi.h | ||
2001 | + * | ||
2002 | + * Restartable sequences system call API | ||
2003 | + * | ||
2004 | + * Copyright (c) 2015-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | ||
2005 | + */ | ||
2006 | + | ||
2007 | +#include <linux/types.h> | ||
2008 | +#include <asm/byteorder.h> | ||
2009 | + | ||
2010 | +enum rseq_abi_cpu_id_state { | ||
2011 | + RSEQ_ABI_CPU_ID_UNINITIALIZED = -1, | ||
2012 | + RSEQ_ABI_CPU_ID_REGISTRATION_FAILED = -2, | ||
2013 | +}; | ||
2014 | + | ||
2015 | +enum rseq_abi_flags { | ||
2016 | + RSEQ_ABI_FLAG_UNREGISTER = (1 << 0), | ||
2017 | +}; | ||
2018 | + | ||
2019 | +enum rseq_abi_cs_flags_bit { | ||
2020 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0, | ||
2021 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1, | ||
2022 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2, | ||
2023 | +}; | ||
2024 | + | ||
2025 | +enum rseq_abi_cs_flags { | ||
2026 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT = | ||
2027 | + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT), | ||
2028 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL = | ||
2029 | + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT), | ||
2030 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE = | ||
2031 | + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT), | ||
2032 | +}; | ||
2033 | + | ||
2034 | +/* | ||
2035 | + * struct rseq_abi_cs is aligned on 4 * 8 bytes to ensure it is always | ||
2036 | + * contained within a single cache-line. It is usually declared as | ||
2037 | + * link-time constant data. | ||
2038 | + */ | ||
2039 | +struct rseq_abi_cs { | ||
2040 | + /* Version of this structure. */ | ||
2041 | + __u32 version; | ||
2042 | + /* enum rseq_abi_cs_flags */ | ||
2043 | + __u32 flags; | ||
2044 | + __u64 start_ip; | ||
2045 | + /* Offset from start_ip. */ | ||
2046 | + __u64 post_commit_offset; | ||
2047 | + __u64 abort_ip; | ||
2048 | +} __attribute__((aligned(4 * sizeof(__u64)))); | ||
2049 | + | ||
2050 | +/* | ||
2051 | + * struct rseq_abi is aligned on 4 * 8 bytes to ensure it is always | ||
2052 | + * contained within a single cache-line. | ||
2053 | + * | ||
2054 | + * A single struct rseq_abi per thread is allowed. | ||
2055 | + */ | ||
2056 | +struct rseq_abi { | ||
2057 | + /* | ||
2058 | + * Restartable sequences cpu_id_start field. Updated by the | ||
2059 | + * kernel. Read by user-space with single-copy atomicity | ||
2060 | + * semantics. This field should only be read by the thread which | ||
2061 | + * registered this data structure. Aligned on 32-bit. Always | ||
2062 | + * contains a value in the range of possible CPUs, although the | ||
2063 | + * value may not be the actual current CPU (e.g. if rseq is not | ||
2064 | + * initialized). This CPU number value should always be compared | ||
2065 | + * against the value of the cpu_id field before performing a rseq | ||
2066 | + * commit or returning a value read from a data structure indexed | ||
2067 | + * using the cpu_id_start value. | ||
2068 | + */ | ||
2069 | + __u32 cpu_id_start; | ||
2070 | + /* | ||
2071 | + * Restartable sequences cpu_id field. Updated by the kernel. | ||
2072 | + * Read by user-space with single-copy atomicity semantics. This | ||
2073 | + * field should only be read by the thread which registered this | ||
2074 | + * data structure. Aligned on 32-bit. Values | ||
2075 | + * RSEQ_CPU_ID_UNINITIALIZED and RSEQ_CPU_ID_REGISTRATION_FAILED | ||
2076 | + * have a special semantic: the former means "rseq uninitialized", | ||
2077 | + * and latter means "rseq initialization failed". This value is | ||
2078 | + * meant to be read within rseq critical sections and compared | ||
2079 | + * with the cpu_id_start value previously read, before performing | ||
2080 | + * the commit instruction, or read and compared with the | ||
2081 | + * cpu_id_start value before returning a value loaded from a data | ||
2082 | + * structure indexed using the cpu_id_start value. | ||
2083 | + */ | ||
2084 | + __u32 cpu_id; | ||
2085 | + /* | ||
2086 | + * Restartable sequences rseq_cs field. | ||
2087 | + * | ||
2088 | + * Contains NULL when no critical section is active for the current | ||
2089 | + * thread, or holds a pointer to the currently active struct rseq_cs. | ||
2090 | + * | ||
2091 | + * Updated by user-space, which sets the address of the currently | ||
2092 | + * active rseq_cs at the beginning of assembly instruction sequence | ||
2093 | + * block, and set to NULL by the kernel when it restarts an assembly | ||
2094 | + * instruction sequence block, as well as when the kernel detects that | ||
2095 | + * it is preempting or delivering a signal outside of the range | ||
2096 | + * targeted by the rseq_cs. Also needs to be set to NULL by user-space | ||
2097 | + * before reclaiming memory that contains the targeted struct rseq_cs. | ||
2098 | + * | ||
2099 | + * Read and set by the kernel. Set by user-space with single-copy | ||
2100 | + * atomicity semantics. This field should only be updated by the | ||
2101 | + * thread which registered this data structure. Aligned on 64-bit. | ||
2102 | + */ | ||
2103 | + union { | ||
2104 | + __u64 ptr64; | ||
2105 | + | ||
2106 | + /* | ||
2107 | + * The "arch" field provides architecture accessor for | ||
2108 | + * the ptr field based on architecture pointer size and | ||
2109 | + * endianness. | ||
2110 | + */ | ||
2111 | + struct { | ||
2112 | +#ifdef __LP64__ | ||
2113 | + __u64 ptr; | ||
2114 | +#elif defined(__BYTE_ORDER) ? (__BYTE_ORDER == __BIG_ENDIAN) : defined(__BIG_ENDIAN) | ||
2115 | + __u32 padding; /* Initialized to zero. */ | ||
2116 | + __u32 ptr; | ||
2117 | +#else | ||
2118 | + __u32 ptr; | ||
2119 | + __u32 padding; /* Initialized to zero. */ | ||
2120 | +#endif | ||
2121 | + } arch; | ||
2122 | + } rseq_cs; | ||
2123 | + | ||
2124 | + /* | ||
2125 | + * Restartable sequences flags field. | ||
2126 | + * | ||
2127 | + * This field should only be updated by the thread which | ||
2128 | + * registered this data structure. Read by the kernel. | ||
2129 | + * Mainly used for single-stepping through rseq critical sections | ||
2130 | + * with debuggers. | ||
2131 | + * | ||
2132 | + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT | ||
2133 | + * Inhibit instruction sequence block restart on preemption | ||
2134 | + * for this thread. | ||
2135 | + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL | ||
2136 | + * Inhibit instruction sequence block restart on signal | ||
2137 | + * delivery for this thread. | ||
2138 | + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE | ||
2139 | + * Inhibit instruction sequence block restart on migration for | ||
2140 | + * this thread. | ||
2141 | + */ | ||
2142 | + __u32 flags; | ||
2143 | +} __attribute__((aligned(4 * sizeof(__u64)))); | ||
2144 | + | ||
2145 | +#endif /* _RSEQ_ABI_H */ | ||
2146 | diff --git a/tools/testing/selftests/rseq/rseq-arm.h b/tools/testing/selftests/rseq/rseq-arm.h | ||
2147 | index 5943c816c07ce..893a11eca9d51 100644 | ||
2148 | --- a/tools/testing/selftests/rseq/rseq-arm.h | ||
2149 | +++ b/tools/testing/selftests/rseq/rseq-arm.h | ||
2150 | @@ -147,14 +147,11 @@ do { \ | ||
2151 | teardown \ | ||
2152 | "b %l[" __rseq_str(cmpfail_label) "]\n\t" | ||
2153 | |||
2154 | -#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") | ||
2155 | - | ||
2156 | static inline __attribute__((always_inline)) | ||
2157 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
2158 | { | ||
2159 | RSEQ_INJECT_C(9) | ||
2160 | |||
2161 | - rseq_workaround_gcc_asm_size_guess(); | ||
2162 | __asm__ __volatile__ goto ( | ||
2163 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2164 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2165 | @@ -185,8 +182,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
2166 | "5:\n\t" | ||
2167 | : /* gcc asm goto does not allow outputs */ | ||
2168 | : [cpu_id] "r" (cpu), | ||
2169 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2170 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2171 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2172 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2173 | [v] "m" (*v), | ||
2174 | [expect] "r" (expect), | ||
2175 | [newv] "r" (newv) | ||
2176 | @@ -198,30 +195,31 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
2177 | , error1, error2 | ||
2178 | #endif | ||
2179 | ); | ||
2180 | - rseq_workaround_gcc_asm_size_guess(); | ||
2181 | + rseq_after_asm_goto(); | ||
2182 | return 0; | ||
2183 | abort: | ||
2184 | - rseq_workaround_gcc_asm_size_guess(); | ||
2185 | + rseq_after_asm_goto(); | ||
2186 | RSEQ_INJECT_FAILED | ||
2187 | return -1; | ||
2188 | cmpfail: | ||
2189 | - rseq_workaround_gcc_asm_size_guess(); | ||
2190 | + rseq_after_asm_goto(); | ||
2191 | return 1; | ||
2192 | #ifdef RSEQ_COMPARE_TWICE | ||
2193 | error1: | ||
2194 | + rseq_after_asm_goto(); | ||
2195 | rseq_bug("cpu_id comparison failed"); | ||
2196 | error2: | ||
2197 | + rseq_after_asm_goto(); | ||
2198 | rseq_bug("expected value comparison failed"); | ||
2199 | #endif | ||
2200 | } | ||
2201 | |||
2202 | static inline __attribute__((always_inline)) | ||
2203 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2204 | - off_t voffp, intptr_t *load, int cpu) | ||
2205 | + long voffp, intptr_t *load, int cpu) | ||
2206 | { | ||
2207 | RSEQ_INJECT_C(9) | ||
2208 | |||
2209 | - rseq_workaround_gcc_asm_size_guess(); | ||
2210 | __asm__ __volatile__ goto ( | ||
2211 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2212 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2213 | @@ -255,8 +253,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2214 | "5:\n\t" | ||
2215 | : /* gcc asm goto does not allow outputs */ | ||
2216 | : [cpu_id] "r" (cpu), | ||
2217 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2218 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2219 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2220 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2221 | /* final store input */ | ||
2222 | [v] "m" (*v), | ||
2223 | [expectnot] "r" (expectnot), | ||
2224 | @@ -270,19 +268,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2225 | , error1, error2 | ||
2226 | #endif | ||
2227 | ); | ||
2228 | - rseq_workaround_gcc_asm_size_guess(); | ||
2229 | + rseq_after_asm_goto(); | ||
2230 | return 0; | ||
2231 | abort: | ||
2232 | - rseq_workaround_gcc_asm_size_guess(); | ||
2233 | + rseq_after_asm_goto(); | ||
2234 | RSEQ_INJECT_FAILED | ||
2235 | return -1; | ||
2236 | cmpfail: | ||
2237 | - rseq_workaround_gcc_asm_size_guess(); | ||
2238 | + rseq_after_asm_goto(); | ||
2239 | return 1; | ||
2240 | #ifdef RSEQ_COMPARE_TWICE | ||
2241 | error1: | ||
2242 | + rseq_after_asm_goto(); | ||
2243 | rseq_bug("cpu_id comparison failed"); | ||
2244 | error2: | ||
2245 | + rseq_after_asm_goto(); | ||
2246 | rseq_bug("expected value comparison failed"); | ||
2247 | #endif | ||
2248 | } | ||
2249 | @@ -292,7 +292,6 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
2250 | { | ||
2251 | RSEQ_INJECT_C(9) | ||
2252 | |||
2253 | - rseq_workaround_gcc_asm_size_guess(); | ||
2254 | __asm__ __volatile__ goto ( | ||
2255 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2256 | #ifdef RSEQ_COMPARE_TWICE | ||
2257 | @@ -316,8 +315,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
2258 | "5:\n\t" | ||
2259 | : /* gcc asm goto does not allow outputs */ | ||
2260 | : [cpu_id] "r" (cpu), | ||
2261 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2262 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2263 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2264 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2265 | [v] "m" (*v), | ||
2266 | [count] "Ir" (count) | ||
2267 | RSEQ_INJECT_INPUT | ||
2268 | @@ -328,14 +327,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
2269 | , error1 | ||
2270 | #endif | ||
2271 | ); | ||
2272 | - rseq_workaround_gcc_asm_size_guess(); | ||
2273 | + rseq_after_asm_goto(); | ||
2274 | return 0; | ||
2275 | abort: | ||
2276 | - rseq_workaround_gcc_asm_size_guess(); | ||
2277 | + rseq_after_asm_goto(); | ||
2278 | RSEQ_INJECT_FAILED | ||
2279 | return -1; | ||
2280 | #ifdef RSEQ_COMPARE_TWICE | ||
2281 | error1: | ||
2282 | + rseq_after_asm_goto(); | ||
2283 | rseq_bug("cpu_id comparison failed"); | ||
2284 | #endif | ||
2285 | } | ||
2286 | @@ -347,7 +347,6 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
2287 | { | ||
2288 | RSEQ_INJECT_C(9) | ||
2289 | |||
2290 | - rseq_workaround_gcc_asm_size_guess(); | ||
2291 | __asm__ __volatile__ goto ( | ||
2292 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2293 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2294 | @@ -381,8 +380,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
2295 | "5:\n\t" | ||
2296 | : /* gcc asm goto does not allow outputs */ | ||
2297 | : [cpu_id] "r" (cpu), | ||
2298 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2299 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2300 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2301 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2302 | /* try store input */ | ||
2303 | [v2] "m" (*v2), | ||
2304 | [newv2] "r" (newv2), | ||
2305 | @@ -398,19 +397,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
2306 | , error1, error2 | ||
2307 | #endif | ||
2308 | ); | ||
2309 | - rseq_workaround_gcc_asm_size_guess(); | ||
2310 | + rseq_after_asm_goto(); | ||
2311 | return 0; | ||
2312 | abort: | ||
2313 | - rseq_workaround_gcc_asm_size_guess(); | ||
2314 | + rseq_after_asm_goto(); | ||
2315 | RSEQ_INJECT_FAILED | ||
2316 | return -1; | ||
2317 | cmpfail: | ||
2318 | - rseq_workaround_gcc_asm_size_guess(); | ||
2319 | + rseq_after_asm_goto(); | ||
2320 | return 1; | ||
2321 | #ifdef RSEQ_COMPARE_TWICE | ||
2322 | error1: | ||
2323 | + rseq_after_asm_goto(); | ||
2324 | rseq_bug("cpu_id comparison failed"); | ||
2325 | error2: | ||
2326 | + rseq_after_asm_goto(); | ||
2327 | rseq_bug("expected value comparison failed"); | ||
2328 | #endif | ||
2329 | } | ||
2330 | @@ -422,7 +423,6 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
2331 | { | ||
2332 | RSEQ_INJECT_C(9) | ||
2333 | |||
2334 | - rseq_workaround_gcc_asm_size_guess(); | ||
2335 | __asm__ __volatile__ goto ( | ||
2336 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2337 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2338 | @@ -457,8 +457,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
2339 | "5:\n\t" | ||
2340 | : /* gcc asm goto does not allow outputs */ | ||
2341 | : [cpu_id] "r" (cpu), | ||
2342 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2343 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2344 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2345 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2346 | /* try store input */ | ||
2347 | [v2] "m" (*v2), | ||
2348 | [newv2] "r" (newv2), | ||
2349 | @@ -474,19 +474,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
2350 | , error1, error2 | ||
2351 | #endif | ||
2352 | ); | ||
2353 | - rseq_workaround_gcc_asm_size_guess(); | ||
2354 | + rseq_after_asm_goto(); | ||
2355 | return 0; | ||
2356 | abort: | ||
2357 | - rseq_workaround_gcc_asm_size_guess(); | ||
2358 | + rseq_after_asm_goto(); | ||
2359 | RSEQ_INJECT_FAILED | ||
2360 | return -1; | ||
2361 | cmpfail: | ||
2362 | - rseq_workaround_gcc_asm_size_guess(); | ||
2363 | + rseq_after_asm_goto(); | ||
2364 | return 1; | ||
2365 | #ifdef RSEQ_COMPARE_TWICE | ||
2366 | error1: | ||
2367 | + rseq_after_asm_goto(); | ||
2368 | rseq_bug("cpu_id comparison failed"); | ||
2369 | error2: | ||
2370 | + rseq_after_asm_goto(); | ||
2371 | rseq_bug("expected value comparison failed"); | ||
2372 | #endif | ||
2373 | } | ||
2374 | @@ -498,7 +500,6 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
2375 | { | ||
2376 | RSEQ_INJECT_C(9) | ||
2377 | |||
2378 | - rseq_workaround_gcc_asm_size_guess(); | ||
2379 | __asm__ __volatile__ goto ( | ||
2380 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2381 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2382 | @@ -537,8 +538,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
2383 | "5:\n\t" | ||
2384 | : /* gcc asm goto does not allow outputs */ | ||
2385 | : [cpu_id] "r" (cpu), | ||
2386 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2387 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2388 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2389 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2390 | /* cmp2 input */ | ||
2391 | [v2] "m" (*v2), | ||
2392 | [expect2] "r" (expect2), | ||
2393 | @@ -554,21 +555,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
2394 | , error1, error2, error3 | ||
2395 | #endif | ||
2396 | ); | ||
2397 | - rseq_workaround_gcc_asm_size_guess(); | ||
2398 | + rseq_after_asm_goto(); | ||
2399 | return 0; | ||
2400 | abort: | ||
2401 | - rseq_workaround_gcc_asm_size_guess(); | ||
2402 | + rseq_after_asm_goto(); | ||
2403 | RSEQ_INJECT_FAILED | ||
2404 | return -1; | ||
2405 | cmpfail: | ||
2406 | - rseq_workaround_gcc_asm_size_guess(); | ||
2407 | + rseq_after_asm_goto(); | ||
2408 | return 1; | ||
2409 | #ifdef RSEQ_COMPARE_TWICE | ||
2410 | error1: | ||
2411 | + rseq_after_asm_goto(); | ||
2412 | rseq_bug("cpu_id comparison failed"); | ||
2413 | error2: | ||
2414 | + rseq_after_asm_goto(); | ||
2415 | rseq_bug("1st expected value comparison failed"); | ||
2416 | error3: | ||
2417 | + rseq_after_asm_goto(); | ||
2418 | rseq_bug("2nd expected value comparison failed"); | ||
2419 | #endif | ||
2420 | } | ||
2421 | @@ -582,7 +586,6 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
2422 | |||
2423 | RSEQ_INJECT_C(9) | ||
2424 | |||
2425 | - rseq_workaround_gcc_asm_size_guess(); | ||
2426 | __asm__ __volatile__ goto ( | ||
2427 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2428 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2429 | @@ -657,8 +660,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
2430 | "8:\n\t" | ||
2431 | : /* gcc asm goto does not allow outputs */ | ||
2432 | : [cpu_id] "r" (cpu), | ||
2433 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2434 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2435 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2436 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2437 | /* final store input */ | ||
2438 | [v] "m" (*v), | ||
2439 | [expect] "r" (expect), | ||
2440 | @@ -678,21 +681,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
2441 | , error1, error2 | ||
2442 | #endif | ||
2443 | ); | ||
2444 | - rseq_workaround_gcc_asm_size_guess(); | ||
2445 | + rseq_after_asm_goto(); | ||
2446 | return 0; | ||
2447 | abort: | ||
2448 | - rseq_workaround_gcc_asm_size_guess(); | ||
2449 | + rseq_after_asm_goto(); | ||
2450 | RSEQ_INJECT_FAILED | ||
2451 | return -1; | ||
2452 | cmpfail: | ||
2453 | - rseq_workaround_gcc_asm_size_guess(); | ||
2454 | + rseq_after_asm_goto(); | ||
2455 | return 1; | ||
2456 | #ifdef RSEQ_COMPARE_TWICE | ||
2457 | error1: | ||
2458 | - rseq_workaround_gcc_asm_size_guess(); | ||
2459 | + rseq_after_asm_goto(); | ||
2460 | rseq_bug("cpu_id comparison failed"); | ||
2461 | error2: | ||
2462 | - rseq_workaround_gcc_asm_size_guess(); | ||
2463 | + rseq_after_asm_goto(); | ||
2464 | rseq_bug("expected value comparison failed"); | ||
2465 | #endif | ||
2466 | } | ||
2467 | @@ -706,7 +709,6 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
2468 | |||
2469 | RSEQ_INJECT_C(9) | ||
2470 | |||
2471 | - rseq_workaround_gcc_asm_size_guess(); | ||
2472 | __asm__ __volatile__ goto ( | ||
2473 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2474 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2475 | @@ -782,8 +784,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
2476 | "8:\n\t" | ||
2477 | : /* gcc asm goto does not allow outputs */ | ||
2478 | : [cpu_id] "r" (cpu), | ||
2479 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2480 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2481 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2482 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2483 | /* final store input */ | ||
2484 | [v] "m" (*v), | ||
2485 | [expect] "r" (expect), | ||
2486 | @@ -803,21 +805,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
2487 | , error1, error2 | ||
2488 | #endif | ||
2489 | ); | ||
2490 | - rseq_workaround_gcc_asm_size_guess(); | ||
2491 | + rseq_after_asm_goto(); | ||
2492 | return 0; | ||
2493 | abort: | ||
2494 | - rseq_workaround_gcc_asm_size_guess(); | ||
2495 | + rseq_after_asm_goto(); | ||
2496 | RSEQ_INJECT_FAILED | ||
2497 | return -1; | ||
2498 | cmpfail: | ||
2499 | - rseq_workaround_gcc_asm_size_guess(); | ||
2500 | + rseq_after_asm_goto(); | ||
2501 | return 1; | ||
2502 | #ifdef RSEQ_COMPARE_TWICE | ||
2503 | error1: | ||
2504 | - rseq_workaround_gcc_asm_size_guess(); | ||
2505 | + rseq_after_asm_goto(); | ||
2506 | rseq_bug("cpu_id comparison failed"); | ||
2507 | error2: | ||
2508 | - rseq_workaround_gcc_asm_size_guess(); | ||
2509 | + rseq_after_asm_goto(); | ||
2510 | rseq_bug("expected value comparison failed"); | ||
2511 | #endif | ||
2512 | } | ||
2513 | diff --git a/tools/testing/selftests/rseq/rseq-arm64.h b/tools/testing/selftests/rseq/rseq-arm64.h | ||
2514 | index 200dae9e4208c..cbe190a4d0056 100644 | ||
2515 | --- a/tools/testing/selftests/rseq/rseq-arm64.h | ||
2516 | +++ b/tools/testing/selftests/rseq/rseq-arm64.h | ||
2517 | @@ -230,8 +230,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
2518 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
2519 | : /* gcc asm goto does not allow outputs */ | ||
2520 | : [cpu_id] "r" (cpu), | ||
2521 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), | ||
2522 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2523 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), | ||
2524 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2525 | [v] "Qo" (*v), | ||
2526 | [expect] "r" (expect), | ||
2527 | [newv] "r" (newv) | ||
2528 | @@ -242,24 +242,28 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
2529 | , error1, error2 | ||
2530 | #endif | ||
2531 | ); | ||
2532 | - | ||
2533 | + rseq_after_asm_goto(); | ||
2534 | return 0; | ||
2535 | abort: | ||
2536 | + rseq_after_asm_goto(); | ||
2537 | RSEQ_INJECT_FAILED | ||
2538 | return -1; | ||
2539 | cmpfail: | ||
2540 | + rseq_after_asm_goto(); | ||
2541 | return 1; | ||
2542 | #ifdef RSEQ_COMPARE_TWICE | ||
2543 | error1: | ||
2544 | + rseq_after_asm_goto(); | ||
2545 | rseq_bug("cpu_id comparison failed"); | ||
2546 | error2: | ||
2547 | + rseq_after_asm_goto(); | ||
2548 | rseq_bug("expected value comparison failed"); | ||
2549 | #endif | ||
2550 | } | ||
2551 | |||
2552 | static inline __attribute__((always_inline)) | ||
2553 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2554 | - off_t voffp, intptr_t *load, int cpu) | ||
2555 | + long voffp, intptr_t *load, int cpu) | ||
2556 | { | ||
2557 | RSEQ_INJECT_C(9) | ||
2558 | |||
2559 | @@ -287,8 +291,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2560 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
2561 | : /* gcc asm goto does not allow outputs */ | ||
2562 | : [cpu_id] "r" (cpu), | ||
2563 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), | ||
2564 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2565 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), | ||
2566 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2567 | [v] "Qo" (*v), | ||
2568 | [expectnot] "r" (expectnot), | ||
2569 | [load] "Qo" (*load), | ||
2570 | @@ -300,16 +304,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2571 | , error1, error2 | ||
2572 | #endif | ||
2573 | ); | ||
2574 | + rseq_after_asm_goto(); | ||
2575 | return 0; | ||
2576 | abort: | ||
2577 | + rseq_after_asm_goto(); | ||
2578 | RSEQ_INJECT_FAILED | ||
2579 | return -1; | ||
2580 | cmpfail: | ||
2581 | + rseq_after_asm_goto(); | ||
2582 | return 1; | ||
2583 | #ifdef RSEQ_COMPARE_TWICE | ||
2584 | error1: | ||
2585 | + rseq_after_asm_goto(); | ||
2586 | rseq_bug("cpu_id comparison failed"); | ||
2587 | error2: | ||
2588 | + rseq_after_asm_goto(); | ||
2589 | rseq_bug("expected value comparison failed"); | ||
2590 | #endif | ||
2591 | } | ||
2592 | @@ -337,8 +346,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
2593 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
2594 | : /* gcc asm goto does not allow outputs */ | ||
2595 | : [cpu_id] "r" (cpu), | ||
2596 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), | ||
2597 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2598 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), | ||
2599 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2600 | [v] "Qo" (*v), | ||
2601 | [count] "r" (count) | ||
2602 | RSEQ_INJECT_INPUT | ||
2603 | @@ -348,12 +357,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
2604 | , error1 | ||
2605 | #endif | ||
2606 | ); | ||
2607 | + rseq_after_asm_goto(); | ||
2608 | return 0; | ||
2609 | abort: | ||
2610 | + rseq_after_asm_goto(); | ||
2611 | RSEQ_INJECT_FAILED | ||
2612 | return -1; | ||
2613 | #ifdef RSEQ_COMPARE_TWICE | ||
2614 | error1: | ||
2615 | + rseq_after_asm_goto(); | ||
2616 | rseq_bug("cpu_id comparison failed"); | ||
2617 | #endif | ||
2618 | } | ||
2619 | @@ -388,8 +400,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
2620 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
2621 | : /* gcc asm goto does not allow outputs */ | ||
2622 | : [cpu_id] "r" (cpu), | ||
2623 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), | ||
2624 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2625 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), | ||
2626 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2627 | [expect] "r" (expect), | ||
2628 | [v] "Qo" (*v), | ||
2629 | [newv] "r" (newv), | ||
2630 | @@ -402,17 +414,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
2631 | , error1, error2 | ||
2632 | #endif | ||
2633 | ); | ||
2634 | - | ||
2635 | + rseq_after_asm_goto(); | ||
2636 | return 0; | ||
2637 | abort: | ||
2638 | + rseq_after_asm_goto(); | ||
2639 | RSEQ_INJECT_FAILED | ||
2640 | return -1; | ||
2641 | cmpfail: | ||
2642 | + rseq_after_asm_goto(); | ||
2643 | return 1; | ||
2644 | #ifdef RSEQ_COMPARE_TWICE | ||
2645 | error1: | ||
2646 | + rseq_after_asm_goto(); | ||
2647 | rseq_bug("cpu_id comparison failed"); | ||
2648 | error2: | ||
2649 | + rseq_after_asm_goto(); | ||
2650 | rseq_bug("expected value comparison failed"); | ||
2651 | #endif | ||
2652 | } | ||
2653 | @@ -447,8 +463,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
2654 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
2655 | : /* gcc asm goto does not allow outputs */ | ||
2656 | : [cpu_id] "r" (cpu), | ||
2657 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), | ||
2658 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2659 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), | ||
2660 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2661 | [expect] "r" (expect), | ||
2662 | [v] "Qo" (*v), | ||
2663 | [newv] "r" (newv), | ||
2664 | @@ -461,17 +477,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
2665 | , error1, error2 | ||
2666 | #endif | ||
2667 | ); | ||
2668 | - | ||
2669 | + rseq_after_asm_goto(); | ||
2670 | return 0; | ||
2671 | abort: | ||
2672 | + rseq_after_asm_goto(); | ||
2673 | RSEQ_INJECT_FAILED | ||
2674 | return -1; | ||
2675 | cmpfail: | ||
2676 | + rseq_after_asm_goto(); | ||
2677 | return 1; | ||
2678 | #ifdef RSEQ_COMPARE_TWICE | ||
2679 | error1: | ||
2680 | + rseq_after_asm_goto(); | ||
2681 | rseq_bug("cpu_id comparison failed"); | ||
2682 | error2: | ||
2683 | + rseq_after_asm_goto(); | ||
2684 | rseq_bug("expected value comparison failed"); | ||
2685 | #endif | ||
2686 | } | ||
2687 | @@ -508,8 +528,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
2688 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
2689 | : /* gcc asm goto does not allow outputs */ | ||
2690 | : [cpu_id] "r" (cpu), | ||
2691 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), | ||
2692 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2693 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), | ||
2694 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2695 | [v] "Qo" (*v), | ||
2696 | [expect] "r" (expect), | ||
2697 | [v2] "Qo" (*v2), | ||
2698 | @@ -522,19 +542,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
2699 | , error1, error2, error3 | ||
2700 | #endif | ||
2701 | ); | ||
2702 | - | ||
2703 | + rseq_after_asm_goto(); | ||
2704 | return 0; | ||
2705 | abort: | ||
2706 | + rseq_after_asm_goto(); | ||
2707 | RSEQ_INJECT_FAILED | ||
2708 | return -1; | ||
2709 | cmpfail: | ||
2710 | + rseq_after_asm_goto(); | ||
2711 | return 1; | ||
2712 | #ifdef RSEQ_COMPARE_TWICE | ||
2713 | error1: | ||
2714 | + rseq_after_asm_goto(); | ||
2715 | rseq_bug("cpu_id comparison failed"); | ||
2716 | error2: | ||
2717 | + rseq_after_asm_goto(); | ||
2718 | rseq_bug("expected value comparison failed"); | ||
2719 | error3: | ||
2720 | + rseq_after_asm_goto(); | ||
2721 | rseq_bug("2nd expected value comparison failed"); | ||
2722 | #endif | ||
2723 | } | ||
2724 | @@ -569,8 +594,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
2725 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
2726 | : /* gcc asm goto does not allow outputs */ | ||
2727 | : [cpu_id] "r" (cpu), | ||
2728 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), | ||
2729 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2730 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), | ||
2731 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2732 | [expect] "r" (expect), | ||
2733 | [v] "Qo" (*v), | ||
2734 | [newv] "r" (newv), | ||
2735 | @@ -584,17 +609,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
2736 | , error1, error2 | ||
2737 | #endif | ||
2738 | ); | ||
2739 | - | ||
2740 | + rseq_after_asm_goto(); | ||
2741 | return 0; | ||
2742 | abort: | ||
2743 | + rseq_after_asm_goto(); | ||
2744 | RSEQ_INJECT_FAILED | ||
2745 | return -1; | ||
2746 | cmpfail: | ||
2747 | + rseq_after_asm_goto(); | ||
2748 | return 1; | ||
2749 | #ifdef RSEQ_COMPARE_TWICE | ||
2750 | error1: | ||
2751 | + rseq_after_asm_goto(); | ||
2752 | rseq_bug("cpu_id comparison failed"); | ||
2753 | error2: | ||
2754 | + rseq_after_asm_goto(); | ||
2755 | rseq_bug("expected value comparison failed"); | ||
2756 | #endif | ||
2757 | } | ||
2758 | @@ -629,8 +658,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
2759 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
2760 | : /* gcc asm goto does not allow outputs */ | ||
2761 | : [cpu_id] "r" (cpu), | ||
2762 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), | ||
2763 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2764 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), | ||
2765 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2766 | [expect] "r" (expect), | ||
2767 | [v] "Qo" (*v), | ||
2768 | [newv] "r" (newv), | ||
2769 | @@ -644,17 +673,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
2770 | , error1, error2 | ||
2771 | #endif | ||
2772 | ); | ||
2773 | - | ||
2774 | + rseq_after_asm_goto(); | ||
2775 | return 0; | ||
2776 | abort: | ||
2777 | + rseq_after_asm_goto(); | ||
2778 | RSEQ_INJECT_FAILED | ||
2779 | return -1; | ||
2780 | cmpfail: | ||
2781 | + rseq_after_asm_goto(); | ||
2782 | return 1; | ||
2783 | #ifdef RSEQ_COMPARE_TWICE | ||
2784 | error1: | ||
2785 | + rseq_after_asm_goto(); | ||
2786 | rseq_bug("cpu_id comparison failed"); | ||
2787 | error2: | ||
2788 | + rseq_after_asm_goto(); | ||
2789 | rseq_bug("expected value comparison failed"); | ||
2790 | #endif | ||
2791 | } | ||
2792 | diff --git a/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h b/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h | ||
2793 | new file mode 100644 | ||
2794 | index 0000000000000..38c5846615714 | ||
2795 | --- /dev/null | ||
2796 | +++ b/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h | ||
2797 | @@ -0,0 +1,25 @@ | ||
2798 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ | ||
2799 | +/* | ||
2800 | + * rseq-generic-thread-pointer.h | ||
2801 | + * | ||
2802 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | ||
2803 | + */ | ||
2804 | + | ||
2805 | +#ifndef _RSEQ_GENERIC_THREAD_POINTER | ||
2806 | +#define _RSEQ_GENERIC_THREAD_POINTER | ||
2807 | + | ||
2808 | +#ifdef __cplusplus | ||
2809 | +extern "C" { | ||
2810 | +#endif | ||
2811 | + | ||
2812 | +/* Use gcc builtin thread pointer. */ | ||
2813 | +static inline void *rseq_thread_pointer(void) | ||
2814 | +{ | ||
2815 | + return __builtin_thread_pointer(); | ||
2816 | +} | ||
2817 | + | ||
2818 | +#ifdef __cplusplus | ||
2819 | +} | ||
2820 | +#endif | ||
2821 | + | ||
2822 | +#endif | ||
2823 | diff --git a/tools/testing/selftests/rseq/rseq-mips.h b/tools/testing/selftests/rseq/rseq-mips.h | ||
2824 | index e989e7c14b097..878739fae2fde 100644 | ||
2825 | --- a/tools/testing/selftests/rseq/rseq-mips.h | ||
2826 | +++ b/tools/testing/selftests/rseq/rseq-mips.h | ||
2827 | @@ -154,14 +154,11 @@ do { \ | ||
2828 | teardown \ | ||
2829 | "b %l[" __rseq_str(cmpfail_label) "]\n\t" | ||
2830 | |||
2831 | -#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") | ||
2832 | - | ||
2833 | static inline __attribute__((always_inline)) | ||
2834 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
2835 | { | ||
2836 | RSEQ_INJECT_C(9) | ||
2837 | |||
2838 | - rseq_workaround_gcc_asm_size_guess(); | ||
2839 | __asm__ __volatile__ goto ( | ||
2840 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2841 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2842 | @@ -190,8 +187,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
2843 | "5:\n\t" | ||
2844 | : /* gcc asm goto does not allow outputs */ | ||
2845 | : [cpu_id] "r" (cpu), | ||
2846 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2847 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2848 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2849 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2850 | [v] "m" (*v), | ||
2851 | [expect] "r" (expect), | ||
2852 | [newv] "r" (newv) | ||
2853 | @@ -203,14 +200,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
2854 | , error1, error2 | ||
2855 | #endif | ||
2856 | ); | ||
2857 | - rseq_workaround_gcc_asm_size_guess(); | ||
2858 | return 0; | ||
2859 | abort: | ||
2860 | - rseq_workaround_gcc_asm_size_guess(); | ||
2861 | RSEQ_INJECT_FAILED | ||
2862 | return -1; | ||
2863 | cmpfail: | ||
2864 | - rseq_workaround_gcc_asm_size_guess(); | ||
2865 | return 1; | ||
2866 | #ifdef RSEQ_COMPARE_TWICE | ||
2867 | error1: | ||
2868 | @@ -222,11 +216,10 @@ error2: | ||
2869 | |||
2870 | static inline __attribute__((always_inline)) | ||
2871 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2872 | - off_t voffp, intptr_t *load, int cpu) | ||
2873 | + long voffp, intptr_t *load, int cpu) | ||
2874 | { | ||
2875 | RSEQ_INJECT_C(9) | ||
2876 | |||
2877 | - rseq_workaround_gcc_asm_size_guess(); | ||
2878 | __asm__ __volatile__ goto ( | ||
2879 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2880 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2881 | @@ -258,8 +251,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2882 | "5:\n\t" | ||
2883 | : /* gcc asm goto does not allow outputs */ | ||
2884 | : [cpu_id] "r" (cpu), | ||
2885 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2886 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2887 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2888 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2889 | /* final store input */ | ||
2890 | [v] "m" (*v), | ||
2891 | [expectnot] "r" (expectnot), | ||
2892 | @@ -273,14 +266,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
2893 | , error1, error2 | ||
2894 | #endif | ||
2895 | ); | ||
2896 | - rseq_workaround_gcc_asm_size_guess(); | ||
2897 | return 0; | ||
2898 | abort: | ||
2899 | - rseq_workaround_gcc_asm_size_guess(); | ||
2900 | RSEQ_INJECT_FAILED | ||
2901 | return -1; | ||
2902 | cmpfail: | ||
2903 | - rseq_workaround_gcc_asm_size_guess(); | ||
2904 | return 1; | ||
2905 | #ifdef RSEQ_COMPARE_TWICE | ||
2906 | error1: | ||
2907 | @@ -295,7 +285,6 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
2908 | { | ||
2909 | RSEQ_INJECT_C(9) | ||
2910 | |||
2911 | - rseq_workaround_gcc_asm_size_guess(); | ||
2912 | __asm__ __volatile__ goto ( | ||
2913 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2914 | #ifdef RSEQ_COMPARE_TWICE | ||
2915 | @@ -319,8 +308,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
2916 | "5:\n\t" | ||
2917 | : /* gcc asm goto does not allow outputs */ | ||
2918 | : [cpu_id] "r" (cpu), | ||
2919 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2920 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2921 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2922 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2923 | [v] "m" (*v), | ||
2924 | [count] "Ir" (count) | ||
2925 | RSEQ_INJECT_INPUT | ||
2926 | @@ -331,10 +320,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
2927 | , error1 | ||
2928 | #endif | ||
2929 | ); | ||
2930 | - rseq_workaround_gcc_asm_size_guess(); | ||
2931 | return 0; | ||
2932 | abort: | ||
2933 | - rseq_workaround_gcc_asm_size_guess(); | ||
2934 | RSEQ_INJECT_FAILED | ||
2935 | return -1; | ||
2936 | #ifdef RSEQ_COMPARE_TWICE | ||
2937 | @@ -350,7 +337,6 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
2938 | { | ||
2939 | RSEQ_INJECT_C(9) | ||
2940 | |||
2941 | - rseq_workaround_gcc_asm_size_guess(); | ||
2942 | __asm__ __volatile__ goto ( | ||
2943 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2944 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2945 | @@ -382,8 +368,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
2946 | "5:\n\t" | ||
2947 | : /* gcc asm goto does not allow outputs */ | ||
2948 | : [cpu_id] "r" (cpu), | ||
2949 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2950 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2951 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2952 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2953 | /* try store input */ | ||
2954 | [v2] "m" (*v2), | ||
2955 | [newv2] "r" (newv2), | ||
2956 | @@ -399,14 +385,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
2957 | , error1, error2 | ||
2958 | #endif | ||
2959 | ); | ||
2960 | - rseq_workaround_gcc_asm_size_guess(); | ||
2961 | return 0; | ||
2962 | abort: | ||
2963 | - rseq_workaround_gcc_asm_size_guess(); | ||
2964 | RSEQ_INJECT_FAILED | ||
2965 | return -1; | ||
2966 | cmpfail: | ||
2967 | - rseq_workaround_gcc_asm_size_guess(); | ||
2968 | return 1; | ||
2969 | #ifdef RSEQ_COMPARE_TWICE | ||
2970 | error1: | ||
2971 | @@ -423,7 +406,6 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
2972 | { | ||
2973 | RSEQ_INJECT_C(9) | ||
2974 | |||
2975 | - rseq_workaround_gcc_asm_size_guess(); | ||
2976 | __asm__ __volatile__ goto ( | ||
2977 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
2978 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
2979 | @@ -456,8 +438,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
2980 | "5:\n\t" | ||
2981 | : /* gcc asm goto does not allow outputs */ | ||
2982 | : [cpu_id] "r" (cpu), | ||
2983 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
2984 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
2985 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
2986 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
2987 | /* try store input */ | ||
2988 | [v2] "m" (*v2), | ||
2989 | [newv2] "r" (newv2), | ||
2990 | @@ -473,14 +455,11 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
2991 | , error1, error2 | ||
2992 | #endif | ||
2993 | ); | ||
2994 | - rseq_workaround_gcc_asm_size_guess(); | ||
2995 | return 0; | ||
2996 | abort: | ||
2997 | - rseq_workaround_gcc_asm_size_guess(); | ||
2998 | RSEQ_INJECT_FAILED | ||
2999 | return -1; | ||
3000 | cmpfail: | ||
3001 | - rseq_workaround_gcc_asm_size_guess(); | ||
3002 | return 1; | ||
3003 | #ifdef RSEQ_COMPARE_TWICE | ||
3004 | error1: | ||
3005 | @@ -497,7 +476,6 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
3006 | { | ||
3007 | RSEQ_INJECT_C(9) | ||
3008 | |||
3009 | - rseq_workaround_gcc_asm_size_guess(); | ||
3010 | __asm__ __volatile__ goto ( | ||
3011 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
3012 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
3013 | @@ -532,8 +510,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
3014 | "5:\n\t" | ||
3015 | : /* gcc asm goto does not allow outputs */ | ||
3016 | : [cpu_id] "r" (cpu), | ||
3017 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3018 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3019 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3020 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3021 | /* cmp2 input */ | ||
3022 | [v2] "m" (*v2), | ||
3023 | [expect2] "r" (expect2), | ||
3024 | @@ -549,14 +527,11 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
3025 | , error1, error2, error3 | ||
3026 | #endif | ||
3027 | ); | ||
3028 | - rseq_workaround_gcc_asm_size_guess(); | ||
3029 | return 0; | ||
3030 | abort: | ||
3031 | - rseq_workaround_gcc_asm_size_guess(); | ||
3032 | RSEQ_INJECT_FAILED | ||
3033 | return -1; | ||
3034 | cmpfail: | ||
3035 | - rseq_workaround_gcc_asm_size_guess(); | ||
3036 | return 1; | ||
3037 | #ifdef RSEQ_COMPARE_TWICE | ||
3038 | error1: | ||
3039 | @@ -577,7 +552,6 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
3040 | |||
3041 | RSEQ_INJECT_C(9) | ||
3042 | |||
3043 | - rseq_workaround_gcc_asm_size_guess(); | ||
3044 | __asm__ __volatile__ goto ( | ||
3045 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
3046 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
3047 | @@ -649,8 +623,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
3048 | "8:\n\t" | ||
3049 | : /* gcc asm goto does not allow outputs */ | ||
3050 | : [cpu_id] "r" (cpu), | ||
3051 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3052 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3053 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3054 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3055 | /* final store input */ | ||
3056 | [v] "m" (*v), | ||
3057 | [expect] "r" (expect), | ||
3058 | @@ -670,21 +644,16 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
3059 | , error1, error2 | ||
3060 | #endif | ||
3061 | ); | ||
3062 | - rseq_workaround_gcc_asm_size_guess(); | ||
3063 | return 0; | ||
3064 | abort: | ||
3065 | - rseq_workaround_gcc_asm_size_guess(); | ||
3066 | RSEQ_INJECT_FAILED | ||
3067 | return -1; | ||
3068 | cmpfail: | ||
3069 | - rseq_workaround_gcc_asm_size_guess(); | ||
3070 | return 1; | ||
3071 | #ifdef RSEQ_COMPARE_TWICE | ||
3072 | error1: | ||
3073 | - rseq_workaround_gcc_asm_size_guess(); | ||
3074 | rseq_bug("cpu_id comparison failed"); | ||
3075 | error2: | ||
3076 | - rseq_workaround_gcc_asm_size_guess(); | ||
3077 | rseq_bug("expected value comparison failed"); | ||
3078 | #endif | ||
3079 | } | ||
3080 | @@ -698,7 +667,6 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
3081 | |||
3082 | RSEQ_INJECT_C(9) | ||
3083 | |||
3084 | - rseq_workaround_gcc_asm_size_guess(); | ||
3085 | __asm__ __volatile__ goto ( | ||
3086 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ | ||
3087 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) | ||
3088 | @@ -771,8 +739,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
3089 | "8:\n\t" | ||
3090 | : /* gcc asm goto does not allow outputs */ | ||
3091 | : [cpu_id] "r" (cpu), | ||
3092 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3093 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3094 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3095 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3096 | /* final store input */ | ||
3097 | [v] "m" (*v), | ||
3098 | [expect] "r" (expect), | ||
3099 | @@ -792,21 +760,16 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
3100 | , error1, error2 | ||
3101 | #endif | ||
3102 | ); | ||
3103 | - rseq_workaround_gcc_asm_size_guess(); | ||
3104 | return 0; | ||
3105 | abort: | ||
3106 | - rseq_workaround_gcc_asm_size_guess(); | ||
3107 | RSEQ_INJECT_FAILED | ||
3108 | return -1; | ||
3109 | cmpfail: | ||
3110 | - rseq_workaround_gcc_asm_size_guess(); | ||
3111 | return 1; | ||
3112 | #ifdef RSEQ_COMPARE_TWICE | ||
3113 | error1: | ||
3114 | - rseq_workaround_gcc_asm_size_guess(); | ||
3115 | rseq_bug("cpu_id comparison failed"); | ||
3116 | error2: | ||
3117 | - rseq_workaround_gcc_asm_size_guess(); | ||
3118 | rseq_bug("expected value comparison failed"); | ||
3119 | #endif | ||
3120 | } | ||
3121 | diff --git a/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h b/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h | ||
3122 | new file mode 100644 | ||
3123 | index 0000000000000..263eee84fb760 | ||
3124 | --- /dev/null | ||
3125 | +++ b/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h | ||
3126 | @@ -0,0 +1,30 @@ | ||
3127 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ | ||
3128 | +/* | ||
3129 | + * rseq-ppc-thread-pointer.h | ||
3130 | + * | ||
3131 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | ||
3132 | + */ | ||
3133 | + | ||
3134 | +#ifndef _RSEQ_PPC_THREAD_POINTER | ||
3135 | +#define _RSEQ_PPC_THREAD_POINTER | ||
3136 | + | ||
3137 | +#ifdef __cplusplus | ||
3138 | +extern "C" { | ||
3139 | +#endif | ||
3140 | + | ||
3141 | +static inline void *rseq_thread_pointer(void) | ||
3142 | +{ | ||
3143 | +#ifdef __powerpc64__ | ||
3144 | + register void *__result asm ("r13"); | ||
3145 | +#else | ||
3146 | + register void *__result asm ("r2"); | ||
3147 | +#endif | ||
3148 | + asm ("" : "=r" (__result)); | ||
3149 | + return __result; | ||
3150 | +} | ||
3151 | + | ||
3152 | +#ifdef __cplusplus | ||
3153 | +} | ||
3154 | +#endif | ||
3155 | + | ||
3156 | +#endif | ||
3157 | diff --git a/tools/testing/selftests/rseq/rseq-ppc.h b/tools/testing/selftests/rseq/rseq-ppc.h | ||
3158 | index 76be90196fe4f..bab8e0b9fb115 100644 | ||
3159 | --- a/tools/testing/selftests/rseq/rseq-ppc.h | ||
3160 | +++ b/tools/testing/selftests/rseq/rseq-ppc.h | ||
3161 | @@ -47,10 +47,13 @@ do { \ | ||
3162 | |||
3163 | #ifdef __PPC64__ | ||
3164 | |||
3165 | -#define STORE_WORD "std " | ||
3166 | -#define LOAD_WORD "ld " | ||
3167 | -#define LOADX_WORD "ldx " | ||
3168 | -#define CMP_WORD "cmpd " | ||
3169 | +#define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ | ||
3170 | +#define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ | ||
3171 | +#define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | ||
3172 | +#define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | ||
3173 | +#define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */ | ||
3174 | +#define RSEQ_CMP_LONG "cmpd " | ||
3175 | +#define RSEQ_CMP_LONG_INT "cmpdi " | ||
3176 | |||
3177 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | ||
3178 | start_ip, post_commit_offset, abort_ip) \ | ||
3179 | @@ -89,10 +92,13 @@ do { \ | ||
3180 | |||
3181 | #else /* #ifdef __PPC64__ */ | ||
3182 | |||
3183 | -#define STORE_WORD "stw " | ||
3184 | -#define LOAD_WORD "lwz " | ||
3185 | -#define LOADX_WORD "lwzx " | ||
3186 | -#define CMP_WORD "cmpw " | ||
3187 | +#define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ | ||
3188 | +#define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */ | ||
3189 | +#define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | ||
3190 | +#define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */ | ||
3191 | +#define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */ | ||
3192 | +#define RSEQ_CMP_LONG "cmpw " | ||
3193 | +#define RSEQ_CMP_LONG_INT "cmpwi " | ||
3194 | |||
3195 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | ||
3196 | start_ip, post_commit_offset, abort_ip) \ | ||
3197 | @@ -125,7 +131,7 @@ do { \ | ||
3198 | RSEQ_INJECT_ASM(1) \ | ||
3199 | "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \ | ||
3200 | "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ | ||
3201 | - "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ | ||
3202 | + RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ | ||
3203 | __rseq_str(label) ":\n\t" | ||
3204 | |||
3205 | #endif /* #ifdef __PPC64__ */ | ||
3206 | @@ -136,7 +142,7 @@ do { \ | ||
3207 | |||
3208 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ | ||
3209 | RSEQ_INJECT_ASM(2) \ | ||
3210 | - "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ | ||
3211 | + RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ | ||
3212 | "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ | ||
3213 | "bne- cr7, " __rseq_str(label) "\n\t" | ||
3214 | |||
3215 | @@ -153,25 +159,25 @@ do { \ | ||
3216 | * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7) | ||
3217 | */ | ||
3218 | #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ | ||
3219 | - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ | ||
3220 | - CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ | ||
3221 | + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ | ||
3222 | + RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ | ||
3223 | "bne- cr7, " __rseq_str(label) "\n\t" | ||
3224 | |||
3225 | #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \ | ||
3226 | - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ | ||
3227 | - CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ | ||
3228 | + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ | ||
3229 | + RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ | ||
3230 | "beq- cr7, " __rseq_str(label) "\n\t" | ||
3231 | |||
3232 | #define RSEQ_ASM_OP_STORE(value, var) \ | ||
3233 | - STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" | ||
3234 | + RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" | ||
3235 | |||
3236 | /* Load @var to r17 */ | ||
3237 | #define RSEQ_ASM_OP_R_LOAD(var) \ | ||
3238 | - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" | ||
3239 | + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" | ||
3240 | |||
3241 | /* Store r17 to @var */ | ||
3242 | #define RSEQ_ASM_OP_R_STORE(var) \ | ||
3243 | - STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" | ||
3244 | + RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" | ||
3245 | |||
3246 | /* Add @count to r17 */ | ||
3247 | #define RSEQ_ASM_OP_R_ADD(count) \ | ||
3248 | @@ -179,11 +185,11 @@ do { \ | ||
3249 | |||
3250 | /* Load (r17 + voffp) to r17 */ | ||
3251 | #define RSEQ_ASM_OP_R_LOADX(voffp) \ | ||
3252 | - LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" | ||
3253 | + RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" | ||
3254 | |||
3255 | /* TODO: implement a faster memcpy. */ | ||
3256 | #define RSEQ_ASM_OP_R_MEMCPY() \ | ||
3257 | - "cmpdi %%r19, 0\n\t" \ | ||
3258 | + RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ | ||
3259 | "beq 333f\n\t" \ | ||
3260 | "addi %%r20, %%r20, -1\n\t" \ | ||
3261 | "addi %%r21, %%r21, -1\n\t" \ | ||
3262 | @@ -191,16 +197,16 @@ do { \ | ||
3263 | "lbzu %%r18, 1(%%r20)\n\t" \ | ||
3264 | "stbu %%r18, 1(%%r21)\n\t" \ | ||
3265 | "addi %%r19, %%r19, -1\n\t" \ | ||
3266 | - "cmpdi %%r19, 0\n\t" \ | ||
3267 | + RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ | ||
3268 | "bne 222b\n\t" \ | ||
3269 | "333:\n\t" \ | ||
3270 | |||
3271 | #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ | ||
3272 | - STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ | ||
3273 | + RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ | ||
3274 | __rseq_str(post_commit_label) ":\n\t" | ||
3275 | |||
3276 | #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ | ||
3277 | - STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ | ||
3278 | + RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ | ||
3279 | __rseq_str(post_commit_label) ":\n\t" | ||
3280 | |||
3281 | static inline __attribute__((always_inline)) | ||
3282 | @@ -235,8 +241,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
3283 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
3284 | : /* gcc asm goto does not allow outputs */ | ||
3285 | : [cpu_id] "r" (cpu), | ||
3286 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3287 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3288 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3289 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3290 | [v] "m" (*v), | ||
3291 | [expect] "r" (expect), | ||
3292 | [newv] "r" (newv) | ||
3293 | @@ -248,23 +254,28 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
3294 | , error1, error2 | ||
3295 | #endif | ||
3296 | ); | ||
3297 | + rseq_after_asm_goto(); | ||
3298 | return 0; | ||
3299 | abort: | ||
3300 | + rseq_after_asm_goto(); | ||
3301 | RSEQ_INJECT_FAILED | ||
3302 | return -1; | ||
3303 | cmpfail: | ||
3304 | + rseq_after_asm_goto(); | ||
3305 | return 1; | ||
3306 | #ifdef RSEQ_COMPARE_TWICE | ||
3307 | error1: | ||
3308 | + rseq_after_asm_goto(); | ||
3309 | rseq_bug("cpu_id comparison failed"); | ||
3310 | error2: | ||
3311 | + rseq_after_asm_goto(); | ||
3312 | rseq_bug("expected value comparison failed"); | ||
3313 | #endif | ||
3314 | } | ||
3315 | |||
3316 | static inline __attribute__((always_inline)) | ||
3317 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3318 | - off_t voffp, intptr_t *load, int cpu) | ||
3319 | + long voffp, intptr_t *load, int cpu) | ||
3320 | { | ||
3321 | RSEQ_INJECT_C(9) | ||
3322 | |||
3323 | @@ -301,8 +312,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3324 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
3325 | : /* gcc asm goto does not allow outputs */ | ||
3326 | : [cpu_id] "r" (cpu), | ||
3327 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3328 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3329 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3330 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3331 | /* final store input */ | ||
3332 | [v] "m" (*v), | ||
3333 | [expectnot] "r" (expectnot), | ||
3334 | @@ -316,16 +327,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3335 | , error1, error2 | ||
3336 | #endif | ||
3337 | ); | ||
3338 | + rseq_after_asm_goto(); | ||
3339 | return 0; | ||
3340 | abort: | ||
3341 | + rseq_after_asm_goto(); | ||
3342 | RSEQ_INJECT_FAILED | ||
3343 | return -1; | ||
3344 | cmpfail: | ||
3345 | + rseq_after_asm_goto(); | ||
3346 | return 1; | ||
3347 | #ifdef RSEQ_COMPARE_TWICE | ||
3348 | error1: | ||
3349 | + rseq_after_asm_goto(); | ||
3350 | rseq_bug("cpu_id comparison failed"); | ||
3351 | error2: | ||
3352 | + rseq_after_asm_goto(); | ||
3353 | rseq_bug("expected value comparison failed"); | ||
3354 | #endif | ||
3355 | } | ||
3356 | @@ -359,8 +375,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
3357 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
3358 | : /* gcc asm goto does not allow outputs */ | ||
3359 | : [cpu_id] "r" (cpu), | ||
3360 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3361 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3362 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3363 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3364 | /* final store input */ | ||
3365 | [v] "m" (*v), | ||
3366 | [count] "r" (count) | ||
3367 | @@ -372,12 +388,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
3368 | , error1 | ||
3369 | #endif | ||
3370 | ); | ||
3371 | + rseq_after_asm_goto(); | ||
3372 | return 0; | ||
3373 | abort: | ||
3374 | + rseq_after_asm_goto(); | ||
3375 | RSEQ_INJECT_FAILED | ||
3376 | return -1; | ||
3377 | #ifdef RSEQ_COMPARE_TWICE | ||
3378 | error1: | ||
3379 | + rseq_after_asm_goto(); | ||
3380 | rseq_bug("cpu_id comparison failed"); | ||
3381 | #endif | ||
3382 | } | ||
3383 | @@ -419,8 +438,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
3384 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
3385 | : /* gcc asm goto does not allow outputs */ | ||
3386 | : [cpu_id] "r" (cpu), | ||
3387 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3388 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3389 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3390 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3391 | /* try store input */ | ||
3392 | [v2] "m" (*v2), | ||
3393 | [newv2] "r" (newv2), | ||
3394 | @@ -436,16 +455,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
3395 | , error1, error2 | ||
3396 | #endif | ||
3397 | ); | ||
3398 | + rseq_after_asm_goto(); | ||
3399 | return 0; | ||
3400 | abort: | ||
3401 | + rseq_after_asm_goto(); | ||
3402 | RSEQ_INJECT_FAILED | ||
3403 | return -1; | ||
3404 | cmpfail: | ||
3405 | + rseq_after_asm_goto(); | ||
3406 | return 1; | ||
3407 | #ifdef RSEQ_COMPARE_TWICE | ||
3408 | error1: | ||
3409 | + rseq_after_asm_goto(); | ||
3410 | rseq_bug("cpu_id comparison failed"); | ||
3411 | error2: | ||
3412 | + rseq_after_asm_goto(); | ||
3413 | rseq_bug("expected value comparison failed"); | ||
3414 | #endif | ||
3415 | } | ||
3416 | @@ -489,8 +513,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
3417 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
3418 | : /* gcc asm goto does not allow outputs */ | ||
3419 | : [cpu_id] "r" (cpu), | ||
3420 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3421 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3422 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3423 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3424 | /* try store input */ | ||
3425 | [v2] "m" (*v2), | ||
3426 | [newv2] "r" (newv2), | ||
3427 | @@ -506,16 +530,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
3428 | , error1, error2 | ||
3429 | #endif | ||
3430 | ); | ||
3431 | + rseq_after_asm_goto(); | ||
3432 | return 0; | ||
3433 | abort: | ||
3434 | + rseq_after_asm_goto(); | ||
3435 | RSEQ_INJECT_FAILED | ||
3436 | return -1; | ||
3437 | cmpfail: | ||
3438 | + rseq_after_asm_goto(); | ||
3439 | return 1; | ||
3440 | #ifdef RSEQ_COMPARE_TWICE | ||
3441 | error1: | ||
3442 | + rseq_after_asm_goto(); | ||
3443 | rseq_bug("cpu_id comparison failed"); | ||
3444 | error2: | ||
3445 | + rseq_after_asm_goto(); | ||
3446 | rseq_bug("expected value comparison failed"); | ||
3447 | #endif | ||
3448 | } | ||
3449 | @@ -560,8 +589,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
3450 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
3451 | : /* gcc asm goto does not allow outputs */ | ||
3452 | : [cpu_id] "r" (cpu), | ||
3453 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3454 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3455 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3456 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3457 | /* cmp2 input */ | ||
3458 | [v2] "m" (*v2), | ||
3459 | [expect2] "r" (expect2), | ||
3460 | @@ -577,18 +606,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
3461 | , error1, error2, error3 | ||
3462 | #endif | ||
3463 | ); | ||
3464 | + rseq_after_asm_goto(); | ||
3465 | return 0; | ||
3466 | abort: | ||
3467 | + rseq_after_asm_goto(); | ||
3468 | RSEQ_INJECT_FAILED | ||
3469 | return -1; | ||
3470 | cmpfail: | ||
3471 | + rseq_after_asm_goto(); | ||
3472 | return 1; | ||
3473 | #ifdef RSEQ_COMPARE_TWICE | ||
3474 | error1: | ||
3475 | + rseq_after_asm_goto(); | ||
3476 | rseq_bug("cpu_id comparison failed"); | ||
3477 | error2: | ||
3478 | + rseq_after_asm_goto(); | ||
3479 | rseq_bug("1st expected value comparison failed"); | ||
3480 | error3: | ||
3481 | + rseq_after_asm_goto(); | ||
3482 | rseq_bug("2nd expected value comparison failed"); | ||
3483 | #endif | ||
3484 | } | ||
3485 | @@ -635,8 +670,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
3486 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
3487 | : /* gcc asm goto does not allow outputs */ | ||
3488 | : [cpu_id] "r" (cpu), | ||
3489 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3490 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3491 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3492 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3493 | /* final store input */ | ||
3494 | [v] "m" (*v), | ||
3495 | [expect] "r" (expect), | ||
3496 | @@ -653,16 +688,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
3497 | , error1, error2 | ||
3498 | #endif | ||
3499 | ); | ||
3500 | + rseq_after_asm_goto(); | ||
3501 | return 0; | ||
3502 | abort: | ||
3503 | + rseq_after_asm_goto(); | ||
3504 | RSEQ_INJECT_FAILED | ||
3505 | return -1; | ||
3506 | cmpfail: | ||
3507 | + rseq_after_asm_goto(); | ||
3508 | return 1; | ||
3509 | #ifdef RSEQ_COMPARE_TWICE | ||
3510 | error1: | ||
3511 | + rseq_after_asm_goto(); | ||
3512 | rseq_bug("cpu_id comparison failed"); | ||
3513 | error2: | ||
3514 | + rseq_after_asm_goto(); | ||
3515 | rseq_bug("expected value comparison failed"); | ||
3516 | #endif | ||
3517 | } | ||
3518 | @@ -711,8 +751,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
3519 | RSEQ_ASM_DEFINE_ABORT(4, abort) | ||
3520 | : /* gcc asm goto does not allow outputs */ | ||
3521 | : [cpu_id] "r" (cpu), | ||
3522 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3523 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3524 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3525 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3526 | /* final store input */ | ||
3527 | [v] "m" (*v), | ||
3528 | [expect] "r" (expect), | ||
3529 | @@ -729,23 +769,23 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
3530 | , error1, error2 | ||
3531 | #endif | ||
3532 | ); | ||
3533 | + rseq_after_asm_goto(); | ||
3534 | return 0; | ||
3535 | abort: | ||
3536 | + rseq_after_asm_goto(); | ||
3537 | RSEQ_INJECT_FAILED | ||
3538 | return -1; | ||
3539 | cmpfail: | ||
3540 | + rseq_after_asm_goto(); | ||
3541 | return 1; | ||
3542 | #ifdef RSEQ_COMPARE_TWICE | ||
3543 | error1: | ||
3544 | + rseq_after_asm_goto(); | ||
3545 | rseq_bug("cpu_id comparison failed"); | ||
3546 | error2: | ||
3547 | + rseq_after_asm_goto(); | ||
3548 | rseq_bug("expected value comparison failed"); | ||
3549 | #endif | ||
3550 | } | ||
3551 | |||
3552 | -#undef STORE_WORD | ||
3553 | -#undef LOAD_WORD | ||
3554 | -#undef LOADX_WORD | ||
3555 | -#undef CMP_WORD | ||
3556 | - | ||
3557 | #endif /* !RSEQ_SKIP_FASTPATH */ | ||
3558 | diff --git a/tools/testing/selftests/rseq/rseq-s390.h b/tools/testing/selftests/rseq/rseq-s390.h | ||
3559 | index 8ef94ad1cbb45..4e6dc5f0cb429 100644 | ||
3560 | --- a/tools/testing/selftests/rseq/rseq-s390.h | ||
3561 | +++ b/tools/testing/selftests/rseq/rseq-s390.h | ||
3562 | @@ -165,8 +165,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
3563 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
3564 | : /* gcc asm goto does not allow outputs */ | ||
3565 | : [cpu_id] "r" (cpu), | ||
3566 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3567 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3568 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3569 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3570 | [v] "m" (*v), | ||
3571 | [expect] "r" (expect), | ||
3572 | [newv] "r" (newv) | ||
3573 | @@ -178,16 +178,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
3574 | , error1, error2 | ||
3575 | #endif | ||
3576 | ); | ||
3577 | + rseq_after_asm_goto(); | ||
3578 | return 0; | ||
3579 | abort: | ||
3580 | + rseq_after_asm_goto(); | ||
3581 | RSEQ_INJECT_FAILED | ||
3582 | return -1; | ||
3583 | cmpfail: | ||
3584 | + rseq_after_asm_goto(); | ||
3585 | return 1; | ||
3586 | #ifdef RSEQ_COMPARE_TWICE | ||
3587 | error1: | ||
3588 | + rseq_after_asm_goto(); | ||
3589 | rseq_bug("cpu_id comparison failed"); | ||
3590 | error2: | ||
3591 | + rseq_after_asm_goto(); | ||
3592 | rseq_bug("expected value comparison failed"); | ||
3593 | #endif | ||
3594 | } | ||
3595 | @@ -198,7 +203,7 @@ error2: | ||
3596 | */ | ||
3597 | static inline __attribute__((always_inline)) | ||
3598 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3599 | - off_t voffp, intptr_t *load, int cpu) | ||
3600 | + long voffp, intptr_t *load, int cpu) | ||
3601 | { | ||
3602 | RSEQ_INJECT_C(9) | ||
3603 | |||
3604 | @@ -233,8 +238,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3605 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
3606 | : /* gcc asm goto does not allow outputs */ | ||
3607 | : [cpu_id] "r" (cpu), | ||
3608 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3609 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3610 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3611 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3612 | /* final store input */ | ||
3613 | [v] "m" (*v), | ||
3614 | [expectnot] "r" (expectnot), | ||
3615 | @@ -248,16 +253,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3616 | , error1, error2 | ||
3617 | #endif | ||
3618 | ); | ||
3619 | + rseq_after_asm_goto(); | ||
3620 | return 0; | ||
3621 | abort: | ||
3622 | + rseq_after_asm_goto(); | ||
3623 | RSEQ_INJECT_FAILED | ||
3624 | return -1; | ||
3625 | cmpfail: | ||
3626 | + rseq_after_asm_goto(); | ||
3627 | return 1; | ||
3628 | #ifdef RSEQ_COMPARE_TWICE | ||
3629 | error1: | ||
3630 | + rseq_after_asm_goto(); | ||
3631 | rseq_bug("cpu_id comparison failed"); | ||
3632 | error2: | ||
3633 | + rseq_after_asm_goto(); | ||
3634 | rseq_bug("expected value comparison failed"); | ||
3635 | #endif | ||
3636 | } | ||
3637 | @@ -288,8 +298,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
3638 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
3639 | : /* gcc asm goto does not allow outputs */ | ||
3640 | : [cpu_id] "r" (cpu), | ||
3641 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3642 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3643 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3644 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3645 | /* final store input */ | ||
3646 | [v] "m" (*v), | ||
3647 | [count] "r" (count) | ||
3648 | @@ -301,12 +311,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
3649 | , error1 | ||
3650 | #endif | ||
3651 | ); | ||
3652 | + rseq_after_asm_goto(); | ||
3653 | return 0; | ||
3654 | abort: | ||
3655 | + rseq_after_asm_goto(); | ||
3656 | RSEQ_INJECT_FAILED | ||
3657 | return -1; | ||
3658 | #ifdef RSEQ_COMPARE_TWICE | ||
3659 | error1: | ||
3660 | + rseq_after_asm_goto(); | ||
3661 | rseq_bug("cpu_id comparison failed"); | ||
3662 | #endif | ||
3663 | } | ||
3664 | @@ -347,8 +360,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
3665 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
3666 | : /* gcc asm goto does not allow outputs */ | ||
3667 | : [cpu_id] "r" (cpu), | ||
3668 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3669 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3670 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3671 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3672 | /* try store input */ | ||
3673 | [v2] "m" (*v2), | ||
3674 | [newv2] "r" (newv2), | ||
3675 | @@ -364,16 +377,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
3676 | , error1, error2 | ||
3677 | #endif | ||
3678 | ); | ||
3679 | + rseq_after_asm_goto(); | ||
3680 | return 0; | ||
3681 | abort: | ||
3682 | + rseq_after_asm_goto(); | ||
3683 | RSEQ_INJECT_FAILED | ||
3684 | return -1; | ||
3685 | cmpfail: | ||
3686 | + rseq_after_asm_goto(); | ||
3687 | return 1; | ||
3688 | #ifdef RSEQ_COMPARE_TWICE | ||
3689 | error1: | ||
3690 | + rseq_after_asm_goto(); | ||
3691 | rseq_bug("cpu_id comparison failed"); | ||
3692 | error2: | ||
3693 | + rseq_after_asm_goto(); | ||
3694 | rseq_bug("expected value comparison failed"); | ||
3695 | #endif | ||
3696 | } | ||
3697 | @@ -426,8 +444,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
3698 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
3699 | : /* gcc asm goto does not allow outputs */ | ||
3700 | : [cpu_id] "r" (cpu), | ||
3701 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3702 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3703 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3704 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3705 | /* cmp2 input */ | ||
3706 | [v2] "m" (*v2), | ||
3707 | [expect2] "r" (expect2), | ||
3708 | @@ -443,18 +461,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
3709 | , error1, error2, error3 | ||
3710 | #endif | ||
3711 | ); | ||
3712 | + rseq_after_asm_goto(); | ||
3713 | return 0; | ||
3714 | abort: | ||
3715 | + rseq_after_asm_goto(); | ||
3716 | RSEQ_INJECT_FAILED | ||
3717 | return -1; | ||
3718 | cmpfail: | ||
3719 | + rseq_after_asm_goto(); | ||
3720 | return 1; | ||
3721 | #ifdef RSEQ_COMPARE_TWICE | ||
3722 | error1: | ||
3723 | + rseq_after_asm_goto(); | ||
3724 | rseq_bug("cpu_id comparison failed"); | ||
3725 | error2: | ||
3726 | + rseq_after_asm_goto(); | ||
3727 | rseq_bug("1st expected value comparison failed"); | ||
3728 | error3: | ||
3729 | + rseq_after_asm_goto(); | ||
3730 | rseq_bug("2nd expected value comparison failed"); | ||
3731 | #endif | ||
3732 | } | ||
3733 | @@ -534,8 +558,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
3734 | #endif | ||
3735 | : /* gcc asm goto does not allow outputs */ | ||
3736 | : [cpu_id] "r" (cpu), | ||
3737 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), | ||
3738 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), | ||
3739 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), | ||
3740 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), | ||
3741 | /* final store input */ | ||
3742 | [v] "m" (*v), | ||
3743 | [expect] "r" (expect), | ||
3744 | @@ -555,16 +579,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
3745 | , error1, error2 | ||
3746 | #endif | ||
3747 | ); | ||
3748 | + rseq_after_asm_goto(); | ||
3749 | return 0; | ||
3750 | abort: | ||
3751 | + rseq_after_asm_goto(); | ||
3752 | RSEQ_INJECT_FAILED | ||
3753 | return -1; | ||
3754 | cmpfail: | ||
3755 | + rseq_after_asm_goto(); | ||
3756 | return 1; | ||
3757 | #ifdef RSEQ_COMPARE_TWICE | ||
3758 | error1: | ||
3759 | + rseq_after_asm_goto(); | ||
3760 | rseq_bug("cpu_id comparison failed"); | ||
3761 | error2: | ||
3762 | + rseq_after_asm_goto(); | ||
3763 | rseq_bug("expected value comparison failed"); | ||
3764 | #endif | ||
3765 | } | ||
3766 | diff --git a/tools/testing/selftests/rseq/rseq-skip.h b/tools/testing/selftests/rseq/rseq-skip.h | ||
3767 | index 72750b5905a96..7b53dac1fcdd9 100644 | ||
3768 | --- a/tools/testing/selftests/rseq/rseq-skip.h | ||
3769 | +++ b/tools/testing/selftests/rseq/rseq-skip.h | ||
3770 | @@ -13,7 +13,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
3771 | |||
3772 | static inline __attribute__((always_inline)) | ||
3773 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3774 | - off_t voffp, intptr_t *load, int cpu) | ||
3775 | + long voffp, intptr_t *load, int cpu) | ||
3776 | { | ||
3777 | return -1; | ||
3778 | } | ||
3779 | diff --git a/tools/testing/selftests/rseq/rseq-thread-pointer.h b/tools/testing/selftests/rseq/rseq-thread-pointer.h | ||
3780 | new file mode 100644 | ||
3781 | index 0000000000000..977c25d758b2a | ||
3782 | --- /dev/null | ||
3783 | +++ b/tools/testing/selftests/rseq/rseq-thread-pointer.h | ||
3784 | @@ -0,0 +1,19 @@ | ||
3785 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ | ||
3786 | +/* | ||
3787 | + * rseq-thread-pointer.h | ||
3788 | + * | ||
3789 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | ||
3790 | + */ | ||
3791 | + | ||
3792 | +#ifndef _RSEQ_THREAD_POINTER | ||
3793 | +#define _RSEQ_THREAD_POINTER | ||
3794 | + | ||
3795 | +#if defined(__x86_64__) || defined(__i386__) | ||
3796 | +#include "rseq-x86-thread-pointer.h" | ||
3797 | +#elif defined(__PPC__) | ||
3798 | +#include "rseq-ppc-thread-pointer.h" | ||
3799 | +#else | ||
3800 | +#include "rseq-generic-thread-pointer.h" | ||
3801 | +#endif | ||
3802 | + | ||
3803 | +#endif | ||
3804 | diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h | ||
3805 | new file mode 100644 | ||
3806 | index 0000000000000..d3133587d9968 | ||
3807 | --- /dev/null | ||
3808 | +++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h | ||
3809 | @@ -0,0 +1,40 @@ | ||
3810 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ | ||
3811 | +/* | ||
3812 | + * rseq-x86-thread-pointer.h | ||
3813 | + * | ||
3814 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | ||
3815 | + */ | ||
3816 | + | ||
3817 | +#ifndef _RSEQ_X86_THREAD_POINTER | ||
3818 | +#define _RSEQ_X86_THREAD_POINTER | ||
3819 | + | ||
3820 | +#include <features.h> | ||
3821 | + | ||
3822 | +#ifdef __cplusplus | ||
3823 | +extern "C" { | ||
3824 | +#endif | ||
3825 | + | ||
3826 | +#if __GNUC_PREREQ (11, 1) | ||
3827 | +static inline void *rseq_thread_pointer(void) | ||
3828 | +{ | ||
3829 | + return __builtin_thread_pointer(); | ||
3830 | +} | ||
3831 | +#else | ||
3832 | +static inline void *rseq_thread_pointer(void) | ||
3833 | +{ | ||
3834 | + void *__result; | ||
3835 | + | ||
3836 | +# ifdef __x86_64__ | ||
3837 | + __asm__ ("mov %%fs:0, %0" : "=r" (__result)); | ||
3838 | +# else | ||
3839 | + __asm__ ("mov %%gs:0, %0" : "=r" (__result)); | ||
3840 | +# endif | ||
3841 | + return __result; | ||
3842 | +} | ||
3843 | +#endif /* !GCC 11 */ | ||
3844 | + | ||
3845 | +#ifdef __cplusplus | ||
3846 | +} | ||
3847 | +#endif | ||
3848 | + | ||
3849 | +#endif | ||
3850 | diff --git a/tools/testing/selftests/rseq/rseq-x86.h b/tools/testing/selftests/rseq/rseq-x86.h | ||
3851 | index b2da6004fe307..bd01dc41ca130 100644 | ||
3852 | --- a/tools/testing/selftests/rseq/rseq-x86.h | ||
3853 | +++ b/tools/testing/selftests/rseq/rseq-x86.h | ||
3854 | @@ -28,6 +28,8 @@ | ||
3855 | |||
3856 | #ifdef __x86_64__ | ||
3857 | |||
3858 | +#define RSEQ_ASM_TP_SEGMENT %%fs | ||
3859 | + | ||
3860 | #define rseq_smp_mb() \ | ||
3861 | __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") | ||
3862 | #define rseq_smp_rmb() rseq_barrier() | ||
3863 | @@ -123,14 +125,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
3864 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | ||
3865 | #endif | ||
3866 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
3867 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
3868 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
3869 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
3870 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
3871 | RSEQ_INJECT_ASM(3) | ||
3872 | "cmpq %[v], %[expect]\n\t" | ||
3873 | "jnz %l[cmpfail]\n\t" | ||
3874 | RSEQ_INJECT_ASM(4) | ||
3875 | #ifdef RSEQ_COMPARE_TWICE | ||
3876 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
3877 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
3878 | "cmpq %[v], %[expect]\n\t" | ||
3879 | "jnz %l[error2]\n\t" | ||
3880 | #endif | ||
3881 | @@ -141,7 +143,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
3882 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
3883 | : /* gcc asm goto does not allow outputs */ | ||
3884 | : [cpu_id] "r" (cpu), | ||
3885 | - [rseq_abi] "r" (&__rseq_abi), | ||
3886 | + [rseq_offset] "r" (rseq_offset), | ||
3887 | [v] "m" (*v), | ||
3888 | [expect] "r" (expect), | ||
3889 | [newv] "r" (newv) | ||
3890 | @@ -152,16 +154,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
3891 | , error1, error2 | ||
3892 | #endif | ||
3893 | ); | ||
3894 | + rseq_after_asm_goto(); | ||
3895 | return 0; | ||
3896 | abort: | ||
3897 | + rseq_after_asm_goto(); | ||
3898 | RSEQ_INJECT_FAILED | ||
3899 | return -1; | ||
3900 | cmpfail: | ||
3901 | + rseq_after_asm_goto(); | ||
3902 | return 1; | ||
3903 | #ifdef RSEQ_COMPARE_TWICE | ||
3904 | error1: | ||
3905 | + rseq_after_asm_goto(); | ||
3906 | rseq_bug("cpu_id comparison failed"); | ||
3907 | error2: | ||
3908 | + rseq_after_asm_goto(); | ||
3909 | rseq_bug("expected value comparison failed"); | ||
3910 | #endif | ||
3911 | } | ||
3912 | @@ -172,7 +179,7 @@ error2: | ||
3913 | */ | ||
3914 | static inline __attribute__((always_inline)) | ||
3915 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3916 | - off_t voffp, intptr_t *load, int cpu) | ||
3917 | + long voffp, intptr_t *load, int cpu) | ||
3918 | { | ||
3919 | RSEQ_INJECT_C(9) | ||
3920 | |||
3921 | @@ -184,15 +191,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3922 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | ||
3923 | #endif | ||
3924 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
3925 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
3926 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
3927 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
3928 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
3929 | RSEQ_INJECT_ASM(3) | ||
3930 | "movq %[v], %%rbx\n\t" | ||
3931 | "cmpq %%rbx, %[expectnot]\n\t" | ||
3932 | "je %l[cmpfail]\n\t" | ||
3933 | RSEQ_INJECT_ASM(4) | ||
3934 | #ifdef RSEQ_COMPARE_TWICE | ||
3935 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
3936 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
3937 | "movq %[v], %%rbx\n\t" | ||
3938 | "cmpq %%rbx, %[expectnot]\n\t" | ||
3939 | "je %l[error2]\n\t" | ||
3940 | @@ -207,7 +214,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3941 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
3942 | : /* gcc asm goto does not allow outputs */ | ||
3943 | : [cpu_id] "r" (cpu), | ||
3944 | - [rseq_abi] "r" (&__rseq_abi), | ||
3945 | + [rseq_offset] "r" (rseq_offset), | ||
3946 | /* final store input */ | ||
3947 | [v] "m" (*v), | ||
3948 | [expectnot] "r" (expectnot), | ||
3949 | @@ -220,16 +227,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
3950 | , error1, error2 | ||
3951 | #endif | ||
3952 | ); | ||
3953 | + rseq_after_asm_goto(); | ||
3954 | return 0; | ||
3955 | abort: | ||
3956 | + rseq_after_asm_goto(); | ||
3957 | RSEQ_INJECT_FAILED | ||
3958 | return -1; | ||
3959 | cmpfail: | ||
3960 | + rseq_after_asm_goto(); | ||
3961 | return 1; | ||
3962 | #ifdef RSEQ_COMPARE_TWICE | ||
3963 | error1: | ||
3964 | + rseq_after_asm_goto(); | ||
3965 | rseq_bug("cpu_id comparison failed"); | ||
3966 | error2: | ||
3967 | + rseq_after_asm_goto(); | ||
3968 | rseq_bug("expected value comparison failed"); | ||
3969 | #endif | ||
3970 | } | ||
3971 | @@ -245,11 +257,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
3972 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | ||
3973 | #endif | ||
3974 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
3975 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
3976 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
3977 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
3978 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
3979 | RSEQ_INJECT_ASM(3) | ||
3980 | #ifdef RSEQ_COMPARE_TWICE | ||
3981 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
3982 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
3983 | #endif | ||
3984 | /* final store */ | ||
3985 | "addq %[count], %[v]\n\t" | ||
3986 | @@ -258,7 +270,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
3987 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
3988 | : /* gcc asm goto does not allow outputs */ | ||
3989 | : [cpu_id] "r" (cpu), | ||
3990 | - [rseq_abi] "r" (&__rseq_abi), | ||
3991 | + [rseq_offset] "r" (rseq_offset), | ||
3992 | /* final store input */ | ||
3993 | [v] "m" (*v), | ||
3994 | [count] "er" (count) | ||
3995 | @@ -267,6 +279,66 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
3996 | : abort | ||
3997 | #ifdef RSEQ_COMPARE_TWICE | ||
3998 | , error1 | ||
3999 | +#endif | ||
4000 | + ); | ||
4001 | + rseq_after_asm_goto(); | ||
4002 | + return 0; | ||
4003 | +abort: | ||
4004 | + rseq_after_asm_goto(); | ||
4005 | + RSEQ_INJECT_FAILED | ||
4006 | + return -1; | ||
4007 | +#ifdef RSEQ_COMPARE_TWICE | ||
4008 | +error1: | ||
4009 | + rseq_after_asm_goto(); | ||
4010 | + rseq_bug("cpu_id comparison failed"); | ||
4011 | +#endif | ||
4012 | +} | ||
4013 | + | ||
4014 | +#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV | ||
4015 | + | ||
4016 | +/* | ||
4017 | + * pval = *(ptr+off) | ||
4018 | + * *pval += inc; | ||
4019 | + */ | ||
4020 | +static inline __attribute__((always_inline)) | ||
4021 | +int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu) | ||
4022 | +{ | ||
4023 | + RSEQ_INJECT_C(9) | ||
4024 | + | ||
4025 | + __asm__ __volatile__ goto ( | ||
4026 | + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | ||
4027 | +#ifdef RSEQ_COMPARE_TWICE | ||
4028 | + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | ||
4029 | +#endif | ||
4030 | + /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4031 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4032 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4033 | + RSEQ_INJECT_ASM(3) | ||
4034 | +#ifdef RSEQ_COMPARE_TWICE | ||
4035 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4036 | +#endif | ||
4037 | + /* get p+v */ | ||
4038 | + "movq %[ptr], %%rbx\n\t" | ||
4039 | + "addq %[off], %%rbx\n\t" | ||
4040 | + /* get pv */ | ||
4041 | + "movq (%%rbx), %%rcx\n\t" | ||
4042 | + /* *pv += inc */ | ||
4043 | + "addq %[inc], (%%rcx)\n\t" | ||
4044 | + "2:\n\t" | ||
4045 | + RSEQ_INJECT_ASM(4) | ||
4046 | + RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4047 | + : /* gcc asm goto does not allow outputs */ | ||
4048 | + : [cpu_id] "r" (cpu), | ||
4049 | + [rseq_offset] "r" (rseq_offset), | ||
4050 | + /* final store input */ | ||
4051 | + [ptr] "m" (*ptr), | ||
4052 | + [off] "er" (off), | ||
4053 | + [inc] "er" (inc) | ||
4054 | + : "memory", "cc", "rax", "rbx", "rcx" | ||
4055 | + RSEQ_INJECT_CLOBBER | ||
4056 | + : abort | ||
4057 | +#ifdef RSEQ_COMPARE_TWICE | ||
4058 | + , error1 | ||
4059 | #endif | ||
4060 | ); | ||
4061 | return 0; | ||
4062 | @@ -294,14 +366,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
4063 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | ||
4064 | #endif | ||
4065 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4066 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4067 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4068 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4069 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4070 | RSEQ_INJECT_ASM(3) | ||
4071 | "cmpq %[v], %[expect]\n\t" | ||
4072 | "jnz %l[cmpfail]\n\t" | ||
4073 | RSEQ_INJECT_ASM(4) | ||
4074 | #ifdef RSEQ_COMPARE_TWICE | ||
4075 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
4076 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4077 | "cmpq %[v], %[expect]\n\t" | ||
4078 | "jnz %l[error2]\n\t" | ||
4079 | #endif | ||
4080 | @@ -315,7 +387,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
4081 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4082 | : /* gcc asm goto does not allow outputs */ | ||
4083 | : [cpu_id] "r" (cpu), | ||
4084 | - [rseq_abi] "r" (&__rseq_abi), | ||
4085 | + [rseq_offset] "r" (rseq_offset), | ||
4086 | /* try store input */ | ||
4087 | [v2] "m" (*v2), | ||
4088 | [newv2] "r" (newv2), | ||
4089 | @@ -330,16 +402,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
4090 | , error1, error2 | ||
4091 | #endif | ||
4092 | ); | ||
4093 | + rseq_after_asm_goto(); | ||
4094 | return 0; | ||
4095 | abort: | ||
4096 | + rseq_after_asm_goto(); | ||
4097 | RSEQ_INJECT_FAILED | ||
4098 | return -1; | ||
4099 | cmpfail: | ||
4100 | + rseq_after_asm_goto(); | ||
4101 | return 1; | ||
4102 | #ifdef RSEQ_COMPARE_TWICE | ||
4103 | error1: | ||
4104 | + rseq_after_asm_goto(); | ||
4105 | rseq_bug("cpu_id comparison failed"); | ||
4106 | error2: | ||
4107 | + rseq_after_asm_goto(); | ||
4108 | rseq_bug("expected value comparison failed"); | ||
4109 | #endif | ||
4110 | } | ||
4111 | @@ -369,8 +446,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
4112 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | ||
4113 | #endif | ||
4114 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4115 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4116 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4117 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4118 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4119 | RSEQ_INJECT_ASM(3) | ||
4120 | "cmpq %[v], %[expect]\n\t" | ||
4121 | "jnz %l[cmpfail]\n\t" | ||
4122 | @@ -379,7 +456,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
4123 | "jnz %l[cmpfail]\n\t" | ||
4124 | RSEQ_INJECT_ASM(5) | ||
4125 | #ifdef RSEQ_COMPARE_TWICE | ||
4126 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
4127 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4128 | "cmpq %[v], %[expect]\n\t" | ||
4129 | "jnz %l[error2]\n\t" | ||
4130 | "cmpq %[v2], %[expect2]\n\t" | ||
4131 | @@ -392,7 +469,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
4132 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4133 | : /* gcc asm goto does not allow outputs */ | ||
4134 | : [cpu_id] "r" (cpu), | ||
4135 | - [rseq_abi] "r" (&__rseq_abi), | ||
4136 | + [rseq_offset] "r" (rseq_offset), | ||
4137 | /* cmp2 input */ | ||
4138 | [v2] "m" (*v2), | ||
4139 | [expect2] "r" (expect2), | ||
4140 | @@ -407,18 +484,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
4141 | , error1, error2, error3 | ||
4142 | #endif | ||
4143 | ); | ||
4144 | + rseq_after_asm_goto(); | ||
4145 | return 0; | ||
4146 | abort: | ||
4147 | + rseq_after_asm_goto(); | ||
4148 | RSEQ_INJECT_FAILED | ||
4149 | return -1; | ||
4150 | cmpfail: | ||
4151 | + rseq_after_asm_goto(); | ||
4152 | return 1; | ||
4153 | #ifdef RSEQ_COMPARE_TWICE | ||
4154 | error1: | ||
4155 | + rseq_after_asm_goto(); | ||
4156 | rseq_bug("cpu_id comparison failed"); | ||
4157 | error2: | ||
4158 | + rseq_after_asm_goto(); | ||
4159 | rseq_bug("1st expected value comparison failed"); | ||
4160 | error3: | ||
4161 | + rseq_after_asm_goto(); | ||
4162 | rseq_bug("2nd expected value comparison failed"); | ||
4163 | #endif | ||
4164 | } | ||
4165 | @@ -443,14 +526,14 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
4166 | "movq %[dst], %[rseq_scratch1]\n\t" | ||
4167 | "movq %[len], %[rseq_scratch2]\n\t" | ||
4168 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4169 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4170 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4171 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4172 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4173 | RSEQ_INJECT_ASM(3) | ||
4174 | "cmpq %[v], %[expect]\n\t" | ||
4175 | "jnz 5f\n\t" | ||
4176 | RSEQ_INJECT_ASM(4) | ||
4177 | #ifdef RSEQ_COMPARE_TWICE | ||
4178 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) | ||
4179 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) | ||
4180 | "cmpq %[v], %[expect]\n\t" | ||
4181 | "jnz 7f\n\t" | ||
4182 | #endif | ||
4183 | @@ -498,7 +581,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
4184 | #endif | ||
4185 | : /* gcc asm goto does not allow outputs */ | ||
4186 | : [cpu_id] "r" (cpu), | ||
4187 | - [rseq_abi] "r" (&__rseq_abi), | ||
4188 | + [rseq_offset] "r" (rseq_offset), | ||
4189 | /* final store input */ | ||
4190 | [v] "m" (*v), | ||
4191 | [expect] "r" (expect), | ||
4192 | @@ -517,16 +600,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
4193 | , error1, error2 | ||
4194 | #endif | ||
4195 | ); | ||
4196 | + rseq_after_asm_goto(); | ||
4197 | return 0; | ||
4198 | abort: | ||
4199 | + rseq_after_asm_goto(); | ||
4200 | RSEQ_INJECT_FAILED | ||
4201 | return -1; | ||
4202 | cmpfail: | ||
4203 | + rseq_after_asm_goto(); | ||
4204 | return 1; | ||
4205 | #ifdef RSEQ_COMPARE_TWICE | ||
4206 | error1: | ||
4207 | + rseq_after_asm_goto(); | ||
4208 | rseq_bug("cpu_id comparison failed"); | ||
4209 | error2: | ||
4210 | + rseq_after_asm_goto(); | ||
4211 | rseq_bug("expected value comparison failed"); | ||
4212 | #endif | ||
4213 | } | ||
4214 | @@ -543,7 +631,9 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
4215 | |||
4216 | #endif /* !RSEQ_SKIP_FASTPATH */ | ||
4217 | |||
4218 | -#elif __i386__ | ||
4219 | +#elif defined(__i386__) | ||
4220 | + | ||
4221 | +#define RSEQ_ASM_TP_SEGMENT %%gs | ||
4222 | |||
4223 | #define rseq_smp_mb() \ | ||
4224 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | ||
4225 | @@ -644,14 +734,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
4226 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | ||
4227 | #endif | ||
4228 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4229 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4230 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4231 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4232 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4233 | RSEQ_INJECT_ASM(3) | ||
4234 | "cmpl %[v], %[expect]\n\t" | ||
4235 | "jnz %l[cmpfail]\n\t" | ||
4236 | RSEQ_INJECT_ASM(4) | ||
4237 | #ifdef RSEQ_COMPARE_TWICE | ||
4238 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
4239 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4240 | "cmpl %[v], %[expect]\n\t" | ||
4241 | "jnz %l[error2]\n\t" | ||
4242 | #endif | ||
4243 | @@ -662,7 +752,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
4244 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4245 | : /* gcc asm goto does not allow outputs */ | ||
4246 | : [cpu_id] "r" (cpu), | ||
4247 | - [rseq_abi] "r" (&__rseq_abi), | ||
4248 | + [rseq_offset] "r" (rseq_offset), | ||
4249 | [v] "m" (*v), | ||
4250 | [expect] "r" (expect), | ||
4251 | [newv] "r" (newv) | ||
4252 | @@ -673,16 +763,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | ||
4253 | , error1, error2 | ||
4254 | #endif | ||
4255 | ); | ||
4256 | + rseq_after_asm_goto(); | ||
4257 | return 0; | ||
4258 | abort: | ||
4259 | + rseq_after_asm_goto(); | ||
4260 | RSEQ_INJECT_FAILED | ||
4261 | return -1; | ||
4262 | cmpfail: | ||
4263 | + rseq_after_asm_goto(); | ||
4264 | return 1; | ||
4265 | #ifdef RSEQ_COMPARE_TWICE | ||
4266 | error1: | ||
4267 | + rseq_after_asm_goto(); | ||
4268 | rseq_bug("cpu_id comparison failed"); | ||
4269 | error2: | ||
4270 | + rseq_after_asm_goto(); | ||
4271 | rseq_bug("expected value comparison failed"); | ||
4272 | #endif | ||
4273 | } | ||
4274 | @@ -693,7 +788,7 @@ error2: | ||
4275 | */ | ||
4276 | static inline __attribute__((always_inline)) | ||
4277 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
4278 | - off_t voffp, intptr_t *load, int cpu) | ||
4279 | + long voffp, intptr_t *load, int cpu) | ||
4280 | { | ||
4281 | RSEQ_INJECT_C(9) | ||
4282 | |||
4283 | @@ -705,15 +800,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
4284 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | ||
4285 | #endif | ||
4286 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4287 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4288 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4289 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4290 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4291 | RSEQ_INJECT_ASM(3) | ||
4292 | "movl %[v], %%ebx\n\t" | ||
4293 | "cmpl %%ebx, %[expectnot]\n\t" | ||
4294 | "je %l[cmpfail]\n\t" | ||
4295 | RSEQ_INJECT_ASM(4) | ||
4296 | #ifdef RSEQ_COMPARE_TWICE | ||
4297 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
4298 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4299 | "movl %[v], %%ebx\n\t" | ||
4300 | "cmpl %%ebx, %[expectnot]\n\t" | ||
4301 | "je %l[error2]\n\t" | ||
4302 | @@ -728,7 +823,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
4303 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4304 | : /* gcc asm goto does not allow outputs */ | ||
4305 | : [cpu_id] "r" (cpu), | ||
4306 | - [rseq_abi] "r" (&__rseq_abi), | ||
4307 | + [rseq_offset] "r" (rseq_offset), | ||
4308 | /* final store input */ | ||
4309 | [v] "m" (*v), | ||
4310 | [expectnot] "r" (expectnot), | ||
4311 | @@ -741,16 +836,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | ||
4312 | , error1, error2 | ||
4313 | #endif | ||
4314 | ); | ||
4315 | + rseq_after_asm_goto(); | ||
4316 | return 0; | ||
4317 | abort: | ||
4318 | + rseq_after_asm_goto(); | ||
4319 | RSEQ_INJECT_FAILED | ||
4320 | return -1; | ||
4321 | cmpfail: | ||
4322 | + rseq_after_asm_goto(); | ||
4323 | return 1; | ||
4324 | #ifdef RSEQ_COMPARE_TWICE | ||
4325 | error1: | ||
4326 | + rseq_after_asm_goto(); | ||
4327 | rseq_bug("cpu_id comparison failed"); | ||
4328 | error2: | ||
4329 | + rseq_after_asm_goto(); | ||
4330 | rseq_bug("expected value comparison failed"); | ||
4331 | #endif | ||
4332 | } | ||
4333 | @@ -766,11 +866,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
4334 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | ||
4335 | #endif | ||
4336 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4337 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4338 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4339 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4340 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4341 | RSEQ_INJECT_ASM(3) | ||
4342 | #ifdef RSEQ_COMPARE_TWICE | ||
4343 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
4344 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4345 | #endif | ||
4346 | /* final store */ | ||
4347 | "addl %[count], %[v]\n\t" | ||
4348 | @@ -779,7 +879,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
4349 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4350 | : /* gcc asm goto does not allow outputs */ | ||
4351 | : [cpu_id] "r" (cpu), | ||
4352 | - [rseq_abi] "r" (&__rseq_abi), | ||
4353 | + [rseq_offset] "r" (rseq_offset), | ||
4354 | /* final store input */ | ||
4355 | [v] "m" (*v), | ||
4356 | [count] "ir" (count) | ||
4357 | @@ -790,12 +890,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) | ||
4358 | , error1 | ||
4359 | #endif | ||
4360 | ); | ||
4361 | + rseq_after_asm_goto(); | ||
4362 | return 0; | ||
4363 | abort: | ||
4364 | + rseq_after_asm_goto(); | ||
4365 | RSEQ_INJECT_FAILED | ||
4366 | return -1; | ||
4367 | #ifdef RSEQ_COMPARE_TWICE | ||
4368 | error1: | ||
4369 | + rseq_after_asm_goto(); | ||
4370 | rseq_bug("cpu_id comparison failed"); | ||
4371 | #endif | ||
4372 | } | ||
4373 | @@ -815,14 +918,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
4374 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | ||
4375 | #endif | ||
4376 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4377 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4378 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4379 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4380 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4381 | RSEQ_INJECT_ASM(3) | ||
4382 | "cmpl %[v], %[expect]\n\t" | ||
4383 | "jnz %l[cmpfail]\n\t" | ||
4384 | RSEQ_INJECT_ASM(4) | ||
4385 | #ifdef RSEQ_COMPARE_TWICE | ||
4386 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
4387 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4388 | "cmpl %[v], %[expect]\n\t" | ||
4389 | "jnz %l[error2]\n\t" | ||
4390 | #endif | ||
4391 | @@ -837,7 +940,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
4392 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4393 | : /* gcc asm goto does not allow outputs */ | ||
4394 | : [cpu_id] "r" (cpu), | ||
4395 | - [rseq_abi] "r" (&__rseq_abi), | ||
4396 | + [rseq_offset] "r" (rseq_offset), | ||
4397 | /* try store input */ | ||
4398 | [v2] "m" (*v2), | ||
4399 | [newv2] "m" (newv2), | ||
4400 | @@ -852,16 +955,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | ||
4401 | , error1, error2 | ||
4402 | #endif | ||
4403 | ); | ||
4404 | + rseq_after_asm_goto(); | ||
4405 | return 0; | ||
4406 | abort: | ||
4407 | + rseq_after_asm_goto(); | ||
4408 | RSEQ_INJECT_FAILED | ||
4409 | return -1; | ||
4410 | cmpfail: | ||
4411 | + rseq_after_asm_goto(); | ||
4412 | return 1; | ||
4413 | #ifdef RSEQ_COMPARE_TWICE | ||
4414 | error1: | ||
4415 | + rseq_after_asm_goto(); | ||
4416 | rseq_bug("cpu_id comparison failed"); | ||
4417 | error2: | ||
4418 | + rseq_after_asm_goto(); | ||
4419 | rseq_bug("expected value comparison failed"); | ||
4420 | #endif | ||
4421 | } | ||
4422 | @@ -881,15 +989,15 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
4423 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | ||
4424 | #endif | ||
4425 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4426 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4427 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4428 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4429 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4430 | RSEQ_INJECT_ASM(3) | ||
4431 | "movl %[expect], %%eax\n\t" | ||
4432 | "cmpl %[v], %%eax\n\t" | ||
4433 | "jnz %l[cmpfail]\n\t" | ||
4434 | RSEQ_INJECT_ASM(4) | ||
4435 | #ifdef RSEQ_COMPARE_TWICE | ||
4436 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
4437 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4438 | "movl %[expect], %%eax\n\t" | ||
4439 | "cmpl %[v], %%eax\n\t" | ||
4440 | "jnz %l[error2]\n\t" | ||
4441 | @@ -905,7 +1013,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
4442 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4443 | : /* gcc asm goto does not allow outputs */ | ||
4444 | : [cpu_id] "r" (cpu), | ||
4445 | - [rseq_abi] "r" (&__rseq_abi), | ||
4446 | + [rseq_offset] "r" (rseq_offset), | ||
4447 | /* try store input */ | ||
4448 | [v2] "m" (*v2), | ||
4449 | [newv2] "r" (newv2), | ||
4450 | @@ -920,16 +1028,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | ||
4451 | , error1, error2 | ||
4452 | #endif | ||
4453 | ); | ||
4454 | + rseq_after_asm_goto(); | ||
4455 | return 0; | ||
4456 | abort: | ||
4457 | + rseq_after_asm_goto(); | ||
4458 | RSEQ_INJECT_FAILED | ||
4459 | return -1; | ||
4460 | cmpfail: | ||
4461 | + rseq_after_asm_goto(); | ||
4462 | return 1; | ||
4463 | #ifdef RSEQ_COMPARE_TWICE | ||
4464 | error1: | ||
4465 | + rseq_after_asm_goto(); | ||
4466 | rseq_bug("cpu_id comparison failed"); | ||
4467 | error2: | ||
4468 | + rseq_after_asm_goto(); | ||
4469 | rseq_bug("expected value comparison failed"); | ||
4470 | #endif | ||
4471 | |||
4472 | @@ -951,8 +1064,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
4473 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | ||
4474 | #endif | ||
4475 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4476 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4477 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4478 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4479 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4480 | RSEQ_INJECT_ASM(3) | ||
4481 | "cmpl %[v], %[expect]\n\t" | ||
4482 | "jnz %l[cmpfail]\n\t" | ||
4483 | @@ -961,7 +1074,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
4484 | "jnz %l[cmpfail]\n\t" | ||
4485 | RSEQ_INJECT_ASM(5) | ||
4486 | #ifdef RSEQ_COMPARE_TWICE | ||
4487 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | ||
4488 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) | ||
4489 | "cmpl %[v], %[expect]\n\t" | ||
4490 | "jnz %l[error2]\n\t" | ||
4491 | "cmpl %[expect2], %[v2]\n\t" | ||
4492 | @@ -975,7 +1088,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
4493 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | ||
4494 | : /* gcc asm goto does not allow outputs */ | ||
4495 | : [cpu_id] "r" (cpu), | ||
4496 | - [rseq_abi] "r" (&__rseq_abi), | ||
4497 | + [rseq_offset] "r" (rseq_offset), | ||
4498 | /* cmp2 input */ | ||
4499 | [v2] "m" (*v2), | ||
4500 | [expect2] "r" (expect2), | ||
4501 | @@ -990,18 +1103,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | ||
4502 | , error1, error2, error3 | ||
4503 | #endif | ||
4504 | ); | ||
4505 | + rseq_after_asm_goto(); | ||
4506 | return 0; | ||
4507 | abort: | ||
4508 | + rseq_after_asm_goto(); | ||
4509 | RSEQ_INJECT_FAILED | ||
4510 | return -1; | ||
4511 | cmpfail: | ||
4512 | + rseq_after_asm_goto(); | ||
4513 | return 1; | ||
4514 | #ifdef RSEQ_COMPARE_TWICE | ||
4515 | error1: | ||
4516 | + rseq_after_asm_goto(); | ||
4517 | rseq_bug("cpu_id comparison failed"); | ||
4518 | error2: | ||
4519 | + rseq_after_asm_goto(); | ||
4520 | rseq_bug("1st expected value comparison failed"); | ||
4521 | error3: | ||
4522 | + rseq_after_asm_goto(); | ||
4523 | rseq_bug("2nd expected value comparison failed"); | ||
4524 | #endif | ||
4525 | } | ||
4526 | @@ -1027,15 +1146,15 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
4527 | "movl %[dst], %[rseq_scratch1]\n\t" | ||
4528 | "movl %[len], %[rseq_scratch2]\n\t" | ||
4529 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4530 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4531 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4532 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4533 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4534 | RSEQ_INJECT_ASM(3) | ||
4535 | "movl %[expect], %%eax\n\t" | ||
4536 | "cmpl %%eax, %[v]\n\t" | ||
4537 | "jnz 5f\n\t" | ||
4538 | RSEQ_INJECT_ASM(4) | ||
4539 | #ifdef RSEQ_COMPARE_TWICE | ||
4540 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) | ||
4541 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) | ||
4542 | "movl %[expect], %%eax\n\t" | ||
4543 | "cmpl %%eax, %[v]\n\t" | ||
4544 | "jnz 7f\n\t" | ||
4545 | @@ -1085,7 +1204,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
4546 | #endif | ||
4547 | : /* gcc asm goto does not allow outputs */ | ||
4548 | : [cpu_id] "r" (cpu), | ||
4549 | - [rseq_abi] "r" (&__rseq_abi), | ||
4550 | + [rseq_offset] "r" (rseq_offset), | ||
4551 | /* final store input */ | ||
4552 | [v] "m" (*v), | ||
4553 | [expect] "m" (expect), | ||
4554 | @@ -1104,16 +1223,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | ||
4555 | , error1, error2 | ||
4556 | #endif | ||
4557 | ); | ||
4558 | + rseq_after_asm_goto(); | ||
4559 | return 0; | ||
4560 | abort: | ||
4561 | + rseq_after_asm_goto(); | ||
4562 | RSEQ_INJECT_FAILED | ||
4563 | return -1; | ||
4564 | cmpfail: | ||
4565 | + rseq_after_asm_goto(); | ||
4566 | return 1; | ||
4567 | #ifdef RSEQ_COMPARE_TWICE | ||
4568 | error1: | ||
4569 | + rseq_after_asm_goto(); | ||
4570 | rseq_bug("cpu_id comparison failed"); | ||
4571 | error2: | ||
4572 | + rseq_after_asm_goto(); | ||
4573 | rseq_bug("expected value comparison failed"); | ||
4574 | #endif | ||
4575 | } | ||
4576 | @@ -1139,15 +1263,15 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
4577 | "movl %[dst], %[rseq_scratch1]\n\t" | ||
4578 | "movl %[len], %[rseq_scratch2]\n\t" | ||
4579 | /* Start rseq by storing table entry pointer into rseq_cs. */ | ||
4580 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | ||
4581 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | ||
4582 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) | ||
4583 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | ||
4584 | RSEQ_INJECT_ASM(3) | ||
4585 | "movl %[expect], %%eax\n\t" | ||
4586 | "cmpl %%eax, %[v]\n\t" | ||
4587 | "jnz 5f\n\t" | ||
4588 | RSEQ_INJECT_ASM(4) | ||
4589 | #ifdef RSEQ_COMPARE_TWICE | ||
4590 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) | ||
4591 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) | ||
4592 | "movl %[expect], %%eax\n\t" | ||
4593 | "cmpl %%eax, %[v]\n\t" | ||
4594 | "jnz 7f\n\t" | ||
4595 | @@ -1198,7 +1322,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
4596 | #endif | ||
4597 | : /* gcc asm goto does not allow outputs */ | ||
4598 | : [cpu_id] "r" (cpu), | ||
4599 | - [rseq_abi] "r" (&__rseq_abi), | ||
4600 | + [rseq_offset] "r" (rseq_offset), | ||
4601 | /* final store input */ | ||
4602 | [v] "m" (*v), | ||
4603 | [expect] "m" (expect), | ||
4604 | @@ -1217,16 +1341,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | ||
4605 | , error1, error2 | ||
4606 | #endif | ||
4607 | ); | ||
4608 | + rseq_after_asm_goto(); | ||
4609 | return 0; | ||
4610 | abort: | ||
4611 | + rseq_after_asm_goto(); | ||
4612 | RSEQ_INJECT_FAILED | ||
4613 | return -1; | ||
4614 | cmpfail: | ||
4615 | + rseq_after_asm_goto(); | ||
4616 | return 1; | ||
4617 | #ifdef RSEQ_COMPARE_TWICE | ||
4618 | error1: | ||
4619 | + rseq_after_asm_goto(); | ||
4620 | rseq_bug("cpu_id comparison failed"); | ||
4621 | error2: | ||
4622 | + rseq_after_asm_goto(); | ||
4623 | rseq_bug("expected value comparison failed"); | ||
4624 | #endif | ||
4625 | } | ||
4626 | diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c | ||
4627 | index 7159eb777fd34..986b9458efb26 100644 | ||
4628 | --- a/tools/testing/selftests/rseq/rseq.c | ||
4629 | +++ b/tools/testing/selftests/rseq/rseq.c | ||
4630 | @@ -26,131 +26,124 @@ | ||
4631 | #include <assert.h> | ||
4632 | #include <signal.h> | ||
4633 | #include <limits.h> | ||
4634 | +#include <dlfcn.h> | ||
4635 | +#include <stddef.h> | ||
4636 | |||
4637 | +#include "../kselftest.h" | ||
4638 | #include "rseq.h" | ||
4639 | |||
4640 | -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) | ||
4641 | +static const ptrdiff_t *libc_rseq_offset_p; | ||
4642 | +static const unsigned int *libc_rseq_size_p; | ||
4643 | +static const unsigned int *libc_rseq_flags_p; | ||
4644 | |||
4645 | -__thread volatile struct rseq __rseq_abi = { | ||
4646 | - .cpu_id = RSEQ_CPU_ID_UNINITIALIZED, | ||
4647 | -}; | ||
4648 | +/* Offset from the thread pointer to the rseq area. */ | ||
4649 | +ptrdiff_t rseq_offset; | ||
4650 | |||
4651 | -/* | ||
4652 | - * Shared with other libraries. This library may take rseq ownership if it is | ||
4653 | - * still 0 when executing the library constructor. Set to 1 by library | ||
4654 | - * constructor when handling rseq. Set to 0 in destructor if handling rseq. | ||
4655 | - */ | ||
4656 | -int __rseq_handled; | ||
4657 | +/* Size of the registered rseq area. 0 if the registration was | ||
4658 | + unsuccessful. */ | ||
4659 | +unsigned int rseq_size = -1U; | ||
4660 | + | ||
4661 | +/* Flags used during rseq registration. */ | ||
4662 | +unsigned int rseq_flags; | ||
4663 | |||
4664 | -/* Whether this library have ownership of rseq registration. */ | ||
4665 | static int rseq_ownership; | ||
4666 | |||
4667 | -static __thread volatile uint32_t __rseq_refcount; | ||
4668 | +static | ||
4669 | +__thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"))) = { | ||
4670 | + .cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED, | ||
4671 | +}; | ||
4672 | |||
4673 | -static void signal_off_save(sigset_t *oldset) | ||
4674 | +static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len, | ||
4675 | + int flags, uint32_t sig) | ||
4676 | { | ||
4677 | - sigset_t set; | ||
4678 | - int ret; | ||
4679 | - | ||
4680 | - sigfillset(&set); | ||
4681 | - ret = pthread_sigmask(SIG_BLOCK, &set, oldset); | ||
4682 | - if (ret) | ||
4683 | - abort(); | ||
4684 | + return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); | ||
4685 | } | ||
4686 | |||
4687 | -static void signal_restore(sigset_t oldset) | ||
4688 | +int rseq_available(void) | ||
4689 | { | ||
4690 | - int ret; | ||
4691 | + int rc; | ||
4692 | |||
4693 | - ret = pthread_sigmask(SIG_SETMASK, &oldset, NULL); | ||
4694 | - if (ret) | ||
4695 | + rc = sys_rseq(NULL, 0, 0, 0); | ||
4696 | + if (rc != -1) | ||
4697 | abort(); | ||
4698 | -} | ||
4699 | - | ||
4700 | -static int sys_rseq(volatile struct rseq *rseq_abi, uint32_t rseq_len, | ||
4701 | - int flags, uint32_t sig) | ||
4702 | -{ | ||
4703 | - return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); | ||
4704 | + switch (errno) { | ||
4705 | + case ENOSYS: | ||
4706 | + return 0; | ||
4707 | + case EINVAL: | ||
4708 | + return 1; | ||
4709 | + default: | ||
4710 | + abort(); | ||
4711 | + } | ||
4712 | } | ||
4713 | |||
4714 | int rseq_register_current_thread(void) | ||
4715 | { | ||
4716 | - int rc, ret = 0; | ||
4717 | - sigset_t oldset; | ||
4718 | + int rc; | ||
4719 | |||
4720 | - if (!rseq_ownership) | ||
4721 | + if (!rseq_ownership) { | ||
4722 | + /* Treat libc's ownership as a successful registration. */ | ||
4723 | return 0; | ||
4724 | - signal_off_save(&oldset); | ||
4725 | - if (__rseq_refcount == UINT_MAX) { | ||
4726 | - ret = -1; | ||
4727 | - goto end; | ||
4728 | - } | ||
4729 | - if (__rseq_refcount++) | ||
4730 | - goto end; | ||
4731 | - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG); | ||
4732 | - if (!rc) { | ||
4733 | - assert(rseq_current_cpu_raw() >= 0); | ||
4734 | - goto end; | ||
4735 | } | ||
4736 | - if (errno != EBUSY) | ||
4737 | - __rseq_abi.cpu_id = RSEQ_CPU_ID_REGISTRATION_FAILED; | ||
4738 | - ret = -1; | ||
4739 | - __rseq_refcount--; | ||
4740 | -end: | ||
4741 | - signal_restore(oldset); | ||
4742 | - return ret; | ||
4743 | + rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), 0, RSEQ_SIG); | ||
4744 | + if (rc) | ||
4745 | + return -1; | ||
4746 | + assert(rseq_current_cpu_raw() >= 0); | ||
4747 | + return 0; | ||
4748 | } | ||
4749 | |||
4750 | int rseq_unregister_current_thread(void) | ||
4751 | { | ||
4752 | - int rc, ret = 0; | ||
4753 | - sigset_t oldset; | ||
4754 | + int rc; | ||
4755 | |||
4756 | - if (!rseq_ownership) | ||
4757 | + if (!rseq_ownership) { | ||
4758 | + /* Treat libc's ownership as a successful unregistration. */ | ||
4759 | return 0; | ||
4760 | - signal_off_save(&oldset); | ||
4761 | - if (!__rseq_refcount) { | ||
4762 | - ret = -1; | ||
4763 | - goto end; | ||
4764 | } | ||
4765 | - if (--__rseq_refcount) | ||
4766 | - goto end; | ||
4767 | - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), | ||
4768 | - RSEQ_FLAG_UNREGISTER, RSEQ_SIG); | ||
4769 | - if (!rc) | ||
4770 | - goto end; | ||
4771 | - __rseq_refcount = 1; | ||
4772 | - ret = -1; | ||
4773 | -end: | ||
4774 | - signal_restore(oldset); | ||
4775 | - return ret; | ||
4776 | + rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); | ||
4777 | + if (rc) | ||
4778 | + return -1; | ||
4779 | + return 0; | ||
4780 | } | ||
4781 | |||
4782 | -int32_t rseq_fallback_current_cpu(void) | ||
4783 | +static __attribute__((constructor)) | ||
4784 | +void rseq_init(void) | ||
4785 | { | ||
4786 | - int32_t cpu; | ||
4787 | - | ||
4788 | - cpu = sched_getcpu(); | ||
4789 | - if (cpu < 0) { | ||
4790 | - perror("sched_getcpu()"); | ||
4791 | - abort(); | ||
4792 | + libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset"); | ||
4793 | + libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size"); | ||
4794 | + libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags"); | ||
4795 | + if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p) { | ||
4796 | + /* rseq registration owned by glibc */ | ||
4797 | + rseq_offset = *libc_rseq_offset_p; | ||
4798 | + rseq_size = *libc_rseq_size_p; | ||
4799 | + rseq_flags = *libc_rseq_flags_p; | ||
4800 | + return; | ||
4801 | } | ||
4802 | - return cpu; | ||
4803 | -} | ||
4804 | - | ||
4805 | -void __attribute__((constructor)) rseq_init(void) | ||
4806 | -{ | ||
4807 | - /* Check whether rseq is handled by another library. */ | ||
4808 | - if (__rseq_handled) | ||
4809 | + if (!rseq_available()) | ||
4810 | return; | ||
4811 | - __rseq_handled = 1; | ||
4812 | rseq_ownership = 1; | ||
4813 | + rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer(); | ||
4814 | + rseq_size = sizeof(struct rseq_abi); | ||
4815 | + rseq_flags = 0; | ||
4816 | } | ||
4817 | |||
4818 | -void __attribute__((destructor)) rseq_fini(void) | ||
4819 | +static __attribute__((destructor)) | ||
4820 | +void rseq_exit(void) | ||
4821 | { | ||
4822 | if (!rseq_ownership) | ||
4823 | return; | ||
4824 | - __rseq_handled = 0; | ||
4825 | + rseq_offset = 0; | ||
4826 | + rseq_size = -1U; | ||
4827 | rseq_ownership = 0; | ||
4828 | } | ||
4829 | + | ||
4830 | +int32_t rseq_fallback_current_cpu(void) | ||
4831 | +{ | ||
4832 | + int32_t cpu; | ||
4833 | + | ||
4834 | + cpu = sched_getcpu(); | ||
4835 | + if (cpu < 0) { | ||
4836 | + perror("sched_getcpu()"); | ||
4837 | + abort(); | ||
4838 | + } | ||
4839 | + return cpu; | ||
4840 | +} | ||
4841 | diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h | ||
4842 | index d40d60e7499e8..c8f49b018dbb0 100644 | ||
4843 | --- a/tools/testing/selftests/rseq/rseq.h | ||
4844 | +++ b/tools/testing/selftests/rseq/rseq.h | ||
4845 | @@ -16,7 +16,9 @@ | ||
4846 | #include <errno.h> | ||
4847 | #include <stdio.h> | ||
4848 | #include <stdlib.h> | ||
4849 | -#include <linux/rseq.h> | ||
4850 | +#include <stddef.h> | ||
4851 | +#include "rseq-abi.h" | ||
4852 | +#include "compiler.h" | ||
4853 | |||
4854 | /* | ||
4855 | * Empty code injection macros, override when testing. | ||
4856 | @@ -43,8 +45,20 @@ | ||
4857 | #define RSEQ_INJECT_FAILED | ||
4858 | #endif | ||
4859 | |||
4860 | -extern __thread volatile struct rseq __rseq_abi; | ||
4861 | -extern int __rseq_handled; | ||
4862 | +#include "rseq-thread-pointer.h" | ||
4863 | + | ||
4864 | +/* Offset from the thread pointer to the rseq area. */ | ||
4865 | +extern ptrdiff_t rseq_offset; | ||
4866 | +/* Size of the registered rseq area. 0 if the registration was | ||
4867 | + unsuccessful. */ | ||
4868 | +extern unsigned int rseq_size; | ||
4869 | +/* Flags used during rseq registration. */ | ||
4870 | +extern unsigned int rseq_flags; | ||
4871 | + | ||
4872 | +static inline struct rseq_abi *rseq_get_abi(void) | ||
4873 | +{ | ||
4874 | + return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); | ||
4875 | +} | ||
4876 | |||
4877 | #define rseq_likely(x) __builtin_expect(!!(x), 1) | ||
4878 | #define rseq_unlikely(x) __builtin_expect(!!(x), 0) | ||
4879 | @@ -108,7 +122,7 @@ int32_t rseq_fallback_current_cpu(void); | ||
4880 | */ | ||
4881 | static inline int32_t rseq_current_cpu_raw(void) | ||
4882 | { | ||
4883 | - return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id); | ||
4884 | + return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id); | ||
4885 | } | ||
4886 | |||
4887 | /* | ||
4888 | @@ -124,7 +138,7 @@ static inline int32_t rseq_current_cpu_raw(void) | ||
4889 | */ | ||
4890 | static inline uint32_t rseq_cpu_start(void) | ||
4891 | { | ||
4892 | - return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id_start); | ||
4893 | + return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start); | ||
4894 | } | ||
4895 | |||
4896 | static inline uint32_t rseq_current_cpu(void) | ||
4897 | @@ -139,11 +153,7 @@ static inline uint32_t rseq_current_cpu(void) | ||
4898 | |||
4899 | static inline void rseq_clear_rseq_cs(void) | ||
4900 | { | ||
4901 | -#ifdef __LP64__ | ||
4902 | - __rseq_abi.rseq_cs.ptr = 0; | ||
4903 | -#else | ||
4904 | - __rseq_abi.rseq_cs.ptr.ptr32 = 0; | ||
4905 | -#endif | ||
4906 | + RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); | ||
4907 | } | ||
4908 | |||
4909 | /* |