Contents of /trunk/kernel-alx/patches-5.4/0303-5.4.204-all-fixes.patch
Parent Directory | Revision Log
Revision 3637 -
(show annotations)
(download)
Mon Oct 24 12:40:44 2022 UTC (18 months, 1 week ago) by niro
File size: 154160 byte(s)
Mon Oct 24 12:40:44 2022 UTC (18 months, 1 week ago) by niro
File size: 154160 byte(s)
-add missing
1 | diff --git a/Makefile b/Makefile |
2 | index 86f518e49c333..b5d1718ae70c6 100644 |
3 | --- a/Makefile |
4 | +++ b/Makefile |
5 | @@ -1,7 +1,7 @@ |
6 | # SPDX-License-Identifier: GPL-2.0 |
7 | VERSION = 5 |
8 | PATCHLEVEL = 4 |
9 | -SUBLEVEL = 203 |
10 | +SUBLEVEL = 204 |
11 | EXTRAVERSION = |
12 | NAME = Kleptomaniac Octopus |
13 | |
14 | diff --git a/arch/arm/xen/p2m.c b/arch/arm/xen/p2m.c |
15 | index acb464547a54f..4a1991a103ea0 100644 |
16 | --- a/arch/arm/xen/p2m.c |
17 | +++ b/arch/arm/xen/p2m.c |
18 | @@ -62,11 +62,12 @@ out: |
19 | |
20 | unsigned long __pfn_to_mfn(unsigned long pfn) |
21 | { |
22 | - struct rb_node *n = phys_to_mach.rb_node; |
23 | + struct rb_node *n; |
24 | struct xen_p2m_entry *entry; |
25 | unsigned long irqflags; |
26 | |
27 | read_lock_irqsave(&p2m_lock, irqflags); |
28 | + n = phys_to_mach.rb_node; |
29 | while (n) { |
30 | entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); |
31 | if (entry->pfn <= pfn && |
32 | @@ -153,10 +154,11 @@ bool __set_phys_to_machine_multi(unsigned long pfn, |
33 | int rc; |
34 | unsigned long irqflags; |
35 | struct xen_p2m_entry *p2m_entry; |
36 | - struct rb_node *n = phys_to_mach.rb_node; |
37 | + struct rb_node *n; |
38 | |
39 | if (mfn == INVALID_P2M_ENTRY) { |
40 | write_lock_irqsave(&p2m_lock, irqflags); |
41 | + n = phys_to_mach.rb_node; |
42 | while (n) { |
43 | p2m_entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); |
44 | if (p2m_entry->pfn <= pfn && |
45 | diff --git a/arch/powerpc/include/asm/bpf_perf_event.h b/arch/powerpc/include/asm/bpf_perf_event.h |
46 | new file mode 100644 |
47 | index 0000000000000..e8a7b4ffb58c2 |
48 | --- /dev/null |
49 | +++ b/arch/powerpc/include/asm/bpf_perf_event.h |
50 | @@ -0,0 +1,9 @@ |
51 | +/* SPDX-License-Identifier: GPL-2.0 */ |
52 | +#ifndef _ASM_POWERPC_BPF_PERF_EVENT_H |
53 | +#define _ASM_POWERPC_BPF_PERF_EVENT_H |
54 | + |
55 | +#include <asm/ptrace.h> |
56 | + |
57 | +typedef struct user_pt_regs bpf_user_pt_regs_t; |
58 | + |
59 | +#endif /* _ASM_POWERPC_BPF_PERF_EVENT_H */ |
60 | diff --git a/arch/powerpc/include/uapi/asm/bpf_perf_event.h b/arch/powerpc/include/uapi/asm/bpf_perf_event.h |
61 | deleted file mode 100644 |
62 | index 5e1e648aeec4c..0000000000000 |
63 | --- a/arch/powerpc/include/uapi/asm/bpf_perf_event.h |
64 | +++ /dev/null |
65 | @@ -1,9 +0,0 @@ |
66 | -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ |
67 | -#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ |
68 | -#define _UAPI__ASM_BPF_PERF_EVENT_H__ |
69 | - |
70 | -#include <asm/ptrace.h> |
71 | - |
72 | -typedef struct user_pt_regs bpf_user_pt_regs_t; |
73 | - |
74 | -#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ |
75 | diff --git a/arch/powerpc/kernel/prom_init_check.sh b/arch/powerpc/kernel/prom_init_check.sh |
76 | index b183ab9c5107c..dfa5f729f774d 100644 |
77 | --- a/arch/powerpc/kernel/prom_init_check.sh |
78 | +++ b/arch/powerpc/kernel/prom_init_check.sh |
79 | @@ -13,7 +13,7 @@ |
80 | # If you really need to reference something from prom_init.o add |
81 | # it to the list below: |
82 | |
83 | -grep "^CONFIG_KASAN=y$" .config >/dev/null |
84 | +grep "^CONFIG_KASAN=y$" ${KCONFIG_CONFIG} >/dev/null |
85 | if [ $? -eq 0 ] |
86 | then |
87 | MEM_FUNCS="__memcpy __memset" |
88 | diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig |
89 | index 0023b78391f14..d7c7c7b80aed1 100644 |
90 | --- a/arch/s390/Kconfig |
91 | +++ b/arch/s390/Kconfig |
92 | @@ -541,7 +541,6 @@ config KEXEC |
93 | config KEXEC_FILE |
94 | bool "kexec file based system call" |
95 | select KEXEC_CORE |
96 | - select BUILD_BIN2C |
97 | depends on CRYPTO |
98 | depends on CRYPTO_SHA256 |
99 | depends on CRYPTO_SHA256_S390 |
100 | diff --git a/arch/s390/crypto/arch_random.c b/arch/s390/crypto/arch_random.c |
101 | index 4cbb4b6d85a83..1f2d40993c4d2 100644 |
102 | --- a/arch/s390/crypto/arch_random.c |
103 | +++ b/arch/s390/crypto/arch_random.c |
104 | @@ -2,126 +2,17 @@ |
105 | /* |
106 | * s390 arch random implementation. |
107 | * |
108 | - * Copyright IBM Corp. 2017, 2018 |
109 | + * Copyright IBM Corp. 2017, 2020 |
110 | * Author(s): Harald Freudenberger |
111 | - * |
112 | - * The s390_arch_random_generate() function may be called from random.c |
113 | - * in interrupt context. So this implementation does the best to be very |
114 | - * fast. There is a buffer of random data which is asynchronously checked |
115 | - * and filled by a workqueue thread. |
116 | - * If there are enough bytes in the buffer the s390_arch_random_generate() |
117 | - * just delivers these bytes. Otherwise false is returned until the |
118 | - * worker thread refills the buffer. |
119 | - * The worker fills the rng buffer by pulling fresh entropy from the |
120 | - * high quality (but slow) true hardware random generator. This entropy |
121 | - * is then spread over the buffer with an pseudo random generator PRNG. |
122 | - * As the arch_get_random_seed_long() fetches 8 bytes and the calling |
123 | - * function add_interrupt_randomness() counts this as 1 bit entropy the |
124 | - * distribution needs to make sure there is in fact 1 bit entropy contained |
125 | - * in 8 bytes of the buffer. The current values pull 32 byte entropy |
126 | - * and scatter this into a 2048 byte buffer. So 8 byte in the buffer |
127 | - * will contain 1 bit of entropy. |
128 | - * The worker thread is rescheduled based on the charge level of the |
129 | - * buffer but at least with 500 ms delay to avoid too much CPU consumption. |
130 | - * So the max. amount of rng data delivered via arch_get_random_seed is |
131 | - * limited to 4k bytes per second. |
132 | */ |
133 | |
134 | #include <linux/kernel.h> |
135 | #include <linux/atomic.h> |
136 | #include <linux/random.h> |
137 | -#include <linux/slab.h> |
138 | #include <linux/static_key.h> |
139 | -#include <linux/workqueue.h> |
140 | #include <asm/cpacf.h> |
141 | |
142 | DEFINE_STATIC_KEY_FALSE(s390_arch_random_available); |
143 | |
144 | atomic64_t s390_arch_random_counter = ATOMIC64_INIT(0); |
145 | EXPORT_SYMBOL(s390_arch_random_counter); |
146 | - |
147 | -#define ARCH_REFILL_TICKS (HZ/2) |
148 | -#define ARCH_PRNG_SEED_SIZE 32 |
149 | -#define ARCH_RNG_BUF_SIZE 2048 |
150 | - |
151 | -static DEFINE_SPINLOCK(arch_rng_lock); |
152 | -static u8 *arch_rng_buf; |
153 | -static unsigned int arch_rng_buf_idx; |
154 | - |
155 | -static void arch_rng_refill_buffer(struct work_struct *); |
156 | -static DECLARE_DELAYED_WORK(arch_rng_work, arch_rng_refill_buffer); |
157 | - |
158 | -bool s390_arch_random_generate(u8 *buf, unsigned int nbytes) |
159 | -{ |
160 | - /* max hunk is ARCH_RNG_BUF_SIZE */ |
161 | - if (nbytes > ARCH_RNG_BUF_SIZE) |
162 | - return false; |
163 | - |
164 | - /* lock rng buffer */ |
165 | - if (!spin_trylock(&arch_rng_lock)) |
166 | - return false; |
167 | - |
168 | - /* try to resolve the requested amount of bytes from the buffer */ |
169 | - arch_rng_buf_idx -= nbytes; |
170 | - if (arch_rng_buf_idx < ARCH_RNG_BUF_SIZE) { |
171 | - memcpy(buf, arch_rng_buf + arch_rng_buf_idx, nbytes); |
172 | - atomic64_add(nbytes, &s390_arch_random_counter); |
173 | - spin_unlock(&arch_rng_lock); |
174 | - return true; |
175 | - } |
176 | - |
177 | - /* not enough bytes in rng buffer, refill is done asynchronously */ |
178 | - spin_unlock(&arch_rng_lock); |
179 | - |
180 | - return false; |
181 | -} |
182 | -EXPORT_SYMBOL(s390_arch_random_generate); |
183 | - |
184 | -static void arch_rng_refill_buffer(struct work_struct *unused) |
185 | -{ |
186 | - unsigned int delay = ARCH_REFILL_TICKS; |
187 | - |
188 | - spin_lock(&arch_rng_lock); |
189 | - if (arch_rng_buf_idx > ARCH_RNG_BUF_SIZE) { |
190 | - /* buffer is exhausted and needs refill */ |
191 | - u8 seed[ARCH_PRNG_SEED_SIZE]; |
192 | - u8 prng_wa[240]; |
193 | - /* fetch ARCH_PRNG_SEED_SIZE bytes of entropy */ |
194 | - cpacf_trng(NULL, 0, seed, sizeof(seed)); |
195 | - /* blow this entropy up to ARCH_RNG_BUF_SIZE with PRNG */ |
196 | - memset(prng_wa, 0, sizeof(prng_wa)); |
197 | - cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED, |
198 | - &prng_wa, NULL, 0, seed, sizeof(seed)); |
199 | - cpacf_prno(CPACF_PRNO_SHA512_DRNG_GEN, |
200 | - &prng_wa, arch_rng_buf, ARCH_RNG_BUF_SIZE, NULL, 0); |
201 | - arch_rng_buf_idx = ARCH_RNG_BUF_SIZE; |
202 | - } |
203 | - delay += (ARCH_REFILL_TICKS * arch_rng_buf_idx) / ARCH_RNG_BUF_SIZE; |
204 | - spin_unlock(&arch_rng_lock); |
205 | - |
206 | - /* kick next check */ |
207 | - queue_delayed_work(system_long_wq, &arch_rng_work, delay); |
208 | -} |
209 | - |
210 | -static int __init s390_arch_random_init(void) |
211 | -{ |
212 | - /* all the needed PRNO subfunctions available ? */ |
213 | - if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG) && |
214 | - cpacf_query_func(CPACF_PRNO, CPACF_PRNO_SHA512_DRNG_GEN)) { |
215 | - |
216 | - /* alloc arch random working buffer */ |
217 | - arch_rng_buf = kmalloc(ARCH_RNG_BUF_SIZE, GFP_KERNEL); |
218 | - if (!arch_rng_buf) |
219 | - return -ENOMEM; |
220 | - |
221 | - /* kick worker queue job to fill the random buffer */ |
222 | - queue_delayed_work(system_long_wq, |
223 | - &arch_rng_work, ARCH_REFILL_TICKS); |
224 | - |
225 | - /* enable arch random to the outside world */ |
226 | - static_branch_enable(&s390_arch_random_available); |
227 | - } |
228 | - |
229 | - return 0; |
230 | -} |
231 | -arch_initcall(s390_arch_random_init); |
232 | diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h |
233 | index 9a6835137a164..2c6e1c6ecbe78 100644 |
234 | --- a/arch/s390/include/asm/archrandom.h |
235 | +++ b/arch/s390/include/asm/archrandom.h |
236 | @@ -2,7 +2,7 @@ |
237 | /* |
238 | * Kernel interface for the s390 arch_random_* functions |
239 | * |
240 | - * Copyright IBM Corp. 2017 |
241 | + * Copyright IBM Corp. 2017, 2020 |
242 | * |
243 | * Author: Harald Freudenberger <freude@de.ibm.com> |
244 | * |
245 | @@ -15,34 +15,37 @@ |
246 | |
247 | #include <linux/static_key.h> |
248 | #include <linux/atomic.h> |
249 | +#include <asm/cpacf.h> |
250 | |
251 | DECLARE_STATIC_KEY_FALSE(s390_arch_random_available); |
252 | extern atomic64_t s390_arch_random_counter; |
253 | |
254 | -bool s390_arch_random_generate(u8 *buf, unsigned int nbytes); |
255 | - |
256 | -static inline bool arch_get_random_long(unsigned long *v) |
257 | +static inline bool __must_check arch_get_random_long(unsigned long *v) |
258 | { |
259 | return false; |
260 | } |
261 | |
262 | -static inline bool arch_get_random_int(unsigned int *v) |
263 | +static inline bool __must_check arch_get_random_int(unsigned int *v) |
264 | { |
265 | return false; |
266 | } |
267 | |
268 | -static inline bool arch_get_random_seed_long(unsigned long *v) |
269 | +static inline bool __must_check arch_get_random_seed_long(unsigned long *v) |
270 | { |
271 | if (static_branch_likely(&s390_arch_random_available)) { |
272 | - return s390_arch_random_generate((u8 *)v, sizeof(*v)); |
273 | + cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v)); |
274 | + atomic64_add(sizeof(*v), &s390_arch_random_counter); |
275 | + return true; |
276 | } |
277 | return false; |
278 | } |
279 | |
280 | -static inline bool arch_get_random_seed_int(unsigned int *v) |
281 | +static inline bool __must_check arch_get_random_seed_int(unsigned int *v) |
282 | { |
283 | if (static_branch_likely(&s390_arch_random_available)) { |
284 | - return s390_arch_random_generate((u8 *)v, sizeof(*v)); |
285 | + cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v)); |
286 | + atomic64_add(sizeof(*v), &s390_arch_random_counter); |
287 | + return true; |
288 | } |
289 | return false; |
290 | } |
291 | diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c |
292 | index 9a0316a067a11..dbc2a718d232d 100644 |
293 | --- a/arch/s390/kernel/setup.c |
294 | +++ b/arch/s390/kernel/setup.c |
295 | @@ -1005,6 +1005,11 @@ static void __init setup_randomness(void) |
296 | if (stsi(vmms, 3, 2, 2) == 0 && vmms->count) |
297 | add_device_randomness(&vmms->vm, sizeof(vmms->vm[0]) * vmms->count); |
298 | memblock_free((unsigned long) vmms, PAGE_SIZE); |
299 | + |
300 | +#ifdef CONFIG_ARCH_RANDOM |
301 | + if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG)) |
302 | + static_branch_enable(&s390_arch_random_available); |
303 | +#endif |
304 | } |
305 | |
306 | /* |
307 | diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c |
308 | index 3731066f2c1ca..d0538c03f0332 100644 |
309 | --- a/drivers/block/xen-blkfront.c |
310 | +++ b/drivers/block/xen-blkfront.c |
311 | @@ -151,6 +151,10 @@ static unsigned int xen_blkif_max_ring_order; |
312 | module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444); |
313 | MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring"); |
314 | |
315 | +static bool __read_mostly xen_blkif_trusted = true; |
316 | +module_param_named(trusted, xen_blkif_trusted, bool, 0644); |
317 | +MODULE_PARM_DESC(trusted, "Is the backend trusted"); |
318 | + |
319 | #define BLK_RING_SIZE(info) \ |
320 | __CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages) |
321 | |
322 | @@ -211,6 +215,7 @@ struct blkfront_info |
323 | unsigned int feature_discard:1; |
324 | unsigned int feature_secdiscard:1; |
325 | unsigned int feature_persistent:1; |
326 | + unsigned int bounce:1; |
327 | unsigned int discard_granularity; |
328 | unsigned int discard_alignment; |
329 | /* Number of 4KB segments handled */ |
330 | @@ -300,8 +305,8 @@ static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num) |
331 | if (!gnt_list_entry) |
332 | goto out_of_memory; |
333 | |
334 | - if (info->feature_persistent) { |
335 | - granted_page = alloc_page(GFP_NOIO); |
336 | + if (info->bounce) { |
337 | + granted_page = alloc_page(GFP_NOIO | __GFP_ZERO); |
338 | if (!granted_page) { |
339 | kfree(gnt_list_entry); |
340 | goto out_of_memory; |
341 | @@ -320,7 +325,7 @@ out_of_memory: |
342 | list_for_each_entry_safe(gnt_list_entry, n, |
343 | &rinfo->grants, node) { |
344 | list_del(&gnt_list_entry->node); |
345 | - if (info->feature_persistent) |
346 | + if (info->bounce) |
347 | __free_page(gnt_list_entry->page); |
348 | kfree(gnt_list_entry); |
349 | i--; |
350 | @@ -366,7 +371,7 @@ static struct grant *get_grant(grant_ref_t *gref_head, |
351 | /* Assign a gref to this page */ |
352 | gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head); |
353 | BUG_ON(gnt_list_entry->gref == -ENOSPC); |
354 | - if (info->feature_persistent) |
355 | + if (info->bounce) |
356 | grant_foreign_access(gnt_list_entry, info); |
357 | else { |
358 | /* Grant access to the GFN passed by the caller */ |
359 | @@ -390,7 +395,7 @@ static struct grant *get_indirect_grant(grant_ref_t *gref_head, |
360 | /* Assign a gref to this page */ |
361 | gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head); |
362 | BUG_ON(gnt_list_entry->gref == -ENOSPC); |
363 | - if (!info->feature_persistent) { |
364 | + if (!info->bounce) { |
365 | struct page *indirect_page; |
366 | |
367 | /* Fetch a pre-allocated page to use for indirect grefs */ |
368 | @@ -705,7 +710,7 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri |
369 | .grant_idx = 0, |
370 | .segments = NULL, |
371 | .rinfo = rinfo, |
372 | - .need_copy = rq_data_dir(req) && info->feature_persistent, |
373 | + .need_copy = rq_data_dir(req) && info->bounce, |
374 | }; |
375 | |
376 | /* |
377 | @@ -1026,11 +1031,12 @@ static void xlvbd_flush(struct blkfront_info *info) |
378 | { |
379 | blk_queue_write_cache(info->rq, info->feature_flush ? true : false, |
380 | info->feature_fua ? true : false); |
381 | - pr_info("blkfront: %s: %s %s %s %s %s\n", |
382 | + pr_info("blkfront: %s: %s %s %s %s %s %s %s\n", |
383 | info->gd->disk_name, flush_info(info), |
384 | "persistent grants:", info->feature_persistent ? |
385 | "enabled;" : "disabled;", "indirect descriptors:", |
386 | - info->max_indirect_segments ? "enabled;" : "disabled;"); |
387 | + info->max_indirect_segments ? "enabled;" : "disabled;", |
388 | + "bounce buffer:", info->bounce ? "enabled" : "disabled;"); |
389 | } |
390 | |
391 | static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset) |
392 | @@ -1265,7 +1271,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo) |
393 | if (!list_empty(&rinfo->indirect_pages)) { |
394 | struct page *indirect_page, *n; |
395 | |
396 | - BUG_ON(info->feature_persistent); |
397 | + BUG_ON(info->bounce); |
398 | list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) { |
399 | list_del(&indirect_page->lru); |
400 | __free_page(indirect_page); |
401 | @@ -1282,7 +1288,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo) |
402 | 0, 0UL); |
403 | rinfo->persistent_gnts_c--; |
404 | } |
405 | - if (info->feature_persistent) |
406 | + if (info->bounce) |
407 | __free_page(persistent_gnt->page); |
408 | kfree(persistent_gnt); |
409 | } |
410 | @@ -1303,7 +1309,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo) |
411 | for (j = 0; j < segs; j++) { |
412 | persistent_gnt = rinfo->shadow[i].grants_used[j]; |
413 | gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL); |
414 | - if (info->feature_persistent) |
415 | + if (info->bounce) |
416 | __free_page(persistent_gnt->page); |
417 | kfree(persistent_gnt); |
418 | } |
419 | @@ -1493,7 +1499,7 @@ static int blkif_completion(unsigned long *id, |
420 | data.s = s; |
421 | num_sg = s->num_sg; |
422 | |
423 | - if (bret->operation == BLKIF_OP_READ && info->feature_persistent) { |
424 | + if (bret->operation == BLKIF_OP_READ && info->bounce) { |
425 | for_each_sg(s->sg, sg, num_sg, i) { |
426 | BUG_ON(sg->offset + sg->length > PAGE_SIZE); |
427 | |
428 | @@ -1552,7 +1558,7 @@ static int blkif_completion(unsigned long *id, |
429 | * Add the used indirect page back to the list of |
430 | * available pages for indirect grefs. |
431 | */ |
432 | - if (!info->feature_persistent) { |
433 | + if (!info->bounce) { |
434 | indirect_page = s->indirect_grants[i]->page; |
435 | list_add(&indirect_page->lru, &rinfo->indirect_pages); |
436 | } |
437 | @@ -1744,7 +1750,7 @@ static int setup_blkring(struct xenbus_device *dev, |
438 | for (i = 0; i < info->nr_ring_pages; i++) |
439 | rinfo->ring_ref[i] = GRANT_INVALID_REF; |
440 | |
441 | - sring = alloc_pages_exact(ring_size, GFP_NOIO); |
442 | + sring = alloc_pages_exact(ring_size, GFP_NOIO | __GFP_ZERO); |
443 | if (!sring) { |
444 | xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring"); |
445 | return -ENOMEM; |
446 | @@ -1847,6 +1853,10 @@ static int talk_to_blkback(struct xenbus_device *dev, |
447 | if (!info) |
448 | return -ENODEV; |
449 | |
450 | + /* Check if backend is trusted. */ |
451 | + info->bounce = !xen_blkif_trusted || |
452 | + !xenbus_read_unsigned(dev->nodename, "trusted", 1); |
453 | + |
454 | max_page_order = xenbus_read_unsigned(info->xbdev->otherend, |
455 | "max-ring-page-order", 0); |
456 | ring_page_order = min(xen_blkif_max_ring_order, max_page_order); |
457 | @@ -2273,17 +2283,18 @@ static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo) |
458 | if (err) |
459 | goto out_of_memory; |
460 | |
461 | - if (!info->feature_persistent && info->max_indirect_segments) { |
462 | + if (!info->bounce && info->max_indirect_segments) { |
463 | /* |
464 | - * We are using indirect descriptors but not persistent |
465 | - * grants, we need to allocate a set of pages that can be |
466 | + * We are using indirect descriptors but don't have a bounce |
467 | + * buffer, we need to allocate a set of pages that can be |
468 | * used for mapping indirect grefs |
469 | */ |
470 | int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info); |
471 | |
472 | BUG_ON(!list_empty(&rinfo->indirect_pages)); |
473 | for (i = 0; i < num; i++) { |
474 | - struct page *indirect_page = alloc_page(GFP_KERNEL); |
475 | + struct page *indirect_page = alloc_page(GFP_KERNEL | |
476 | + __GFP_ZERO); |
477 | if (!indirect_page) |
478 | goto out_of_memory; |
479 | list_add(&indirect_page->lru, &rinfo->indirect_pages); |
480 | @@ -2375,6 +2386,8 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) |
481 | info->feature_persistent = |
482 | !!xenbus_read_unsigned(info->xbdev->otherend, |
483 | "feature-persistent", 0); |
484 | + if (info->feature_persistent) |
485 | + info->bounce = true; |
486 | |
487 | indirect_segments = xenbus_read_unsigned(info->xbdev->otherend, |
488 | "feature-max-indirect-segments", 0); |
489 | @@ -2750,6 +2763,13 @@ static void blkfront_delay_work(struct work_struct *work) |
490 | struct blkfront_info *info; |
491 | bool need_schedule_work = false; |
492 | |
493 | + /* |
494 | + * Note that when using bounce buffers but not persistent grants |
495 | + * there's no need to run blkfront_delay_work because grants are |
496 | + * revoked in blkif_completion or else an error is reported and the |
497 | + * connection is closed. |
498 | + */ |
499 | + |
500 | mutex_lock(&blkfront_mutex); |
501 | |
502 | list_for_each_entry(info, &info_list, info_list) { |
503 | diff --git a/drivers/clocksource/timer-ixp4xx.c b/drivers/clocksource/timer-ixp4xx.c |
504 | index 9396745e1c179..ad904bbbac6fd 100644 |
505 | --- a/drivers/clocksource/timer-ixp4xx.c |
506 | +++ b/drivers/clocksource/timer-ixp4xx.c |
507 | @@ -258,7 +258,6 @@ void __init ixp4xx_timer_setup(resource_size_t timerbase, |
508 | } |
509 | ixp4xx_timer_register(base, timer_irq, timer_freq); |
510 | } |
511 | -EXPORT_SYMBOL_GPL(ixp4xx_timer_setup); |
512 | |
513 | #ifdef CONFIG_OF |
514 | static __init int ixp4xx_of_timer_init(struct device_node *np) |
515 | diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c |
516 | index c4873bb791f88..94e1727d29998 100644 |
517 | --- a/drivers/devfreq/event/exynos-ppmu.c |
518 | +++ b/drivers/devfreq/event/exynos-ppmu.c |
519 | @@ -514,15 +514,19 @@ static int of_get_devfreq_events(struct device_node *np, |
520 | |
521 | count = of_get_child_count(events_np); |
522 | desc = devm_kcalloc(dev, count, sizeof(*desc), GFP_KERNEL); |
523 | - if (!desc) |
524 | + if (!desc) { |
525 | + of_node_put(events_np); |
526 | return -ENOMEM; |
527 | + } |
528 | info->num_events = count; |
529 | |
530 | of_id = of_match_device(exynos_ppmu_id_match, dev); |
531 | if (of_id) |
532 | info->ppmu_type = (enum exynos_ppmu_type)of_id->data; |
533 | - else |
534 | + else { |
535 | + of_node_put(events_np); |
536 | return -EINVAL; |
537 | + } |
538 | |
539 | j = 0; |
540 | for_each_child_of_node(events_np, node) { |
541 | diff --git a/drivers/hwmon/ibmaem.c b/drivers/hwmon/ibmaem.c |
542 | index d05ab713566df..00a4ffe010393 100644 |
543 | --- a/drivers/hwmon/ibmaem.c |
544 | +++ b/drivers/hwmon/ibmaem.c |
545 | @@ -550,7 +550,7 @@ static int aem_init_aem1_inst(struct aem_ipmi_data *probe, u8 module_handle) |
546 | |
547 | res = platform_device_add(data->pdev); |
548 | if (res) |
549 | - goto ipmi_err; |
550 | + goto dev_add_err; |
551 | |
552 | platform_set_drvdata(data->pdev, data); |
553 | |
554 | @@ -598,7 +598,9 @@ hwmon_reg_err: |
555 | ipmi_destroy_user(data->ipmi.user); |
556 | ipmi_err: |
557 | platform_set_drvdata(data->pdev, NULL); |
558 | - platform_device_unregister(data->pdev); |
559 | + platform_device_del(data->pdev); |
560 | +dev_add_err: |
561 | + platform_device_put(data->pdev); |
562 | dev_err: |
563 | ida_simple_remove(&aem_ida, data->id); |
564 | id_err: |
565 | @@ -690,7 +692,7 @@ static int aem_init_aem2_inst(struct aem_ipmi_data *probe, |
566 | |
567 | res = platform_device_add(data->pdev); |
568 | if (res) |
569 | - goto ipmi_err; |
570 | + goto dev_add_err; |
571 | |
572 | platform_set_drvdata(data->pdev, data); |
573 | |
574 | @@ -738,7 +740,9 @@ hwmon_reg_err: |
575 | ipmi_destroy_user(data->ipmi.user); |
576 | ipmi_err: |
577 | platform_set_drvdata(data->pdev, NULL); |
578 | - platform_device_unregister(data->pdev); |
579 | + platform_device_del(data->pdev); |
580 | +dev_add_err: |
581 | + platform_device_put(data->pdev); |
582 | dev_err: |
583 | ida_simple_remove(&aem_ida, data->id); |
584 | id_err: |
585 | diff --git a/drivers/infiniband/hw/qedr/qedr.h b/drivers/infiniband/hw/qedr/qedr.h |
586 | index ed56df319d2df..f4b60a2d8aa91 100644 |
587 | --- a/drivers/infiniband/hw/qedr/qedr.h |
588 | +++ b/drivers/infiniband/hw/qedr/qedr.h |
589 | @@ -416,6 +416,7 @@ struct qedr_qp { |
590 | u32 sq_psn; |
591 | u32 qkey; |
592 | u32 dest_qp_num; |
593 | + u8 timeout; |
594 | |
595 | /* Relevant to qps created from kernel space only (ULPs) */ |
596 | u8 prev_wqe_size; |
597 | diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c |
598 | index 717f174647647..798a96c2278e8 100644 |
599 | --- a/drivers/infiniband/hw/qedr/verbs.c |
600 | +++ b/drivers/infiniband/hw/qedr/verbs.c |
601 | @@ -2259,6 +2259,8 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, |
602 | 1 << max_t(int, attr->timeout - 8, 0); |
603 | else |
604 | qp_params.ack_timeout = 0; |
605 | + |
606 | + qp->timeout = attr->timeout; |
607 | } |
608 | |
609 | if (attr_mask & IB_QP_RETRY_CNT) { |
610 | @@ -2418,7 +2420,7 @@ int qedr_query_qp(struct ib_qp *ibqp, |
611 | rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); |
612 | rdma_ah_set_port_num(&qp_attr->ah_attr, 1); |
613 | rdma_ah_set_sl(&qp_attr->ah_attr, 0); |
614 | - qp_attr->timeout = params.timeout; |
615 | + qp_attr->timeout = qp->timeout; |
616 | qp_attr->rnr_retry = params.rnr_retry; |
617 | qp_attr->retry_cnt = params.retry_cnt; |
618 | qp_attr->min_rnr_timer = params.min_rnr_nak_timer; |
619 | diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c |
620 | index 5e73cc6ad0ce4..42151c9fc6e56 100644 |
621 | --- a/drivers/md/dm-raid.c |
622 | +++ b/drivers/md/dm-raid.c |
623 | @@ -998,12 +998,13 @@ static int validate_region_size(struct raid_set *rs, unsigned long region_size) |
624 | static int validate_raid_redundancy(struct raid_set *rs) |
625 | { |
626 | unsigned int i, rebuild_cnt = 0; |
627 | - unsigned int rebuilds_per_group = 0, copies; |
628 | + unsigned int rebuilds_per_group = 0, copies, raid_disks; |
629 | unsigned int group_size, last_group_start; |
630 | |
631 | - for (i = 0; i < rs->md.raid_disks; i++) |
632 | - if (!test_bit(In_sync, &rs->dev[i].rdev.flags) || |
633 | - !rs->dev[i].rdev.sb_page) |
634 | + for (i = 0; i < rs->raid_disks; i++) |
635 | + if (!test_bit(FirstUse, &rs->dev[i].rdev.flags) && |
636 | + ((!test_bit(In_sync, &rs->dev[i].rdev.flags) || |
637 | + !rs->dev[i].rdev.sb_page))) |
638 | rebuild_cnt++; |
639 | |
640 | switch (rs->md.level) { |
641 | @@ -1043,8 +1044,9 @@ static int validate_raid_redundancy(struct raid_set *rs) |
642 | * A A B B C |
643 | * C D D E E |
644 | */ |
645 | + raid_disks = min(rs->raid_disks, rs->md.raid_disks); |
646 | if (__is_raid10_near(rs->md.new_layout)) { |
647 | - for (i = 0; i < rs->md.raid_disks; i++) { |
648 | + for (i = 0; i < raid_disks; i++) { |
649 | if (!(i % copies)) |
650 | rebuilds_per_group = 0; |
651 | if ((!rs->dev[i].rdev.sb_page || |
652 | @@ -1067,10 +1069,10 @@ static int validate_raid_redundancy(struct raid_set *rs) |
653 | * results in the need to treat the last (potentially larger) |
654 | * set differently. |
655 | */ |
656 | - group_size = (rs->md.raid_disks / copies); |
657 | - last_group_start = (rs->md.raid_disks / group_size) - 1; |
658 | + group_size = (raid_disks / copies); |
659 | + last_group_start = (raid_disks / group_size) - 1; |
660 | last_group_start *= group_size; |
661 | - for (i = 0; i < rs->md.raid_disks; i++) { |
662 | + for (i = 0; i < raid_disks; i++) { |
663 | if (!(i % copies) && !(i > last_group_start)) |
664 | rebuilds_per_group = 0; |
665 | if ((!rs->dev[i].rdev.sb_page || |
666 | @@ -1585,7 +1587,7 @@ static sector_t __rdev_sectors(struct raid_set *rs) |
667 | { |
668 | int i; |
669 | |
670 | - for (i = 0; i < rs->md.raid_disks; i++) { |
671 | + for (i = 0; i < rs->raid_disks; i++) { |
672 | struct md_rdev *rdev = &rs->dev[i].rdev; |
673 | |
674 | if (!test_bit(Journal, &rdev->flags) && |
675 | @@ -3746,13 +3748,13 @@ static int raid_iterate_devices(struct dm_target *ti, |
676 | unsigned int i; |
677 | int r = 0; |
678 | |
679 | - for (i = 0; !r && i < rs->md.raid_disks; i++) |
680 | - if (rs->dev[i].data_dev) |
681 | - r = fn(ti, |
682 | - rs->dev[i].data_dev, |
683 | - 0, /* No offset on data devs */ |
684 | - rs->md.dev_sectors, |
685 | - data); |
686 | + for (i = 0; !r && i < rs->raid_disks; i++) { |
687 | + if (rs->dev[i].data_dev) { |
688 | + r = fn(ti, rs->dev[i].data_dev, |
689 | + 0, /* No offset on data devs */ |
690 | + rs->md.dev_sectors, data); |
691 | + } |
692 | + } |
693 | |
694 | return r; |
695 | } |
696 | diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c |
697 | index c40327ad6ad53..474cf6abefea5 100644 |
698 | --- a/drivers/md/raid5.c |
699 | +++ b/drivers/md/raid5.c |
700 | @@ -7722,6 +7722,7 @@ static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) |
701 | */ |
702 | if (rdev->saved_raid_disk >= 0 && |
703 | rdev->saved_raid_disk >= first && |
704 | + rdev->saved_raid_disk <= last && |
705 | conf->disks[rdev->saved_raid_disk].rdev == NULL) |
706 | first = rdev->saved_raid_disk; |
707 | |
708 | diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c |
709 | index 41e58bd2e6069..31ed7616e84e7 100644 |
710 | --- a/drivers/net/bonding/bond_3ad.c |
711 | +++ b/drivers/net/bonding/bond_3ad.c |
712 | @@ -2218,7 +2218,8 @@ void bond_3ad_unbind_slave(struct slave *slave) |
713 | temp_aggregator->num_of_ports--; |
714 | if (__agg_active_ports(temp_aggregator) == 0) { |
715 | select_new_active_agg = temp_aggregator->is_active; |
716 | - ad_clear_agg(temp_aggregator); |
717 | + if (temp_aggregator->num_of_ports == 0) |
718 | + ad_clear_agg(temp_aggregator); |
719 | if (select_new_active_agg) { |
720 | slave_info(bond->dev, slave->dev, "Removing an active aggregator\n"); |
721 | /* select new active aggregator */ |
722 | diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c |
723 | index a4af185fa466d..8bee935c8f90f 100644 |
724 | --- a/drivers/net/bonding/bond_alb.c |
725 | +++ b/drivers/net/bonding/bond_alb.c |
726 | @@ -1276,12 +1276,12 @@ int bond_alb_initialize(struct bonding *bond, int rlb_enabled) |
727 | return res; |
728 | |
729 | if (rlb_enabled) { |
730 | - bond->alb_info.rlb_enabled = 1; |
731 | res = rlb_initialize(bond); |
732 | if (res) { |
733 | tlb_deinitialize(bond); |
734 | return res; |
735 | } |
736 | + bond->alb_info.rlb_enabled = 1; |
737 | } else { |
738 | bond->alb_info.rlb_enabled = 0; |
739 | } |
740 | diff --git a/drivers/net/caif/caif_virtio.c b/drivers/net/caif/caif_virtio.c |
741 | index eb426822ad066..7f2c551e5d690 100644 |
742 | --- a/drivers/net/caif/caif_virtio.c |
743 | +++ b/drivers/net/caif/caif_virtio.c |
744 | @@ -723,13 +723,21 @@ static int cfv_probe(struct virtio_device *vdev) |
745 | /* Carrier is off until netdevice is opened */ |
746 | netif_carrier_off(netdev); |
747 | |
748 | + /* serialize netdev register + virtio_device_ready() with ndo_open() */ |
749 | + rtnl_lock(); |
750 | + |
751 | /* register Netdev */ |
752 | - err = register_netdev(netdev); |
753 | + err = register_netdevice(netdev); |
754 | if (err) { |
755 | + rtnl_unlock(); |
756 | dev_err(&vdev->dev, "Unable to register netdev (%d)\n", err); |
757 | goto err; |
758 | } |
759 | |
760 | + virtio_device_ready(vdev); |
761 | + |
762 | + rtnl_unlock(); |
763 | + |
764 | debugfs_init(cfv); |
765 | |
766 | return 0; |
767 | diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c |
768 | index 0ee1c0a7b165b..282e9e2a50d9c 100644 |
769 | --- a/drivers/net/dsa/bcm_sf2.c |
770 | +++ b/drivers/net/dsa/bcm_sf2.c |
771 | @@ -602,6 +602,11 @@ force_link: |
772 | reg |= LINK_STS; |
773 | if (state->duplex == DUPLEX_FULL) |
774 | reg |= DUPLX_MODE; |
775 | + if (state->pause & MLO_PAUSE_TXRX_MASK) { |
776 | + if (state->pause & MLO_PAUSE_TX) |
777 | + reg |= TXFLOW_CNTL; |
778 | + reg |= RXFLOW_CNTL; |
779 | + } |
780 | |
781 | core_writel(priv, reg, offset); |
782 | } |
783 | diff --git a/drivers/net/tun.c b/drivers/net/tun.c |
784 | index d9993884a97d9..dd02fcc972774 100644 |
785 | --- a/drivers/net/tun.c |
786 | +++ b/drivers/net/tun.c |
787 | @@ -327,6 +327,12 @@ static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, |
788 | } |
789 | } |
790 | |
791 | +static void tun_napi_enable(struct tun_file *tfile) |
792 | +{ |
793 | + if (tfile->napi_enabled) |
794 | + napi_enable(&tfile->napi); |
795 | +} |
796 | + |
797 | static void tun_napi_disable(struct tun_file *tfile) |
798 | { |
799 | if (tfile->napi_enabled) |
800 | @@ -690,7 +696,8 @@ static void __tun_detach(struct tun_file *tfile, bool clean) |
801 | tun = rtnl_dereference(tfile->tun); |
802 | |
803 | if (tun && clean) { |
804 | - tun_napi_disable(tfile); |
805 | + if (!tfile->detached) |
806 | + tun_napi_disable(tfile); |
807 | tun_napi_del(tfile); |
808 | } |
809 | |
810 | @@ -709,8 +716,10 @@ static void __tun_detach(struct tun_file *tfile, bool clean) |
811 | if (clean) { |
812 | RCU_INIT_POINTER(tfile->tun, NULL); |
813 | sock_put(&tfile->sk); |
814 | - } else |
815 | + } else { |
816 | tun_disable_queue(tun, tfile); |
817 | + tun_napi_disable(tfile); |
818 | + } |
819 | |
820 | synchronize_net(); |
821 | tun_flow_delete_by_queue(tun, tun->numqueues + 1); |
822 | @@ -783,6 +792,7 @@ static void tun_detach_all(struct net_device *dev) |
823 | sock_put(&tfile->sk); |
824 | } |
825 | list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { |
826 | + tun_napi_del(tfile); |
827 | tun_enable_queue(tfile); |
828 | tun_queue_purge(tfile); |
829 | xdp_rxq_info_unreg(&tfile->xdp_rxq); |
830 | @@ -863,6 +873,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file, |
831 | |
832 | if (tfile->detached) { |
833 | tun_enable_queue(tfile); |
834 | + tun_napi_enable(tfile); |
835 | } else { |
836 | sock_hold(&tfile->sk); |
837 | tun_napi_init(tun, tfile, napi, napi_frags); |
838 | diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c |
839 | index a06c3924e0396..ea9c8361bf464 100644 |
840 | --- a/drivers/net/usb/ax88179_178a.c |
841 | +++ b/drivers/net/usb/ax88179_178a.c |
842 | @@ -1365,6 +1365,42 @@ static int ax88179_rx_fixup(struct usbnet *dev, struct sk_buff *skb) |
843 | * are bundled into this buffer and where we can find an array of |
844 | * per-packet metadata (which contains elements encoded into u16). |
845 | */ |
846 | + |
847 | + /* SKB contents for current firmware: |
848 | + * <packet 1> <padding> |
849 | + * ... |
850 | + * <packet N> <padding> |
851 | + * <per-packet metadata entry 1> <dummy header> |
852 | + * ... |
853 | + * <per-packet metadata entry N> <dummy header> |
854 | + * <padding2> <rx_hdr> |
855 | + * |
856 | + * where: |
857 | + * <packet N> contains pkt_len bytes: |
858 | + * 2 bytes of IP alignment pseudo header |
859 | + * packet received |
860 | + * <per-packet metadata entry N> contains 4 bytes: |
861 | + * pkt_len and fields AX_RXHDR_* |
862 | + * <padding> 0-7 bytes to terminate at |
863 | + * 8 bytes boundary (64-bit). |
864 | + * <padding2> 4 bytes to make rx_hdr terminate at |
865 | + * 8 bytes boundary (64-bit) |
866 | + * <dummy-header> contains 4 bytes: |
867 | + * pkt_len=0 and AX_RXHDR_DROP_ERR |
868 | + * <rx-hdr> contains 4 bytes: |
869 | + * pkt_cnt and hdr_off (offset of |
870 | + * <per-packet metadata entry 1>) |
871 | + * |
872 | + * pkt_cnt is number of entrys in the per-packet metadata. |
873 | + * In current firmware there is 2 entrys per packet. |
874 | + * The first points to the packet and the |
875 | + * second is a dummy header. |
876 | + * This was done probably to align fields in 64-bit and |
877 | + * maintain compatibility with old firmware. |
878 | + * This code assumes that <dummy header> and <padding2> are |
879 | + * optional. |
880 | + */ |
881 | + |
882 | if (skb->len < 4) |
883 | return 0; |
884 | skb_trim(skb, skb->len - 4); |
885 | @@ -1378,51 +1414,66 @@ static int ax88179_rx_fixup(struct usbnet *dev, struct sk_buff *skb) |
886 | /* Make sure that the bounds of the metadata array are inside the SKB |
887 | * (and in front of the counter at the end). |
888 | */ |
889 | - if (pkt_cnt * 2 + hdr_off > skb->len) |
890 | + if (pkt_cnt * 4 + hdr_off > skb->len) |
891 | return 0; |
892 | pkt_hdr = (u32 *)(skb->data + hdr_off); |
893 | |
894 | /* Packets must not overlap the metadata array */ |
895 | skb_trim(skb, hdr_off); |
896 | |
897 | - for (; ; pkt_cnt--, pkt_hdr++) { |
898 | + for (; pkt_cnt > 0; pkt_cnt--, pkt_hdr++) { |
899 | + u16 pkt_len_plus_padd; |
900 | u16 pkt_len; |
901 | |
902 | le32_to_cpus(pkt_hdr); |
903 | pkt_len = (*pkt_hdr >> 16) & 0x1fff; |
904 | + pkt_len_plus_padd = (pkt_len + 7) & 0xfff8; |
905 | |
906 | - if (pkt_len > skb->len) |
907 | + /* Skip dummy header used for alignment |
908 | + */ |
909 | + if (pkt_len == 0) |
910 | + continue; |
911 | + |
912 | + if (pkt_len_plus_padd > skb->len) |
913 | return 0; |
914 | |
915 | /* Check CRC or runt packet */ |
916 | - if (((*pkt_hdr & (AX_RXHDR_CRC_ERR | AX_RXHDR_DROP_ERR)) == 0) && |
917 | - pkt_len >= 2 + ETH_HLEN) { |
918 | - bool last = (pkt_cnt == 0); |
919 | - |
920 | - if (last) { |
921 | - ax_skb = skb; |
922 | - } else { |
923 | - ax_skb = skb_clone(skb, GFP_ATOMIC); |
924 | - if (!ax_skb) |
925 | - return 0; |
926 | - } |
927 | - ax_skb->len = pkt_len; |
928 | - /* Skip IP alignment pseudo header */ |
929 | - skb_pull(ax_skb, 2); |
930 | - skb_set_tail_pointer(ax_skb, ax_skb->len); |
931 | - ax_skb->truesize = pkt_len + sizeof(struct sk_buff); |
932 | - ax88179_rx_checksum(ax_skb, pkt_hdr); |
933 | + if ((*pkt_hdr & (AX_RXHDR_CRC_ERR | AX_RXHDR_DROP_ERR)) || |
934 | + pkt_len < 2 + ETH_HLEN) { |
935 | + dev->net->stats.rx_errors++; |
936 | + skb_pull(skb, pkt_len_plus_padd); |
937 | + continue; |
938 | + } |
939 | |
940 | - if (last) |
941 | - return 1; |
942 | + /* last packet */ |
943 | + if (pkt_len_plus_padd == skb->len) { |
944 | + skb_trim(skb, pkt_len); |
945 | |
946 | - usbnet_skb_return(dev, ax_skb); |
947 | + /* Skip IP alignment pseudo header */ |
948 | + skb_pull(skb, 2); |
949 | + |
950 | + skb->truesize = SKB_TRUESIZE(pkt_len_plus_padd); |
951 | + ax88179_rx_checksum(skb, pkt_hdr); |
952 | + return 1; |
953 | } |
954 | |
955 | - /* Trim this packet away from the SKB */ |
956 | - if (!skb_pull(skb, (pkt_len + 7) & 0xFFF8)) |
957 | + ax_skb = skb_clone(skb, GFP_ATOMIC); |
958 | + if (!ax_skb) |
959 | return 0; |
960 | + skb_trim(ax_skb, pkt_len); |
961 | + |
962 | + /* Skip IP alignment pseudo header */ |
963 | + skb_pull(ax_skb, 2); |
964 | + |
965 | + skb->truesize = pkt_len_plus_padd + |
966 | + SKB_DATA_ALIGN(sizeof(struct sk_buff)); |
967 | + ax88179_rx_checksum(ax_skb, pkt_hdr); |
968 | + usbnet_skb_return(dev, ax_skb); |
969 | + |
970 | + skb_pull(skb, pkt_len_plus_padd); |
971 | } |
972 | + |
973 | + return 0; |
974 | } |
975 | |
976 | static struct sk_buff * |
977 | diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c |
978 | index 566ea48fd6078..8ef0a013874c5 100644 |
979 | --- a/drivers/net/usb/qmi_wwan.c |
980 | +++ b/drivers/net/usb/qmi_wwan.c |
981 | @@ -1315,6 +1315,8 @@ static const struct usb_device_id products[] = { |
982 | {QMI_QUIRK_SET_DTR(0x1bc7, 0x1031, 3)}, /* Telit LE910C1-EUX */ |
983 | {QMI_QUIRK_SET_DTR(0x1bc7, 0x1040, 2)}, /* Telit LE922A */ |
984 | {QMI_QUIRK_SET_DTR(0x1bc7, 0x1050, 2)}, /* Telit FN980 */ |
985 | + {QMI_QUIRK_SET_DTR(0x1bc7, 0x1060, 2)}, /* Telit LN920 */ |
986 | + {QMI_QUIRK_SET_DTR(0x1bc7, 0x1070, 2)}, /* Telit FN990 */ |
987 | {QMI_FIXED_INTF(0x1bc7, 0x1100, 3)}, /* Telit ME910 */ |
988 | {QMI_FIXED_INTF(0x1bc7, 0x1101, 3)}, /* Telit ME910 dual modem */ |
989 | {QMI_FIXED_INTF(0x1bc7, 0x1200, 5)}, /* Telit LE920 */ |
990 | diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c |
991 | index b8b9df82f51ef..997936c3a30fd 100644 |
992 | --- a/drivers/net/usb/usbnet.c |
993 | +++ b/drivers/net/usb/usbnet.c |
994 | @@ -1987,7 +1987,7 @@ static int __usbnet_read_cmd(struct usbnet *dev, u8 cmd, u8 reqtype, |
995 | cmd, reqtype, value, index, size); |
996 | |
997 | if (size) { |
998 | - buf = kmalloc(size, GFP_KERNEL); |
999 | + buf = kmalloc(size, GFP_NOIO); |
1000 | if (!buf) |
1001 | goto out; |
1002 | } |
1003 | @@ -2019,7 +2019,7 @@ static int __usbnet_write_cmd(struct usbnet *dev, u8 cmd, u8 reqtype, |
1004 | cmd, reqtype, value, index, size); |
1005 | |
1006 | if (data) { |
1007 | - buf = kmemdup(data, size, GFP_KERNEL); |
1008 | + buf = kmemdup(data, size, GFP_NOIO); |
1009 | if (!buf) |
1010 | goto out; |
1011 | } else { |
1012 | diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c |
1013 | index f4c03518d7d23..e14842fbe3d62 100644 |
1014 | --- a/drivers/net/virtio_net.c |
1015 | +++ b/drivers/net/virtio_net.c |
1016 | @@ -3219,14 +3219,20 @@ static int virtnet_probe(struct virtio_device *vdev) |
1017 | } |
1018 | } |
1019 | |
1020 | - err = register_netdev(dev); |
1021 | + /* serialize netdev register + virtio_device_ready() with ndo_open() */ |
1022 | + rtnl_lock(); |
1023 | + |
1024 | + err = register_netdevice(dev); |
1025 | if (err) { |
1026 | pr_debug("virtio_net: registering device failed\n"); |
1027 | + rtnl_unlock(); |
1028 | goto free_failover; |
1029 | } |
1030 | |
1031 | virtio_device_ready(vdev); |
1032 | |
1033 | + rtnl_unlock(); |
1034 | + |
1035 | err = virtnet_cpu_notif_add(vi); |
1036 | if (err) { |
1037 | pr_debug("virtio_net: registering cpu notifier failed\n"); |
1038 | diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c |
1039 | index 94dd6edd18006..810fa9968be7d 100644 |
1040 | --- a/drivers/net/xen-netfront.c |
1041 | +++ b/drivers/net/xen-netfront.c |
1042 | @@ -63,6 +63,10 @@ module_param_named(max_queues, xennet_max_queues, uint, 0644); |
1043 | MODULE_PARM_DESC(max_queues, |
1044 | "Maximum number of queues per virtual interface"); |
1045 | |
1046 | +static bool __read_mostly xennet_trusted = true; |
1047 | +module_param_named(trusted, xennet_trusted, bool, 0644); |
1048 | +MODULE_PARM_DESC(trusted, "Is the backend trusted"); |
1049 | + |
1050 | #define XENNET_TIMEOUT (5 * HZ) |
1051 | |
1052 | static const struct ethtool_ops xennet_ethtool_ops; |
1053 | @@ -163,6 +167,9 @@ struct netfront_info { |
1054 | /* Is device behaving sane? */ |
1055 | bool broken; |
1056 | |
1057 | + /* Should skbs be bounced into a zeroed buffer? */ |
1058 | + bool bounce; |
1059 | + |
1060 | atomic_t rx_gso_checksum_fixup; |
1061 | }; |
1062 | |
1063 | @@ -261,7 +268,7 @@ static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue) |
1064 | if (unlikely(!skb)) |
1065 | return NULL; |
1066 | |
1067 | - page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); |
1068 | + page = alloc_page(GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO); |
1069 | if (!page) { |
1070 | kfree_skb(skb); |
1071 | return NULL; |
1072 | @@ -590,6 +597,34 @@ static void xennet_mark_tx_pending(struct netfront_queue *queue) |
1073 | queue->tx_link[i] = TX_PENDING; |
1074 | } |
1075 | |
1076 | +struct sk_buff *bounce_skb(const struct sk_buff *skb) |
1077 | +{ |
1078 | + unsigned int headerlen = skb_headroom(skb); |
1079 | + /* Align size to allocate full pages and avoid contiguous data leaks */ |
1080 | + unsigned int size = ALIGN(skb_end_offset(skb) + skb->data_len, |
1081 | + XEN_PAGE_SIZE); |
1082 | + struct sk_buff *n = alloc_skb(size, GFP_ATOMIC | __GFP_ZERO); |
1083 | + |
1084 | + if (!n) |
1085 | + return NULL; |
1086 | + |
1087 | + if (!IS_ALIGNED((uintptr_t)n->head, XEN_PAGE_SIZE)) { |
1088 | + WARN_ONCE(1, "misaligned skb allocated\n"); |
1089 | + kfree_skb(n); |
1090 | + return NULL; |
1091 | + } |
1092 | + |
1093 | + /* Set the data pointer */ |
1094 | + skb_reserve(n, headerlen); |
1095 | + /* Set the tail pointer and length */ |
1096 | + skb_put(n, skb->len); |
1097 | + |
1098 | + BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); |
1099 | + |
1100 | + skb_copy_header(n, skb); |
1101 | + return n; |
1102 | +} |
1103 | + |
1104 | #define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1) |
1105 | |
1106 | static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev) |
1107 | @@ -642,9 +677,13 @@ static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev |
1108 | |
1109 | /* The first req should be at least ETH_HLEN size or the packet will be |
1110 | * dropped by netback. |
1111 | + * |
1112 | + * If the backend is not trusted bounce all data to zeroed pages to |
1113 | + * avoid exposing contiguous data on the granted page not belonging to |
1114 | + * the skb. |
1115 | */ |
1116 | - if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) { |
1117 | - nskb = skb_copy(skb, GFP_ATOMIC); |
1118 | + if (np->bounce || unlikely(PAGE_SIZE - offset < ETH_HLEN)) { |
1119 | + nskb = bounce_skb(skb); |
1120 | if (!nskb) |
1121 | goto drop; |
1122 | dev_consume_skb_any(skb); |
1123 | @@ -1950,6 +1989,10 @@ static int talk_to_netback(struct xenbus_device *dev, |
1124 | |
1125 | info->netdev->irq = 0; |
1126 | |
1127 | + /* Check if backend is trusted. */ |
1128 | + info->bounce = !xennet_trusted || |
1129 | + !xenbus_read_unsigned(dev->nodename, "trusted", 1); |
1130 | + |
1131 | /* Check if backend supports multiple queues */ |
1132 | max_queues = xenbus_read_unsigned(info->xbdev->otherend, |
1133 | "multi-queue-max-queues", 1); |
1134 | @@ -2103,6 +2146,9 @@ static int xennet_connect(struct net_device *dev) |
1135 | err = talk_to_netback(np->xbdev, np); |
1136 | if (err) |
1137 | return err; |
1138 | + if (np->bounce) |
1139 | + dev_info(&np->xbdev->dev, |
1140 | + "bouncing transmitted data to zeroed pages\n"); |
1141 | |
1142 | /* talk_to_netback() sets the correct number of queues */ |
1143 | num_queues = dev->real_num_tx_queues; |
1144 | diff --git a/drivers/nfc/nfcmrvl/i2c.c b/drivers/nfc/nfcmrvl/i2c.c |
1145 | index 0f22379887ca8..919b4d2f5d8b5 100644 |
1146 | --- a/drivers/nfc/nfcmrvl/i2c.c |
1147 | +++ b/drivers/nfc/nfcmrvl/i2c.c |
1148 | @@ -186,9 +186,9 @@ static int nfcmrvl_i2c_parse_dt(struct device_node *node, |
1149 | pdata->irq_polarity = IRQF_TRIGGER_RISING; |
1150 | |
1151 | ret = irq_of_parse_and_map(node, 0); |
1152 | - if (ret < 0) { |
1153 | - pr_err("Unable to get irq, error: %d\n", ret); |
1154 | - return ret; |
1155 | + if (!ret) { |
1156 | + pr_err("Unable to get irq\n"); |
1157 | + return -EINVAL; |
1158 | } |
1159 | pdata->irq = ret; |
1160 | |
1161 | diff --git a/drivers/nfc/nfcmrvl/spi.c b/drivers/nfc/nfcmrvl/spi.c |
1162 | index 8e0ddb4347704..1f4120e3314b2 100644 |
1163 | --- a/drivers/nfc/nfcmrvl/spi.c |
1164 | +++ b/drivers/nfc/nfcmrvl/spi.c |
1165 | @@ -129,9 +129,9 @@ static int nfcmrvl_spi_parse_dt(struct device_node *node, |
1166 | } |
1167 | |
1168 | ret = irq_of_parse_and_map(node, 0); |
1169 | - if (ret < 0) { |
1170 | - pr_err("Unable to get irq, error: %d\n", ret); |
1171 | - return ret; |
1172 | + if (!ret) { |
1173 | + pr_err("Unable to get irq\n"); |
1174 | + return -EINVAL; |
1175 | } |
1176 | pdata->irq = ret; |
1177 | |
1178 | diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c |
1179 | index 9f60e4dc5a908..3943a30053b3b 100644 |
1180 | --- a/drivers/nfc/nxp-nci/i2c.c |
1181 | +++ b/drivers/nfc/nxp-nci/i2c.c |
1182 | @@ -162,6 +162,9 @@ static int nxp_nci_i2c_nci_read(struct nxp_nci_i2c_phy *phy, |
1183 | |
1184 | skb_put_data(*skb, (void *)&header, NCI_CTRL_HDR_SIZE); |
1185 | |
1186 | + if (!header.plen) |
1187 | + return 0; |
1188 | + |
1189 | r = i2c_master_recv(client, skb_put(*skb, header.plen), header.plen); |
1190 | if (r != header.plen) { |
1191 | nfc_err(&client->dev, |
1192 | diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c |
1193 | index 5e5c6aafc070b..56f189c3129af 100644 |
1194 | --- a/drivers/nvdimm/bus.c |
1195 | +++ b/drivers/nvdimm/bus.c |
1196 | @@ -187,8 +187,8 @@ static int nvdimm_clear_badblocks_region(struct device *dev, void *data) |
1197 | ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1; |
1198 | |
1199 | /* make sure we are in the region */ |
1200 | - if (ctx->phys < nd_region->ndr_start |
1201 | - || (ctx->phys + ctx->cleared) > ndr_end) |
1202 | + if (ctx->phys < nd_region->ndr_start || |
1203 | + (ctx->phys + ctx->cleared - 1) > ndr_end) |
1204 | return 0; |
1205 | |
1206 | sector = (ctx->phys - nd_region->ndr_start) / 512; |
1207 | diff --git a/drivers/xen/gntdev-common.h b/drivers/xen/gntdev-common.h |
1208 | index 2f8b949c3eeb1..fab6f5a54d5b2 100644 |
1209 | --- a/drivers/xen/gntdev-common.h |
1210 | +++ b/drivers/xen/gntdev-common.h |
1211 | @@ -15,6 +15,8 @@ |
1212 | #include <linux/mman.h> |
1213 | #include <linux/mmu_notifier.h> |
1214 | #include <linux/types.h> |
1215 | +#include <xen/interface/event_channel.h> |
1216 | +#include <xen/grant_table.h> |
1217 | |
1218 | struct gntdev_dmabuf_priv; |
1219 | |
1220 | @@ -61,6 +63,7 @@ struct gntdev_grant_map { |
1221 | struct gnttab_unmap_grant_ref *unmap_ops; |
1222 | struct gnttab_map_grant_ref *kmap_ops; |
1223 | struct gnttab_unmap_grant_ref *kunmap_ops; |
1224 | + bool *being_removed; |
1225 | struct page **pages; |
1226 | unsigned long pages_vm_start; |
1227 | |
1228 | @@ -78,6 +81,11 @@ struct gntdev_grant_map { |
1229 | /* Needed to avoid allocation in gnttab_dma_free_pages(). */ |
1230 | xen_pfn_t *frames; |
1231 | #endif |
1232 | + |
1233 | + /* Number of live grants */ |
1234 | + atomic_t live_grants; |
1235 | + /* Needed to avoid allocation in __unmap_grant_pages */ |
1236 | + struct gntab_unmap_queue_data unmap_data; |
1237 | }; |
1238 | |
1239 | struct gntdev_grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count, |
1240 | diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c |
1241 | index e953ea34b3e43..f464793477650 100644 |
1242 | --- a/drivers/xen/gntdev.c |
1243 | +++ b/drivers/xen/gntdev.c |
1244 | @@ -35,6 +35,7 @@ |
1245 | #include <linux/slab.h> |
1246 | #include <linux/highmem.h> |
1247 | #include <linux/refcount.h> |
1248 | +#include <linux/workqueue.h> |
1249 | |
1250 | #include <xen/xen.h> |
1251 | #include <xen/grant_table.h> |
1252 | @@ -62,11 +63,12 @@ MODULE_PARM_DESC(limit, "Maximum number of grants that may be mapped by " |
1253 | |
1254 | static atomic_t pages_mapped = ATOMIC_INIT(0); |
1255 | |
1256 | +/* True in PV mode, false otherwise */ |
1257 | static int use_ptemod; |
1258 | #define populate_freeable_maps use_ptemod |
1259 | |
1260 | -static int unmap_grant_pages(struct gntdev_grant_map *map, |
1261 | - int offset, int pages); |
1262 | +static void unmap_grant_pages(struct gntdev_grant_map *map, |
1263 | + int offset, int pages); |
1264 | |
1265 | static struct miscdevice gntdev_miscdev; |
1266 | |
1267 | @@ -123,6 +125,7 @@ static void gntdev_free_map(struct gntdev_grant_map *map) |
1268 | kfree(map->unmap_ops); |
1269 | kfree(map->kmap_ops); |
1270 | kfree(map->kunmap_ops); |
1271 | + kfree(map->being_removed); |
1272 | kfree(map); |
1273 | } |
1274 | |
1275 | @@ -142,12 +145,15 @@ struct gntdev_grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count, |
1276 | add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL); |
1277 | add->kunmap_ops = kcalloc(count, sizeof(add->kunmap_ops[0]), GFP_KERNEL); |
1278 | add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL); |
1279 | + add->being_removed = |
1280 | + kcalloc(count, sizeof(add->being_removed[0]), GFP_KERNEL); |
1281 | if (NULL == add->grants || |
1282 | NULL == add->map_ops || |
1283 | NULL == add->unmap_ops || |
1284 | NULL == add->kmap_ops || |
1285 | NULL == add->kunmap_ops || |
1286 | - NULL == add->pages) |
1287 | + NULL == add->pages || |
1288 | + NULL == add->being_removed) |
1289 | goto err; |
1290 | |
1291 | #ifdef CONFIG_XEN_GRANT_DMA_ALLOC |
1292 | @@ -243,6 +249,35 @@ void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map) |
1293 | return; |
1294 | |
1295 | atomic_sub(map->count, &pages_mapped); |
1296 | + if (map->pages && !use_ptemod) { |
1297 | + /* |
1298 | + * Increment the reference count. This ensures that the |
1299 | + * subsequent call to unmap_grant_pages() will not wind up |
1300 | + * re-entering itself. It *can* wind up calling |
1301 | + * gntdev_put_map() recursively, but such calls will be with a |
1302 | + * reference count greater than 1, so they will return before |
1303 | + * this code is reached. The recursion depth is thus limited to |
1304 | + * 1. Do NOT use refcount_inc() here, as it will detect that |
1305 | + * the reference count is zero and WARN(). |
1306 | + */ |
1307 | + refcount_set(&map->users, 1); |
1308 | + |
1309 | + /* |
1310 | + * Unmap the grants. This may or may not be asynchronous, so it |
1311 | + * is possible that the reference count is 1 on return, but it |
1312 | + * could also be greater than 1. |
1313 | + */ |
1314 | + unmap_grant_pages(map, 0, map->count); |
1315 | + |
1316 | + /* Check if the memory now needs to be freed */ |
1317 | + if (!refcount_dec_and_test(&map->users)) |
1318 | + return; |
1319 | + |
1320 | + /* |
1321 | + * All pages have been returned to the hypervisor, so free the |
1322 | + * map. |
1323 | + */ |
1324 | + } |
1325 | |
1326 | if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) { |
1327 | notify_remote_via_evtchn(map->notify.event); |
1328 | @@ -298,6 +333,7 @@ static int set_grant_ptes_as_special(pte_t *pte, unsigned long addr, void *data) |
1329 | |
1330 | int gntdev_map_grant_pages(struct gntdev_grant_map *map) |
1331 | { |
1332 | + size_t alloced = 0; |
1333 | int i, err = 0; |
1334 | |
1335 | if (!use_ptemod) { |
1336 | @@ -346,87 +382,109 @@ int gntdev_map_grant_pages(struct gntdev_grant_map *map) |
1337 | map->pages, map->count); |
1338 | |
1339 | for (i = 0; i < map->count; i++) { |
1340 | - if (map->map_ops[i].status == GNTST_okay) |
1341 | + if (map->map_ops[i].status == GNTST_okay) { |
1342 | map->unmap_ops[i].handle = map->map_ops[i].handle; |
1343 | - else if (!err) |
1344 | + if (!use_ptemod) |
1345 | + alloced++; |
1346 | + } else if (!err) |
1347 | err = -EINVAL; |
1348 | |
1349 | if (map->flags & GNTMAP_device_map) |
1350 | map->unmap_ops[i].dev_bus_addr = map->map_ops[i].dev_bus_addr; |
1351 | |
1352 | if (use_ptemod) { |
1353 | - if (map->kmap_ops[i].status == GNTST_okay) |
1354 | + if (map->kmap_ops[i].status == GNTST_okay) { |
1355 | + if (map->map_ops[i].status == GNTST_okay) |
1356 | + alloced++; |
1357 | map->kunmap_ops[i].handle = map->kmap_ops[i].handle; |
1358 | - else if (!err) |
1359 | + } else if (!err) |
1360 | err = -EINVAL; |
1361 | } |
1362 | } |
1363 | + atomic_add(alloced, &map->live_grants); |
1364 | return err; |
1365 | } |
1366 | |
1367 | -static int __unmap_grant_pages(struct gntdev_grant_map *map, int offset, |
1368 | - int pages) |
1369 | +static void __unmap_grant_pages_done(int result, |
1370 | + struct gntab_unmap_queue_data *data) |
1371 | { |
1372 | - int i, err = 0; |
1373 | - struct gntab_unmap_queue_data unmap_data; |
1374 | + unsigned int i; |
1375 | + struct gntdev_grant_map *map = data->data; |
1376 | + unsigned int offset = data->unmap_ops - map->unmap_ops; |
1377 | |
1378 | + for (i = 0; i < data->count; i++) { |
1379 | + WARN_ON(map->unmap_ops[offset+i].status); |
1380 | + pr_debug("unmap handle=%d st=%d\n", |
1381 | + map->unmap_ops[offset+i].handle, |
1382 | + map->unmap_ops[offset+i].status); |
1383 | + map->unmap_ops[offset+i].handle = -1; |
1384 | + } |
1385 | + /* |
1386 | + * Decrease the live-grant counter. This must happen after the loop to |
1387 | + * prevent premature reuse of the grants by gnttab_mmap(). |
1388 | + */ |
1389 | + atomic_sub(data->count, &map->live_grants); |
1390 | + |
1391 | + /* Release reference taken by __unmap_grant_pages */ |
1392 | + gntdev_put_map(NULL, map); |
1393 | +} |
1394 | + |
1395 | +static void __unmap_grant_pages(struct gntdev_grant_map *map, int offset, |
1396 | + int pages) |
1397 | +{ |
1398 | if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { |
1399 | int pgno = (map->notify.addr >> PAGE_SHIFT); |
1400 | + |
1401 | if (pgno >= offset && pgno < offset + pages) { |
1402 | /* No need for kmap, pages are in lowmem */ |
1403 | uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno])); |
1404 | + |
1405 | tmp[map->notify.addr & (PAGE_SIZE-1)] = 0; |
1406 | map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE; |
1407 | } |
1408 | } |
1409 | |
1410 | - unmap_data.unmap_ops = map->unmap_ops + offset; |
1411 | - unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL; |
1412 | - unmap_data.pages = map->pages + offset; |
1413 | - unmap_data.count = pages; |
1414 | + map->unmap_data.unmap_ops = map->unmap_ops + offset; |
1415 | + map->unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL; |
1416 | + map->unmap_data.pages = map->pages + offset; |
1417 | + map->unmap_data.count = pages; |
1418 | + map->unmap_data.done = __unmap_grant_pages_done; |
1419 | + map->unmap_data.data = map; |
1420 | + refcount_inc(&map->users); /* to keep map alive during async call below */ |
1421 | |
1422 | - err = gnttab_unmap_refs_sync(&unmap_data); |
1423 | - if (err) |
1424 | - return err; |
1425 | - |
1426 | - for (i = 0; i < pages; i++) { |
1427 | - if (map->unmap_ops[offset+i].status) |
1428 | - err = -EINVAL; |
1429 | - pr_debug("unmap handle=%d st=%d\n", |
1430 | - map->unmap_ops[offset+i].handle, |
1431 | - map->unmap_ops[offset+i].status); |
1432 | - map->unmap_ops[offset+i].handle = -1; |
1433 | - } |
1434 | - return err; |
1435 | + gnttab_unmap_refs_async(&map->unmap_data); |
1436 | } |
1437 | |
1438 | -static int unmap_grant_pages(struct gntdev_grant_map *map, int offset, |
1439 | - int pages) |
1440 | +static void unmap_grant_pages(struct gntdev_grant_map *map, int offset, |
1441 | + int pages) |
1442 | { |
1443 | - int range, err = 0; |
1444 | + int range; |
1445 | + |
1446 | + if (atomic_read(&map->live_grants) == 0) |
1447 | + return; /* Nothing to do */ |
1448 | |
1449 | pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages); |
1450 | |
1451 | /* It is possible the requested range will have a "hole" where we |
1452 | * already unmapped some of the grants. Only unmap valid ranges. |
1453 | */ |
1454 | - while (pages && !err) { |
1455 | - while (pages && map->unmap_ops[offset].handle == -1) { |
1456 | + while (pages) { |
1457 | + while (pages && map->being_removed[offset]) { |
1458 | offset++; |
1459 | pages--; |
1460 | } |
1461 | range = 0; |
1462 | while (range < pages) { |
1463 | - if (map->unmap_ops[offset+range].handle == -1) |
1464 | + if (map->being_removed[offset + range]) |
1465 | break; |
1466 | + map->being_removed[offset + range] = true; |
1467 | range++; |
1468 | } |
1469 | - err = __unmap_grant_pages(map, offset, range); |
1470 | + if (range) |
1471 | + __unmap_grant_pages(map, offset, range); |
1472 | offset += range; |
1473 | pages -= range; |
1474 | } |
1475 | - |
1476 | - return err; |
1477 | } |
1478 | |
1479 | /* ------------------------------------------------------------------ */ |
1480 | @@ -496,7 +554,6 @@ static int unmap_if_in_range(struct gntdev_grant_map *map, |
1481 | bool blockable) |
1482 | { |
1483 | unsigned long mstart, mend; |
1484 | - int err; |
1485 | |
1486 | if (!in_range(map, start, end)) |
1487 | return 0; |
1488 | @@ -510,10 +567,9 @@ static int unmap_if_in_range(struct gntdev_grant_map *map, |
1489 | map->index, map->count, |
1490 | map->vma->vm_start, map->vma->vm_end, |
1491 | start, end, mstart, mend); |
1492 | - err = unmap_grant_pages(map, |
1493 | + unmap_grant_pages(map, |
1494 | (mstart - map->vma->vm_start) >> PAGE_SHIFT, |
1495 | (mend - mstart) >> PAGE_SHIFT); |
1496 | - WARN_ON(err); |
1497 | |
1498 | return 0; |
1499 | } |
1500 | @@ -554,7 +610,6 @@ static void mn_release(struct mmu_notifier *mn, |
1501 | { |
1502 | struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); |
1503 | struct gntdev_grant_map *map; |
1504 | - int err; |
1505 | |
1506 | mutex_lock(&priv->lock); |
1507 | list_for_each_entry(map, &priv->maps, next) { |
1508 | @@ -563,8 +618,7 @@ static void mn_release(struct mmu_notifier *mn, |
1509 | pr_debug("map %d+%d (%lx %lx)\n", |
1510 | map->index, map->count, |
1511 | map->vma->vm_start, map->vma->vm_end); |
1512 | - err = unmap_grant_pages(map, /* offset */ 0, map->count); |
1513 | - WARN_ON(err); |
1514 | + unmap_grant_pages(map, /* offset */ 0, map->count); |
1515 | } |
1516 | list_for_each_entry(map, &priv->freeable_maps, next) { |
1517 | if (!map->vma) |
1518 | @@ -572,8 +626,7 @@ static void mn_release(struct mmu_notifier *mn, |
1519 | pr_debug("map %d+%d (%lx %lx)\n", |
1520 | map->index, map->count, |
1521 | map->vma->vm_start, map->vma->vm_end); |
1522 | - err = unmap_grant_pages(map, /* offset */ 0, map->count); |
1523 | - WARN_ON(err); |
1524 | + unmap_grant_pages(map, /* offset */ 0, map->count); |
1525 | } |
1526 | mutex_unlock(&priv->lock); |
1527 | } |
1528 | @@ -1102,6 +1155,10 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) |
1529 | goto unlock_out; |
1530 | } |
1531 | |
1532 | + if (atomic_read(&map->live_grants)) { |
1533 | + err = -EAGAIN; |
1534 | + goto unlock_out; |
1535 | + } |
1536 | refcount_inc(&map->users); |
1537 | |
1538 | vma->vm_ops = &gntdev_vmops; |
1539 | diff --git a/include/linux/dim.h b/include/linux/dim.h |
1540 | index 9fa4b3f88c397..2571da63877c5 100644 |
1541 | --- a/include/linux/dim.h |
1542 | +++ b/include/linux/dim.h |
1543 | @@ -17,7 +17,7 @@ |
1544 | * We consider 10% difference as significant. |
1545 | */ |
1546 | #define IS_SIGNIFICANT_DIFF(val, ref) \ |
1547 | - (((100UL * abs((val) - (ref))) / (ref)) > 10) |
1548 | + ((ref) && (((100UL * abs((val) - (ref))) / (ref)) > 10)) |
1549 | |
1550 | /** |
1551 | * Calculate the gap between two values. |
1552 | diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c |
1553 | index e29553e4f4ee3..a0123760fb2c7 100644 |
1554 | --- a/net/ipv6/addrconf.c |
1555 | +++ b/net/ipv6/addrconf.c |
1556 | @@ -1102,10 +1102,6 @@ ipv6_add_addr(struct inet6_dev *idev, struct ifa6_config *cfg, |
1557 | goto out; |
1558 | } |
1559 | |
1560 | - if (net->ipv6.devconf_all->disable_policy || |
1561 | - idev->cnf.disable_policy) |
1562 | - f6i->dst_nopolicy = true; |
1563 | - |
1564 | neigh_parms_data_state_setall(idev->nd_parms); |
1565 | |
1566 | ifa->addr = *cfg->pfx; |
1567 | diff --git a/net/ipv6/route.c b/net/ipv6/route.c |
1568 | index 619d9dffa9e44..00732ee6bbd8a 100644 |
1569 | --- a/net/ipv6/route.c |
1570 | +++ b/net/ipv6/route.c |
1571 | @@ -4483,8 +4483,15 @@ struct fib6_info *addrconf_f6i_alloc(struct net *net, |
1572 | } |
1573 | |
1574 | f6i = ip6_route_info_create(&cfg, gfp_flags, NULL); |
1575 | - if (!IS_ERR(f6i)) |
1576 | + if (!IS_ERR(f6i)) { |
1577 | f6i->dst_nocount = true; |
1578 | + |
1579 | + if (!anycast && |
1580 | + (net->ipv6.devconf_all->disable_policy || |
1581 | + idev->cnf.disable_policy)) |
1582 | + f6i->dst_nopolicy = true; |
1583 | + } |
1584 | + |
1585 | return f6i; |
1586 | } |
1587 | |
1588 | diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c |
1589 | index a4cad71c42047..65394abf4736b 100644 |
1590 | --- a/net/ipv6/seg6_hmac.c |
1591 | +++ b/net/ipv6/seg6_hmac.c |
1592 | @@ -410,7 +410,6 @@ int __net_init seg6_hmac_net_init(struct net *net) |
1593 | |
1594 | return 0; |
1595 | } |
1596 | -EXPORT_SYMBOL(seg6_hmac_net_init); |
1597 | |
1598 | void seg6_hmac_exit(void) |
1599 | { |
1600 | diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c |
1601 | index 16e75a996b749..117d374695fe6 100644 |
1602 | --- a/net/ipv6/sit.c |
1603 | +++ b/net/ipv6/sit.c |
1604 | @@ -310,9 +310,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t, |
1605 | kcalloc(cmax, sizeof(*kp), GFP_KERNEL | __GFP_NOWARN) : |
1606 | NULL; |
1607 | |
1608 | - rcu_read_lock(); |
1609 | - |
1610 | - ca = t->prl_count < cmax ? t->prl_count : cmax; |
1611 | + ca = min(t->prl_count, cmax); |
1612 | |
1613 | if (!kp) { |
1614 | /* We don't try hard to allocate much memory for |
1615 | @@ -327,7 +325,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t, |
1616 | } |
1617 | } |
1618 | |
1619 | - c = 0; |
1620 | + rcu_read_lock(); |
1621 | for_each_prl_rcu(t->prl) { |
1622 | if (c >= cmax) |
1623 | break; |
1624 | @@ -339,7 +337,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t, |
1625 | if (kprl.addr != htonl(INADDR_ANY)) |
1626 | break; |
1627 | } |
1628 | -out: |
1629 | + |
1630 | rcu_read_unlock(); |
1631 | |
1632 | len = sizeof(*kp) * c; |
1633 | @@ -348,7 +346,7 @@ out: |
1634 | ret = -EFAULT; |
1635 | |
1636 | kfree(kp); |
1637 | - |
1638 | +out: |
1639 | return ret; |
1640 | } |
1641 | |
1642 | diff --git a/net/netfilter/nft_set_hash.c b/net/netfilter/nft_set_hash.c |
1643 | index 9de0eb20e9544..e7eb56b4b89e5 100644 |
1644 | --- a/net/netfilter/nft_set_hash.c |
1645 | +++ b/net/netfilter/nft_set_hash.c |
1646 | @@ -142,6 +142,7 @@ static bool nft_rhash_update(struct nft_set *set, const u32 *key, |
1647 | /* Another cpu may race to insert the element with the same key */ |
1648 | if (prev) { |
1649 | nft_set_elem_destroy(set, he, true); |
1650 | + atomic_dec(&set->nelems); |
1651 | he = prev; |
1652 | } |
1653 | |
1654 | @@ -151,6 +152,7 @@ out: |
1655 | |
1656 | err2: |
1657 | nft_set_elem_destroy(set, he, true); |
1658 | + atomic_dec(&set->nelems); |
1659 | err1: |
1660 | return false; |
1661 | } |
1662 | diff --git a/net/rose/rose_timer.c b/net/rose/rose_timer.c |
1663 | index b3138fc2e552e..f06ddbed3fed6 100644 |
1664 | --- a/net/rose/rose_timer.c |
1665 | +++ b/net/rose/rose_timer.c |
1666 | @@ -31,89 +31,89 @@ static void rose_idletimer_expiry(struct timer_list *); |
1667 | |
1668 | void rose_start_heartbeat(struct sock *sk) |
1669 | { |
1670 | - del_timer(&sk->sk_timer); |
1671 | + sk_stop_timer(sk, &sk->sk_timer); |
1672 | |
1673 | sk->sk_timer.function = rose_heartbeat_expiry; |
1674 | sk->sk_timer.expires = jiffies + 5 * HZ; |
1675 | |
1676 | - add_timer(&sk->sk_timer); |
1677 | + sk_reset_timer(sk, &sk->sk_timer, sk->sk_timer.expires); |
1678 | } |
1679 | |
1680 | void rose_start_t1timer(struct sock *sk) |
1681 | { |
1682 | struct rose_sock *rose = rose_sk(sk); |
1683 | |
1684 | - del_timer(&rose->timer); |
1685 | + sk_stop_timer(sk, &rose->timer); |
1686 | |
1687 | rose->timer.function = rose_timer_expiry; |
1688 | rose->timer.expires = jiffies + rose->t1; |
1689 | |
1690 | - add_timer(&rose->timer); |
1691 | + sk_reset_timer(sk, &rose->timer, rose->timer.expires); |
1692 | } |
1693 | |
1694 | void rose_start_t2timer(struct sock *sk) |
1695 | { |
1696 | struct rose_sock *rose = rose_sk(sk); |
1697 | |
1698 | - del_timer(&rose->timer); |
1699 | + sk_stop_timer(sk, &rose->timer); |
1700 | |
1701 | rose->timer.function = rose_timer_expiry; |
1702 | rose->timer.expires = jiffies + rose->t2; |
1703 | |
1704 | - add_timer(&rose->timer); |
1705 | + sk_reset_timer(sk, &rose->timer, rose->timer.expires); |
1706 | } |
1707 | |
1708 | void rose_start_t3timer(struct sock *sk) |
1709 | { |
1710 | struct rose_sock *rose = rose_sk(sk); |
1711 | |
1712 | - del_timer(&rose->timer); |
1713 | + sk_stop_timer(sk, &rose->timer); |
1714 | |
1715 | rose->timer.function = rose_timer_expiry; |
1716 | rose->timer.expires = jiffies + rose->t3; |
1717 | |
1718 | - add_timer(&rose->timer); |
1719 | + sk_reset_timer(sk, &rose->timer, rose->timer.expires); |
1720 | } |
1721 | |
1722 | void rose_start_hbtimer(struct sock *sk) |
1723 | { |
1724 | struct rose_sock *rose = rose_sk(sk); |
1725 | |
1726 | - del_timer(&rose->timer); |
1727 | + sk_stop_timer(sk, &rose->timer); |
1728 | |
1729 | rose->timer.function = rose_timer_expiry; |
1730 | rose->timer.expires = jiffies + rose->hb; |
1731 | |
1732 | - add_timer(&rose->timer); |
1733 | + sk_reset_timer(sk, &rose->timer, rose->timer.expires); |
1734 | } |
1735 | |
1736 | void rose_start_idletimer(struct sock *sk) |
1737 | { |
1738 | struct rose_sock *rose = rose_sk(sk); |
1739 | |
1740 | - del_timer(&rose->idletimer); |
1741 | + sk_stop_timer(sk, &rose->idletimer); |
1742 | |
1743 | if (rose->idle > 0) { |
1744 | rose->idletimer.function = rose_idletimer_expiry; |
1745 | rose->idletimer.expires = jiffies + rose->idle; |
1746 | |
1747 | - add_timer(&rose->idletimer); |
1748 | + sk_reset_timer(sk, &rose->idletimer, rose->idletimer.expires); |
1749 | } |
1750 | } |
1751 | |
1752 | void rose_stop_heartbeat(struct sock *sk) |
1753 | { |
1754 | - del_timer(&sk->sk_timer); |
1755 | + sk_stop_timer(sk, &sk->sk_timer); |
1756 | } |
1757 | |
1758 | void rose_stop_timer(struct sock *sk) |
1759 | { |
1760 | - del_timer(&rose_sk(sk)->timer); |
1761 | + sk_stop_timer(sk, &rose_sk(sk)->timer); |
1762 | } |
1763 | |
1764 | void rose_stop_idletimer(struct sock *sk) |
1765 | { |
1766 | - del_timer(&rose_sk(sk)->idletimer); |
1767 | + sk_stop_timer(sk, &rose_sk(sk)->idletimer); |
1768 | } |
1769 | |
1770 | static void rose_heartbeat_expiry(struct timer_list *t) |
1771 | @@ -130,6 +130,7 @@ static void rose_heartbeat_expiry(struct timer_list *t) |
1772 | (sk->sk_state == TCP_LISTEN && sock_flag(sk, SOCK_DEAD))) { |
1773 | bh_unlock_sock(sk); |
1774 | rose_destroy_socket(sk); |
1775 | + sock_put(sk); |
1776 | return; |
1777 | } |
1778 | break; |
1779 | @@ -152,6 +153,7 @@ static void rose_heartbeat_expiry(struct timer_list *t) |
1780 | |
1781 | rose_start_heartbeat(sk); |
1782 | bh_unlock_sock(sk); |
1783 | + sock_put(sk); |
1784 | } |
1785 | |
1786 | static void rose_timer_expiry(struct timer_list *t) |
1787 | @@ -181,6 +183,7 @@ static void rose_timer_expiry(struct timer_list *t) |
1788 | break; |
1789 | } |
1790 | bh_unlock_sock(sk); |
1791 | + sock_put(sk); |
1792 | } |
1793 | |
1794 | static void rose_idletimer_expiry(struct timer_list *t) |
1795 | @@ -205,4 +208,5 @@ static void rose_idletimer_expiry(struct timer_list *t) |
1796 | sock_set_flag(sk, SOCK_DEAD); |
1797 | } |
1798 | bh_unlock_sock(sk); |
1799 | + sock_put(sk); |
1800 | } |
1801 | diff --git a/net/sched/act_api.c b/net/sched/act_api.c |
1802 | index ab277ee950322..db1c0139f99c6 100644 |
1803 | --- a/net/sched/act_api.c |
1804 | +++ b/net/sched/act_api.c |
1805 | @@ -287,7 +287,8 @@ static int tcf_idr_release_unsafe(struct tc_action *p) |
1806 | } |
1807 | |
1808 | static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, |
1809 | - const struct tc_action_ops *ops) |
1810 | + const struct tc_action_ops *ops, |
1811 | + struct netlink_ext_ack *extack) |
1812 | { |
1813 | struct nlattr *nest; |
1814 | int n_i = 0; |
1815 | @@ -303,20 +304,25 @@ static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, |
1816 | if (nla_put_string(skb, TCA_KIND, ops->kind)) |
1817 | goto nla_put_failure; |
1818 | |
1819 | + ret = 0; |
1820 | mutex_lock(&idrinfo->lock); |
1821 | idr_for_each_entry_ul(idr, p, tmp, id) { |
1822 | if (IS_ERR(p)) |
1823 | continue; |
1824 | ret = tcf_idr_release_unsafe(p); |
1825 | - if (ret == ACT_P_DELETED) { |
1826 | + if (ret == ACT_P_DELETED) |
1827 | module_put(ops->owner); |
1828 | - n_i++; |
1829 | - } else if (ret < 0) { |
1830 | - mutex_unlock(&idrinfo->lock); |
1831 | - goto nla_put_failure; |
1832 | - } |
1833 | + else if (ret < 0) |
1834 | + break; |
1835 | + n_i++; |
1836 | } |
1837 | mutex_unlock(&idrinfo->lock); |
1838 | + if (ret < 0) { |
1839 | + if (n_i) |
1840 | + NL_SET_ERR_MSG(extack, "Unable to flush all TC actions"); |
1841 | + else |
1842 | + goto nla_put_failure; |
1843 | + } |
1844 | |
1845 | ret = nla_put_u32(skb, TCA_FCNT, n_i); |
1846 | if (ret) |
1847 | @@ -337,7 +343,7 @@ int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb, |
1848 | struct tcf_idrinfo *idrinfo = tn->idrinfo; |
1849 | |
1850 | if (type == RTM_DELACTION) { |
1851 | - return tcf_del_walker(idrinfo, skb, ops); |
1852 | + return tcf_del_walker(idrinfo, skb, ops, extack); |
1853 | } else if (type == RTM_GETACTION) { |
1854 | return tcf_dump_walker(idrinfo, skb, cb); |
1855 | } else { |
1856 | diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c |
1857 | index cb8740d156336..daa4165f11794 100644 |
1858 | --- a/net/sunrpc/xdr.c |
1859 | +++ b/net/sunrpc/xdr.c |
1860 | @@ -608,7 +608,7 @@ static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, |
1861 | */ |
1862 | xdr->p = (void *)p + frag2bytes; |
1863 | space_left = xdr->buf->buflen - xdr->buf->len; |
1864 | - if (space_left - nbytes >= PAGE_SIZE) |
1865 | + if (space_left - frag1bytes >= PAGE_SIZE) |
1866 | xdr->end = (void *)p + PAGE_SIZE; |
1867 | else |
1868 | xdr->end = (void *)p + space_left - frag1bytes; |
1869 | diff --git a/tools/testing/selftests/net/udpgso_bench.sh b/tools/testing/selftests/net/udpgso_bench.sh |
1870 | index 80b5d352702e5..dc932fd653634 100755 |
1871 | --- a/tools/testing/selftests/net/udpgso_bench.sh |
1872 | +++ b/tools/testing/selftests/net/udpgso_bench.sh |
1873 | @@ -120,7 +120,7 @@ run_all() { |
1874 | run_udp "${ipv4_args}" |
1875 | |
1876 | echo "ipv6" |
1877 | - run_tcp "${ipv4_args}" |
1878 | + run_tcp "${ipv6_args}" |
1879 | run_udp "${ipv6_args}" |
1880 | } |
1881 | |
1882 | diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile |
1883 | index 2af9d39a97168..215e1067f0376 100644 |
1884 | --- a/tools/testing/selftests/rseq/Makefile |
1885 | +++ b/tools/testing/selftests/rseq/Makefile |
1886 | @@ -6,7 +6,7 @@ endif |
1887 | |
1888 | CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L$(OUTPUT) -Wl,-rpath=./ \ |
1889 | $(CLANG_FLAGS) |
1890 | -LDLIBS += -lpthread |
1891 | +LDLIBS += -lpthread -ldl |
1892 | |
1893 | # Own dependencies because we only want to build against 1st prerequisite, but |
1894 | # still track changes to header files and depend on shared object. |
1895 | diff --git a/tools/testing/selftests/rseq/basic_percpu_ops_test.c b/tools/testing/selftests/rseq/basic_percpu_ops_test.c |
1896 | index eb3f6db36d369..517756afc2a4e 100644 |
1897 | --- a/tools/testing/selftests/rseq/basic_percpu_ops_test.c |
1898 | +++ b/tools/testing/selftests/rseq/basic_percpu_ops_test.c |
1899 | @@ -9,10 +9,9 @@ |
1900 | #include <string.h> |
1901 | #include <stddef.h> |
1902 | |
1903 | +#include "../kselftest.h" |
1904 | #include "rseq.h" |
1905 | |
1906 | -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) |
1907 | - |
1908 | struct percpu_lock_entry { |
1909 | intptr_t v; |
1910 | } __attribute__((aligned(128))); |
1911 | @@ -168,7 +167,7 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, |
1912 | for (;;) { |
1913 | struct percpu_list_node *head; |
1914 | intptr_t *targetptr, expectnot, *load; |
1915 | - off_t offset; |
1916 | + long offset; |
1917 | int ret, cpu; |
1918 | |
1919 | cpu = rseq_cpu_start(); |
1920 | diff --git a/tools/testing/selftests/rseq/compiler.h b/tools/testing/selftests/rseq/compiler.h |
1921 | new file mode 100644 |
1922 | index 0000000000000..876eb6a7f75be |
1923 | --- /dev/null |
1924 | +++ b/tools/testing/selftests/rseq/compiler.h |
1925 | @@ -0,0 +1,30 @@ |
1926 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
1927 | +/* |
1928 | + * rseq/compiler.h |
1929 | + * |
1930 | + * Work-around asm goto compiler bugs. |
1931 | + * |
1932 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
1933 | + */ |
1934 | + |
1935 | +#ifndef RSEQ_COMPILER_H |
1936 | +#define RSEQ_COMPILER_H |
1937 | + |
1938 | +/* |
1939 | + * gcc prior to 4.8.2 miscompiles asm goto. |
1940 | + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 |
1941 | + * |
1942 | + * gcc prior to 8.1.0 miscompiles asm goto at O1. |
1943 | + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103908 |
1944 | + * |
1945 | + * clang prior to version 13.0.1 miscompiles asm goto at O2. |
1946 | + * https://github.com/llvm/llvm-project/issues/52735 |
1947 | + * |
1948 | + * Work around these issues by adding a volatile inline asm with |
1949 | + * memory clobber in the fallthrough after the asm goto and at each |
1950 | + * label target. Emit this for all compilers in case other similar |
1951 | + * issues are found in the future. |
1952 | + */ |
1953 | +#define rseq_after_asm_goto() asm volatile ("" : : : "memory") |
1954 | + |
1955 | +#endif /* RSEQ_COMPILER_H_ */ |
1956 | diff --git a/tools/testing/selftests/rseq/param_test.c b/tools/testing/selftests/rseq/param_test.c |
1957 | index e8a657a5f48a0..1f399aaf1efe2 100644 |
1958 | --- a/tools/testing/selftests/rseq/param_test.c |
1959 | +++ b/tools/testing/selftests/rseq/param_test.c |
1960 | @@ -159,7 +159,7 @@ unsigned int yield_mod_cnt, nr_abort; |
1961 | " cbnz " INJECT_ASM_REG ", 222b\n" \ |
1962 | "333:\n" |
1963 | |
1964 | -#elif __PPC__ |
1965 | +#elif defined(__PPC__) |
1966 | |
1967 | #define RSEQ_INJECT_INPUT \ |
1968 | , [loop_cnt_1]"m"(loop_cnt[1]) \ |
1969 | @@ -366,9 +366,7 @@ void *test_percpu_spinlock_thread(void *arg) |
1970 | abort(); |
1971 | reps = thread_data->reps; |
1972 | for (i = 0; i < reps; i++) { |
1973 | - int cpu = rseq_cpu_start(); |
1974 | - |
1975 | - cpu = rseq_this_cpu_lock(&data->lock); |
1976 | + int cpu = rseq_this_cpu_lock(&data->lock); |
1977 | data->c[cpu].count++; |
1978 | rseq_percpu_unlock(&data->lock, cpu); |
1979 | #ifndef BENCHMARK |
1980 | @@ -549,7 +547,7 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, |
1981 | for (;;) { |
1982 | struct percpu_list_node *head; |
1983 | intptr_t *targetptr, expectnot, *load; |
1984 | - off_t offset; |
1985 | + long offset; |
1986 | int ret; |
1987 | |
1988 | cpu = rseq_cpu_start(); |
1989 | diff --git a/tools/testing/selftests/rseq/rseq-abi.h b/tools/testing/selftests/rseq/rseq-abi.h |
1990 | new file mode 100644 |
1991 | index 0000000000000..a8c44d9af71fb |
1992 | --- /dev/null |
1993 | +++ b/tools/testing/selftests/rseq/rseq-abi.h |
1994 | @@ -0,0 +1,151 @@ |
1995 | +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ |
1996 | +#ifndef _RSEQ_ABI_H |
1997 | +#define _RSEQ_ABI_H |
1998 | + |
1999 | +/* |
2000 | + * rseq-abi.h |
2001 | + * |
2002 | + * Restartable sequences system call API |
2003 | + * |
2004 | + * Copyright (c) 2015-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
2005 | + */ |
2006 | + |
2007 | +#include <linux/types.h> |
2008 | +#include <asm/byteorder.h> |
2009 | + |
2010 | +enum rseq_abi_cpu_id_state { |
2011 | + RSEQ_ABI_CPU_ID_UNINITIALIZED = -1, |
2012 | + RSEQ_ABI_CPU_ID_REGISTRATION_FAILED = -2, |
2013 | +}; |
2014 | + |
2015 | +enum rseq_abi_flags { |
2016 | + RSEQ_ABI_FLAG_UNREGISTER = (1 << 0), |
2017 | +}; |
2018 | + |
2019 | +enum rseq_abi_cs_flags_bit { |
2020 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0, |
2021 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1, |
2022 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2, |
2023 | +}; |
2024 | + |
2025 | +enum rseq_abi_cs_flags { |
2026 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT = |
2027 | + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT), |
2028 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL = |
2029 | + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT), |
2030 | + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE = |
2031 | + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT), |
2032 | +}; |
2033 | + |
2034 | +/* |
2035 | + * struct rseq_abi_cs is aligned on 4 * 8 bytes to ensure it is always |
2036 | + * contained within a single cache-line. It is usually declared as |
2037 | + * link-time constant data. |
2038 | + */ |
2039 | +struct rseq_abi_cs { |
2040 | + /* Version of this structure. */ |
2041 | + __u32 version; |
2042 | + /* enum rseq_abi_cs_flags */ |
2043 | + __u32 flags; |
2044 | + __u64 start_ip; |
2045 | + /* Offset from start_ip. */ |
2046 | + __u64 post_commit_offset; |
2047 | + __u64 abort_ip; |
2048 | +} __attribute__((aligned(4 * sizeof(__u64)))); |
2049 | + |
2050 | +/* |
2051 | + * struct rseq_abi is aligned on 4 * 8 bytes to ensure it is always |
2052 | + * contained within a single cache-line. |
2053 | + * |
2054 | + * A single struct rseq_abi per thread is allowed. |
2055 | + */ |
2056 | +struct rseq_abi { |
2057 | + /* |
2058 | + * Restartable sequences cpu_id_start field. Updated by the |
2059 | + * kernel. Read by user-space with single-copy atomicity |
2060 | + * semantics. This field should only be read by the thread which |
2061 | + * registered this data structure. Aligned on 32-bit. Always |
2062 | + * contains a value in the range of possible CPUs, although the |
2063 | + * value may not be the actual current CPU (e.g. if rseq is not |
2064 | + * initialized). This CPU number value should always be compared |
2065 | + * against the value of the cpu_id field before performing a rseq |
2066 | + * commit or returning a value read from a data structure indexed |
2067 | + * using the cpu_id_start value. |
2068 | + */ |
2069 | + __u32 cpu_id_start; |
2070 | + /* |
2071 | + * Restartable sequences cpu_id field. Updated by the kernel. |
2072 | + * Read by user-space with single-copy atomicity semantics. This |
2073 | + * field should only be read by the thread which registered this |
2074 | + * data structure. Aligned on 32-bit. Values |
2075 | + * RSEQ_CPU_ID_UNINITIALIZED and RSEQ_CPU_ID_REGISTRATION_FAILED |
2076 | + * have a special semantic: the former means "rseq uninitialized", |
2077 | + * and latter means "rseq initialization failed". This value is |
2078 | + * meant to be read within rseq critical sections and compared |
2079 | + * with the cpu_id_start value previously read, before performing |
2080 | + * the commit instruction, or read and compared with the |
2081 | + * cpu_id_start value before returning a value loaded from a data |
2082 | + * structure indexed using the cpu_id_start value. |
2083 | + */ |
2084 | + __u32 cpu_id; |
2085 | + /* |
2086 | + * Restartable sequences rseq_cs field. |
2087 | + * |
2088 | + * Contains NULL when no critical section is active for the current |
2089 | + * thread, or holds a pointer to the currently active struct rseq_cs. |
2090 | + * |
2091 | + * Updated by user-space, which sets the address of the currently |
2092 | + * active rseq_cs at the beginning of assembly instruction sequence |
2093 | + * block, and set to NULL by the kernel when it restarts an assembly |
2094 | + * instruction sequence block, as well as when the kernel detects that |
2095 | + * it is preempting or delivering a signal outside of the range |
2096 | + * targeted by the rseq_cs. Also needs to be set to NULL by user-space |
2097 | + * before reclaiming memory that contains the targeted struct rseq_cs. |
2098 | + * |
2099 | + * Read and set by the kernel. Set by user-space with single-copy |
2100 | + * atomicity semantics. This field should only be updated by the |
2101 | + * thread which registered this data structure. Aligned on 64-bit. |
2102 | + */ |
2103 | + union { |
2104 | + __u64 ptr64; |
2105 | + |
2106 | + /* |
2107 | + * The "arch" field provides architecture accessor for |
2108 | + * the ptr field based on architecture pointer size and |
2109 | + * endianness. |
2110 | + */ |
2111 | + struct { |
2112 | +#ifdef __LP64__ |
2113 | + __u64 ptr; |
2114 | +#elif defined(__BYTE_ORDER) ? (__BYTE_ORDER == __BIG_ENDIAN) : defined(__BIG_ENDIAN) |
2115 | + __u32 padding; /* Initialized to zero. */ |
2116 | + __u32 ptr; |
2117 | +#else |
2118 | + __u32 ptr; |
2119 | + __u32 padding; /* Initialized to zero. */ |
2120 | +#endif |
2121 | + } arch; |
2122 | + } rseq_cs; |
2123 | + |
2124 | + /* |
2125 | + * Restartable sequences flags field. |
2126 | + * |
2127 | + * This field should only be updated by the thread which |
2128 | + * registered this data structure. Read by the kernel. |
2129 | + * Mainly used for single-stepping through rseq critical sections |
2130 | + * with debuggers. |
2131 | + * |
2132 | + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT |
2133 | + * Inhibit instruction sequence block restart on preemption |
2134 | + * for this thread. |
2135 | + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL |
2136 | + * Inhibit instruction sequence block restart on signal |
2137 | + * delivery for this thread. |
2138 | + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE |
2139 | + * Inhibit instruction sequence block restart on migration for |
2140 | + * this thread. |
2141 | + */ |
2142 | + __u32 flags; |
2143 | +} __attribute__((aligned(4 * sizeof(__u64)))); |
2144 | + |
2145 | +#endif /* _RSEQ_ABI_H */ |
2146 | diff --git a/tools/testing/selftests/rseq/rseq-arm.h b/tools/testing/selftests/rseq/rseq-arm.h |
2147 | index 5943c816c07ce..893a11eca9d51 100644 |
2148 | --- a/tools/testing/selftests/rseq/rseq-arm.h |
2149 | +++ b/tools/testing/selftests/rseq/rseq-arm.h |
2150 | @@ -147,14 +147,11 @@ do { \ |
2151 | teardown \ |
2152 | "b %l[" __rseq_str(cmpfail_label) "]\n\t" |
2153 | |
2154 | -#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") |
2155 | - |
2156 | static inline __attribute__((always_inline)) |
2157 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
2158 | { |
2159 | RSEQ_INJECT_C(9) |
2160 | |
2161 | - rseq_workaround_gcc_asm_size_guess(); |
2162 | __asm__ __volatile__ goto ( |
2163 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2164 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2165 | @@ -185,8 +182,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
2166 | "5:\n\t" |
2167 | : /* gcc asm goto does not allow outputs */ |
2168 | : [cpu_id] "r" (cpu), |
2169 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2170 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2171 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2172 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2173 | [v] "m" (*v), |
2174 | [expect] "r" (expect), |
2175 | [newv] "r" (newv) |
2176 | @@ -198,30 +195,31 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
2177 | , error1, error2 |
2178 | #endif |
2179 | ); |
2180 | - rseq_workaround_gcc_asm_size_guess(); |
2181 | + rseq_after_asm_goto(); |
2182 | return 0; |
2183 | abort: |
2184 | - rseq_workaround_gcc_asm_size_guess(); |
2185 | + rseq_after_asm_goto(); |
2186 | RSEQ_INJECT_FAILED |
2187 | return -1; |
2188 | cmpfail: |
2189 | - rseq_workaround_gcc_asm_size_guess(); |
2190 | + rseq_after_asm_goto(); |
2191 | return 1; |
2192 | #ifdef RSEQ_COMPARE_TWICE |
2193 | error1: |
2194 | + rseq_after_asm_goto(); |
2195 | rseq_bug("cpu_id comparison failed"); |
2196 | error2: |
2197 | + rseq_after_asm_goto(); |
2198 | rseq_bug("expected value comparison failed"); |
2199 | #endif |
2200 | } |
2201 | |
2202 | static inline __attribute__((always_inline)) |
2203 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2204 | - off_t voffp, intptr_t *load, int cpu) |
2205 | + long voffp, intptr_t *load, int cpu) |
2206 | { |
2207 | RSEQ_INJECT_C(9) |
2208 | |
2209 | - rseq_workaround_gcc_asm_size_guess(); |
2210 | __asm__ __volatile__ goto ( |
2211 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2212 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2213 | @@ -255,8 +253,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2214 | "5:\n\t" |
2215 | : /* gcc asm goto does not allow outputs */ |
2216 | : [cpu_id] "r" (cpu), |
2217 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2218 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2219 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2220 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2221 | /* final store input */ |
2222 | [v] "m" (*v), |
2223 | [expectnot] "r" (expectnot), |
2224 | @@ -270,19 +268,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2225 | , error1, error2 |
2226 | #endif |
2227 | ); |
2228 | - rseq_workaround_gcc_asm_size_guess(); |
2229 | + rseq_after_asm_goto(); |
2230 | return 0; |
2231 | abort: |
2232 | - rseq_workaround_gcc_asm_size_guess(); |
2233 | + rseq_after_asm_goto(); |
2234 | RSEQ_INJECT_FAILED |
2235 | return -1; |
2236 | cmpfail: |
2237 | - rseq_workaround_gcc_asm_size_guess(); |
2238 | + rseq_after_asm_goto(); |
2239 | return 1; |
2240 | #ifdef RSEQ_COMPARE_TWICE |
2241 | error1: |
2242 | + rseq_after_asm_goto(); |
2243 | rseq_bug("cpu_id comparison failed"); |
2244 | error2: |
2245 | + rseq_after_asm_goto(); |
2246 | rseq_bug("expected value comparison failed"); |
2247 | #endif |
2248 | } |
2249 | @@ -292,7 +292,6 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
2250 | { |
2251 | RSEQ_INJECT_C(9) |
2252 | |
2253 | - rseq_workaround_gcc_asm_size_guess(); |
2254 | __asm__ __volatile__ goto ( |
2255 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2256 | #ifdef RSEQ_COMPARE_TWICE |
2257 | @@ -316,8 +315,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
2258 | "5:\n\t" |
2259 | : /* gcc asm goto does not allow outputs */ |
2260 | : [cpu_id] "r" (cpu), |
2261 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2262 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2263 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2264 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2265 | [v] "m" (*v), |
2266 | [count] "Ir" (count) |
2267 | RSEQ_INJECT_INPUT |
2268 | @@ -328,14 +327,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
2269 | , error1 |
2270 | #endif |
2271 | ); |
2272 | - rseq_workaround_gcc_asm_size_guess(); |
2273 | + rseq_after_asm_goto(); |
2274 | return 0; |
2275 | abort: |
2276 | - rseq_workaround_gcc_asm_size_guess(); |
2277 | + rseq_after_asm_goto(); |
2278 | RSEQ_INJECT_FAILED |
2279 | return -1; |
2280 | #ifdef RSEQ_COMPARE_TWICE |
2281 | error1: |
2282 | + rseq_after_asm_goto(); |
2283 | rseq_bug("cpu_id comparison failed"); |
2284 | #endif |
2285 | } |
2286 | @@ -347,7 +347,6 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
2287 | { |
2288 | RSEQ_INJECT_C(9) |
2289 | |
2290 | - rseq_workaround_gcc_asm_size_guess(); |
2291 | __asm__ __volatile__ goto ( |
2292 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2293 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2294 | @@ -381,8 +380,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
2295 | "5:\n\t" |
2296 | : /* gcc asm goto does not allow outputs */ |
2297 | : [cpu_id] "r" (cpu), |
2298 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2299 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2300 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2301 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2302 | /* try store input */ |
2303 | [v2] "m" (*v2), |
2304 | [newv2] "r" (newv2), |
2305 | @@ -398,19 +397,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
2306 | , error1, error2 |
2307 | #endif |
2308 | ); |
2309 | - rseq_workaround_gcc_asm_size_guess(); |
2310 | + rseq_after_asm_goto(); |
2311 | return 0; |
2312 | abort: |
2313 | - rseq_workaround_gcc_asm_size_guess(); |
2314 | + rseq_after_asm_goto(); |
2315 | RSEQ_INJECT_FAILED |
2316 | return -1; |
2317 | cmpfail: |
2318 | - rseq_workaround_gcc_asm_size_guess(); |
2319 | + rseq_after_asm_goto(); |
2320 | return 1; |
2321 | #ifdef RSEQ_COMPARE_TWICE |
2322 | error1: |
2323 | + rseq_after_asm_goto(); |
2324 | rseq_bug("cpu_id comparison failed"); |
2325 | error2: |
2326 | + rseq_after_asm_goto(); |
2327 | rseq_bug("expected value comparison failed"); |
2328 | #endif |
2329 | } |
2330 | @@ -422,7 +423,6 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
2331 | { |
2332 | RSEQ_INJECT_C(9) |
2333 | |
2334 | - rseq_workaround_gcc_asm_size_guess(); |
2335 | __asm__ __volatile__ goto ( |
2336 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2337 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2338 | @@ -457,8 +457,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
2339 | "5:\n\t" |
2340 | : /* gcc asm goto does not allow outputs */ |
2341 | : [cpu_id] "r" (cpu), |
2342 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2343 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2344 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2345 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2346 | /* try store input */ |
2347 | [v2] "m" (*v2), |
2348 | [newv2] "r" (newv2), |
2349 | @@ -474,19 +474,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
2350 | , error1, error2 |
2351 | #endif |
2352 | ); |
2353 | - rseq_workaround_gcc_asm_size_guess(); |
2354 | + rseq_after_asm_goto(); |
2355 | return 0; |
2356 | abort: |
2357 | - rseq_workaround_gcc_asm_size_guess(); |
2358 | + rseq_after_asm_goto(); |
2359 | RSEQ_INJECT_FAILED |
2360 | return -1; |
2361 | cmpfail: |
2362 | - rseq_workaround_gcc_asm_size_guess(); |
2363 | + rseq_after_asm_goto(); |
2364 | return 1; |
2365 | #ifdef RSEQ_COMPARE_TWICE |
2366 | error1: |
2367 | + rseq_after_asm_goto(); |
2368 | rseq_bug("cpu_id comparison failed"); |
2369 | error2: |
2370 | + rseq_after_asm_goto(); |
2371 | rseq_bug("expected value comparison failed"); |
2372 | #endif |
2373 | } |
2374 | @@ -498,7 +500,6 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
2375 | { |
2376 | RSEQ_INJECT_C(9) |
2377 | |
2378 | - rseq_workaround_gcc_asm_size_guess(); |
2379 | __asm__ __volatile__ goto ( |
2380 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2381 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2382 | @@ -537,8 +538,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
2383 | "5:\n\t" |
2384 | : /* gcc asm goto does not allow outputs */ |
2385 | : [cpu_id] "r" (cpu), |
2386 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2387 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2388 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2389 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2390 | /* cmp2 input */ |
2391 | [v2] "m" (*v2), |
2392 | [expect2] "r" (expect2), |
2393 | @@ -554,21 +555,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
2394 | , error1, error2, error3 |
2395 | #endif |
2396 | ); |
2397 | - rseq_workaround_gcc_asm_size_guess(); |
2398 | + rseq_after_asm_goto(); |
2399 | return 0; |
2400 | abort: |
2401 | - rseq_workaround_gcc_asm_size_guess(); |
2402 | + rseq_after_asm_goto(); |
2403 | RSEQ_INJECT_FAILED |
2404 | return -1; |
2405 | cmpfail: |
2406 | - rseq_workaround_gcc_asm_size_guess(); |
2407 | + rseq_after_asm_goto(); |
2408 | return 1; |
2409 | #ifdef RSEQ_COMPARE_TWICE |
2410 | error1: |
2411 | + rseq_after_asm_goto(); |
2412 | rseq_bug("cpu_id comparison failed"); |
2413 | error2: |
2414 | + rseq_after_asm_goto(); |
2415 | rseq_bug("1st expected value comparison failed"); |
2416 | error3: |
2417 | + rseq_after_asm_goto(); |
2418 | rseq_bug("2nd expected value comparison failed"); |
2419 | #endif |
2420 | } |
2421 | @@ -582,7 +586,6 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
2422 | |
2423 | RSEQ_INJECT_C(9) |
2424 | |
2425 | - rseq_workaround_gcc_asm_size_guess(); |
2426 | __asm__ __volatile__ goto ( |
2427 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2428 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2429 | @@ -657,8 +660,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
2430 | "8:\n\t" |
2431 | : /* gcc asm goto does not allow outputs */ |
2432 | : [cpu_id] "r" (cpu), |
2433 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2434 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2435 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2436 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2437 | /* final store input */ |
2438 | [v] "m" (*v), |
2439 | [expect] "r" (expect), |
2440 | @@ -678,21 +681,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
2441 | , error1, error2 |
2442 | #endif |
2443 | ); |
2444 | - rseq_workaround_gcc_asm_size_guess(); |
2445 | + rseq_after_asm_goto(); |
2446 | return 0; |
2447 | abort: |
2448 | - rseq_workaround_gcc_asm_size_guess(); |
2449 | + rseq_after_asm_goto(); |
2450 | RSEQ_INJECT_FAILED |
2451 | return -1; |
2452 | cmpfail: |
2453 | - rseq_workaround_gcc_asm_size_guess(); |
2454 | + rseq_after_asm_goto(); |
2455 | return 1; |
2456 | #ifdef RSEQ_COMPARE_TWICE |
2457 | error1: |
2458 | - rseq_workaround_gcc_asm_size_guess(); |
2459 | + rseq_after_asm_goto(); |
2460 | rseq_bug("cpu_id comparison failed"); |
2461 | error2: |
2462 | - rseq_workaround_gcc_asm_size_guess(); |
2463 | + rseq_after_asm_goto(); |
2464 | rseq_bug("expected value comparison failed"); |
2465 | #endif |
2466 | } |
2467 | @@ -706,7 +709,6 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
2468 | |
2469 | RSEQ_INJECT_C(9) |
2470 | |
2471 | - rseq_workaround_gcc_asm_size_guess(); |
2472 | __asm__ __volatile__ goto ( |
2473 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2474 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2475 | @@ -782,8 +784,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
2476 | "8:\n\t" |
2477 | : /* gcc asm goto does not allow outputs */ |
2478 | : [cpu_id] "r" (cpu), |
2479 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2480 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2481 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2482 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2483 | /* final store input */ |
2484 | [v] "m" (*v), |
2485 | [expect] "r" (expect), |
2486 | @@ -803,21 +805,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
2487 | , error1, error2 |
2488 | #endif |
2489 | ); |
2490 | - rseq_workaround_gcc_asm_size_guess(); |
2491 | + rseq_after_asm_goto(); |
2492 | return 0; |
2493 | abort: |
2494 | - rseq_workaround_gcc_asm_size_guess(); |
2495 | + rseq_after_asm_goto(); |
2496 | RSEQ_INJECT_FAILED |
2497 | return -1; |
2498 | cmpfail: |
2499 | - rseq_workaround_gcc_asm_size_guess(); |
2500 | + rseq_after_asm_goto(); |
2501 | return 1; |
2502 | #ifdef RSEQ_COMPARE_TWICE |
2503 | error1: |
2504 | - rseq_workaround_gcc_asm_size_guess(); |
2505 | + rseq_after_asm_goto(); |
2506 | rseq_bug("cpu_id comparison failed"); |
2507 | error2: |
2508 | - rseq_workaround_gcc_asm_size_guess(); |
2509 | + rseq_after_asm_goto(); |
2510 | rseq_bug("expected value comparison failed"); |
2511 | #endif |
2512 | } |
2513 | diff --git a/tools/testing/selftests/rseq/rseq-arm64.h b/tools/testing/selftests/rseq/rseq-arm64.h |
2514 | index 200dae9e4208c..cbe190a4d0056 100644 |
2515 | --- a/tools/testing/selftests/rseq/rseq-arm64.h |
2516 | +++ b/tools/testing/selftests/rseq/rseq-arm64.h |
2517 | @@ -230,8 +230,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
2518 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
2519 | : /* gcc asm goto does not allow outputs */ |
2520 | : [cpu_id] "r" (cpu), |
2521 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), |
2522 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2523 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), |
2524 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2525 | [v] "Qo" (*v), |
2526 | [expect] "r" (expect), |
2527 | [newv] "r" (newv) |
2528 | @@ -242,24 +242,28 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
2529 | , error1, error2 |
2530 | #endif |
2531 | ); |
2532 | - |
2533 | + rseq_after_asm_goto(); |
2534 | return 0; |
2535 | abort: |
2536 | + rseq_after_asm_goto(); |
2537 | RSEQ_INJECT_FAILED |
2538 | return -1; |
2539 | cmpfail: |
2540 | + rseq_after_asm_goto(); |
2541 | return 1; |
2542 | #ifdef RSEQ_COMPARE_TWICE |
2543 | error1: |
2544 | + rseq_after_asm_goto(); |
2545 | rseq_bug("cpu_id comparison failed"); |
2546 | error2: |
2547 | + rseq_after_asm_goto(); |
2548 | rseq_bug("expected value comparison failed"); |
2549 | #endif |
2550 | } |
2551 | |
2552 | static inline __attribute__((always_inline)) |
2553 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2554 | - off_t voffp, intptr_t *load, int cpu) |
2555 | + long voffp, intptr_t *load, int cpu) |
2556 | { |
2557 | RSEQ_INJECT_C(9) |
2558 | |
2559 | @@ -287,8 +291,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2560 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
2561 | : /* gcc asm goto does not allow outputs */ |
2562 | : [cpu_id] "r" (cpu), |
2563 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), |
2564 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2565 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), |
2566 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2567 | [v] "Qo" (*v), |
2568 | [expectnot] "r" (expectnot), |
2569 | [load] "Qo" (*load), |
2570 | @@ -300,16 +304,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2571 | , error1, error2 |
2572 | #endif |
2573 | ); |
2574 | + rseq_after_asm_goto(); |
2575 | return 0; |
2576 | abort: |
2577 | + rseq_after_asm_goto(); |
2578 | RSEQ_INJECT_FAILED |
2579 | return -1; |
2580 | cmpfail: |
2581 | + rseq_after_asm_goto(); |
2582 | return 1; |
2583 | #ifdef RSEQ_COMPARE_TWICE |
2584 | error1: |
2585 | + rseq_after_asm_goto(); |
2586 | rseq_bug("cpu_id comparison failed"); |
2587 | error2: |
2588 | + rseq_after_asm_goto(); |
2589 | rseq_bug("expected value comparison failed"); |
2590 | #endif |
2591 | } |
2592 | @@ -337,8 +346,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
2593 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
2594 | : /* gcc asm goto does not allow outputs */ |
2595 | : [cpu_id] "r" (cpu), |
2596 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), |
2597 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2598 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), |
2599 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2600 | [v] "Qo" (*v), |
2601 | [count] "r" (count) |
2602 | RSEQ_INJECT_INPUT |
2603 | @@ -348,12 +357,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
2604 | , error1 |
2605 | #endif |
2606 | ); |
2607 | + rseq_after_asm_goto(); |
2608 | return 0; |
2609 | abort: |
2610 | + rseq_after_asm_goto(); |
2611 | RSEQ_INJECT_FAILED |
2612 | return -1; |
2613 | #ifdef RSEQ_COMPARE_TWICE |
2614 | error1: |
2615 | + rseq_after_asm_goto(); |
2616 | rseq_bug("cpu_id comparison failed"); |
2617 | #endif |
2618 | } |
2619 | @@ -388,8 +400,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
2620 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
2621 | : /* gcc asm goto does not allow outputs */ |
2622 | : [cpu_id] "r" (cpu), |
2623 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), |
2624 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2625 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), |
2626 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2627 | [expect] "r" (expect), |
2628 | [v] "Qo" (*v), |
2629 | [newv] "r" (newv), |
2630 | @@ -402,17 +414,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
2631 | , error1, error2 |
2632 | #endif |
2633 | ); |
2634 | - |
2635 | + rseq_after_asm_goto(); |
2636 | return 0; |
2637 | abort: |
2638 | + rseq_after_asm_goto(); |
2639 | RSEQ_INJECT_FAILED |
2640 | return -1; |
2641 | cmpfail: |
2642 | + rseq_after_asm_goto(); |
2643 | return 1; |
2644 | #ifdef RSEQ_COMPARE_TWICE |
2645 | error1: |
2646 | + rseq_after_asm_goto(); |
2647 | rseq_bug("cpu_id comparison failed"); |
2648 | error2: |
2649 | + rseq_after_asm_goto(); |
2650 | rseq_bug("expected value comparison failed"); |
2651 | #endif |
2652 | } |
2653 | @@ -447,8 +463,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
2654 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
2655 | : /* gcc asm goto does not allow outputs */ |
2656 | : [cpu_id] "r" (cpu), |
2657 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), |
2658 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2659 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), |
2660 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2661 | [expect] "r" (expect), |
2662 | [v] "Qo" (*v), |
2663 | [newv] "r" (newv), |
2664 | @@ -461,17 +477,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
2665 | , error1, error2 |
2666 | #endif |
2667 | ); |
2668 | - |
2669 | + rseq_after_asm_goto(); |
2670 | return 0; |
2671 | abort: |
2672 | + rseq_after_asm_goto(); |
2673 | RSEQ_INJECT_FAILED |
2674 | return -1; |
2675 | cmpfail: |
2676 | + rseq_after_asm_goto(); |
2677 | return 1; |
2678 | #ifdef RSEQ_COMPARE_TWICE |
2679 | error1: |
2680 | + rseq_after_asm_goto(); |
2681 | rseq_bug("cpu_id comparison failed"); |
2682 | error2: |
2683 | + rseq_after_asm_goto(); |
2684 | rseq_bug("expected value comparison failed"); |
2685 | #endif |
2686 | } |
2687 | @@ -508,8 +528,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
2688 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
2689 | : /* gcc asm goto does not allow outputs */ |
2690 | : [cpu_id] "r" (cpu), |
2691 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), |
2692 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2693 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), |
2694 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2695 | [v] "Qo" (*v), |
2696 | [expect] "r" (expect), |
2697 | [v2] "Qo" (*v2), |
2698 | @@ -522,19 +542,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
2699 | , error1, error2, error3 |
2700 | #endif |
2701 | ); |
2702 | - |
2703 | + rseq_after_asm_goto(); |
2704 | return 0; |
2705 | abort: |
2706 | + rseq_after_asm_goto(); |
2707 | RSEQ_INJECT_FAILED |
2708 | return -1; |
2709 | cmpfail: |
2710 | + rseq_after_asm_goto(); |
2711 | return 1; |
2712 | #ifdef RSEQ_COMPARE_TWICE |
2713 | error1: |
2714 | + rseq_after_asm_goto(); |
2715 | rseq_bug("cpu_id comparison failed"); |
2716 | error2: |
2717 | + rseq_after_asm_goto(); |
2718 | rseq_bug("expected value comparison failed"); |
2719 | error3: |
2720 | + rseq_after_asm_goto(); |
2721 | rseq_bug("2nd expected value comparison failed"); |
2722 | #endif |
2723 | } |
2724 | @@ -569,8 +594,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
2725 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
2726 | : /* gcc asm goto does not allow outputs */ |
2727 | : [cpu_id] "r" (cpu), |
2728 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), |
2729 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2730 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), |
2731 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2732 | [expect] "r" (expect), |
2733 | [v] "Qo" (*v), |
2734 | [newv] "r" (newv), |
2735 | @@ -584,17 +609,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
2736 | , error1, error2 |
2737 | #endif |
2738 | ); |
2739 | - |
2740 | + rseq_after_asm_goto(); |
2741 | return 0; |
2742 | abort: |
2743 | + rseq_after_asm_goto(); |
2744 | RSEQ_INJECT_FAILED |
2745 | return -1; |
2746 | cmpfail: |
2747 | + rseq_after_asm_goto(); |
2748 | return 1; |
2749 | #ifdef RSEQ_COMPARE_TWICE |
2750 | error1: |
2751 | + rseq_after_asm_goto(); |
2752 | rseq_bug("cpu_id comparison failed"); |
2753 | error2: |
2754 | + rseq_after_asm_goto(); |
2755 | rseq_bug("expected value comparison failed"); |
2756 | #endif |
2757 | } |
2758 | @@ -629,8 +658,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
2759 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
2760 | : /* gcc asm goto does not allow outputs */ |
2761 | : [cpu_id] "r" (cpu), |
2762 | - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), |
2763 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2764 | + [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), |
2765 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2766 | [expect] "r" (expect), |
2767 | [v] "Qo" (*v), |
2768 | [newv] "r" (newv), |
2769 | @@ -644,17 +673,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
2770 | , error1, error2 |
2771 | #endif |
2772 | ); |
2773 | - |
2774 | + rseq_after_asm_goto(); |
2775 | return 0; |
2776 | abort: |
2777 | + rseq_after_asm_goto(); |
2778 | RSEQ_INJECT_FAILED |
2779 | return -1; |
2780 | cmpfail: |
2781 | + rseq_after_asm_goto(); |
2782 | return 1; |
2783 | #ifdef RSEQ_COMPARE_TWICE |
2784 | error1: |
2785 | + rseq_after_asm_goto(); |
2786 | rseq_bug("cpu_id comparison failed"); |
2787 | error2: |
2788 | + rseq_after_asm_goto(); |
2789 | rseq_bug("expected value comparison failed"); |
2790 | #endif |
2791 | } |
2792 | diff --git a/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h b/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h |
2793 | new file mode 100644 |
2794 | index 0000000000000..38c5846615714 |
2795 | --- /dev/null |
2796 | +++ b/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h |
2797 | @@ -0,0 +1,25 @@ |
2798 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
2799 | +/* |
2800 | + * rseq-generic-thread-pointer.h |
2801 | + * |
2802 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
2803 | + */ |
2804 | + |
2805 | +#ifndef _RSEQ_GENERIC_THREAD_POINTER |
2806 | +#define _RSEQ_GENERIC_THREAD_POINTER |
2807 | + |
2808 | +#ifdef __cplusplus |
2809 | +extern "C" { |
2810 | +#endif |
2811 | + |
2812 | +/* Use gcc builtin thread pointer. */ |
2813 | +static inline void *rseq_thread_pointer(void) |
2814 | +{ |
2815 | + return __builtin_thread_pointer(); |
2816 | +} |
2817 | + |
2818 | +#ifdef __cplusplus |
2819 | +} |
2820 | +#endif |
2821 | + |
2822 | +#endif |
2823 | diff --git a/tools/testing/selftests/rseq/rseq-mips.h b/tools/testing/selftests/rseq/rseq-mips.h |
2824 | index e989e7c14b097..878739fae2fde 100644 |
2825 | --- a/tools/testing/selftests/rseq/rseq-mips.h |
2826 | +++ b/tools/testing/selftests/rseq/rseq-mips.h |
2827 | @@ -154,14 +154,11 @@ do { \ |
2828 | teardown \ |
2829 | "b %l[" __rseq_str(cmpfail_label) "]\n\t" |
2830 | |
2831 | -#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") |
2832 | - |
2833 | static inline __attribute__((always_inline)) |
2834 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
2835 | { |
2836 | RSEQ_INJECT_C(9) |
2837 | |
2838 | - rseq_workaround_gcc_asm_size_guess(); |
2839 | __asm__ __volatile__ goto ( |
2840 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2841 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2842 | @@ -190,8 +187,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
2843 | "5:\n\t" |
2844 | : /* gcc asm goto does not allow outputs */ |
2845 | : [cpu_id] "r" (cpu), |
2846 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2847 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2848 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2849 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2850 | [v] "m" (*v), |
2851 | [expect] "r" (expect), |
2852 | [newv] "r" (newv) |
2853 | @@ -203,14 +200,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
2854 | , error1, error2 |
2855 | #endif |
2856 | ); |
2857 | - rseq_workaround_gcc_asm_size_guess(); |
2858 | return 0; |
2859 | abort: |
2860 | - rseq_workaround_gcc_asm_size_guess(); |
2861 | RSEQ_INJECT_FAILED |
2862 | return -1; |
2863 | cmpfail: |
2864 | - rseq_workaround_gcc_asm_size_guess(); |
2865 | return 1; |
2866 | #ifdef RSEQ_COMPARE_TWICE |
2867 | error1: |
2868 | @@ -222,11 +216,10 @@ error2: |
2869 | |
2870 | static inline __attribute__((always_inline)) |
2871 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2872 | - off_t voffp, intptr_t *load, int cpu) |
2873 | + long voffp, intptr_t *load, int cpu) |
2874 | { |
2875 | RSEQ_INJECT_C(9) |
2876 | |
2877 | - rseq_workaround_gcc_asm_size_guess(); |
2878 | __asm__ __volatile__ goto ( |
2879 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2880 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2881 | @@ -258,8 +251,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2882 | "5:\n\t" |
2883 | : /* gcc asm goto does not allow outputs */ |
2884 | : [cpu_id] "r" (cpu), |
2885 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2886 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2887 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2888 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2889 | /* final store input */ |
2890 | [v] "m" (*v), |
2891 | [expectnot] "r" (expectnot), |
2892 | @@ -273,14 +266,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
2893 | , error1, error2 |
2894 | #endif |
2895 | ); |
2896 | - rseq_workaround_gcc_asm_size_guess(); |
2897 | return 0; |
2898 | abort: |
2899 | - rseq_workaround_gcc_asm_size_guess(); |
2900 | RSEQ_INJECT_FAILED |
2901 | return -1; |
2902 | cmpfail: |
2903 | - rseq_workaround_gcc_asm_size_guess(); |
2904 | return 1; |
2905 | #ifdef RSEQ_COMPARE_TWICE |
2906 | error1: |
2907 | @@ -295,7 +285,6 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
2908 | { |
2909 | RSEQ_INJECT_C(9) |
2910 | |
2911 | - rseq_workaround_gcc_asm_size_guess(); |
2912 | __asm__ __volatile__ goto ( |
2913 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2914 | #ifdef RSEQ_COMPARE_TWICE |
2915 | @@ -319,8 +308,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
2916 | "5:\n\t" |
2917 | : /* gcc asm goto does not allow outputs */ |
2918 | : [cpu_id] "r" (cpu), |
2919 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2920 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2921 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2922 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2923 | [v] "m" (*v), |
2924 | [count] "Ir" (count) |
2925 | RSEQ_INJECT_INPUT |
2926 | @@ -331,10 +320,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
2927 | , error1 |
2928 | #endif |
2929 | ); |
2930 | - rseq_workaround_gcc_asm_size_guess(); |
2931 | return 0; |
2932 | abort: |
2933 | - rseq_workaround_gcc_asm_size_guess(); |
2934 | RSEQ_INJECT_FAILED |
2935 | return -1; |
2936 | #ifdef RSEQ_COMPARE_TWICE |
2937 | @@ -350,7 +337,6 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
2938 | { |
2939 | RSEQ_INJECT_C(9) |
2940 | |
2941 | - rseq_workaround_gcc_asm_size_guess(); |
2942 | __asm__ __volatile__ goto ( |
2943 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2944 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2945 | @@ -382,8 +368,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
2946 | "5:\n\t" |
2947 | : /* gcc asm goto does not allow outputs */ |
2948 | : [cpu_id] "r" (cpu), |
2949 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2950 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2951 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2952 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2953 | /* try store input */ |
2954 | [v2] "m" (*v2), |
2955 | [newv2] "r" (newv2), |
2956 | @@ -399,14 +385,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
2957 | , error1, error2 |
2958 | #endif |
2959 | ); |
2960 | - rseq_workaround_gcc_asm_size_guess(); |
2961 | return 0; |
2962 | abort: |
2963 | - rseq_workaround_gcc_asm_size_guess(); |
2964 | RSEQ_INJECT_FAILED |
2965 | return -1; |
2966 | cmpfail: |
2967 | - rseq_workaround_gcc_asm_size_guess(); |
2968 | return 1; |
2969 | #ifdef RSEQ_COMPARE_TWICE |
2970 | error1: |
2971 | @@ -423,7 +406,6 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
2972 | { |
2973 | RSEQ_INJECT_C(9) |
2974 | |
2975 | - rseq_workaround_gcc_asm_size_guess(); |
2976 | __asm__ __volatile__ goto ( |
2977 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
2978 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
2979 | @@ -456,8 +438,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
2980 | "5:\n\t" |
2981 | : /* gcc asm goto does not allow outputs */ |
2982 | : [cpu_id] "r" (cpu), |
2983 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
2984 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
2985 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
2986 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
2987 | /* try store input */ |
2988 | [v2] "m" (*v2), |
2989 | [newv2] "r" (newv2), |
2990 | @@ -473,14 +455,11 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
2991 | , error1, error2 |
2992 | #endif |
2993 | ); |
2994 | - rseq_workaround_gcc_asm_size_guess(); |
2995 | return 0; |
2996 | abort: |
2997 | - rseq_workaround_gcc_asm_size_guess(); |
2998 | RSEQ_INJECT_FAILED |
2999 | return -1; |
3000 | cmpfail: |
3001 | - rseq_workaround_gcc_asm_size_guess(); |
3002 | return 1; |
3003 | #ifdef RSEQ_COMPARE_TWICE |
3004 | error1: |
3005 | @@ -497,7 +476,6 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
3006 | { |
3007 | RSEQ_INJECT_C(9) |
3008 | |
3009 | - rseq_workaround_gcc_asm_size_guess(); |
3010 | __asm__ __volatile__ goto ( |
3011 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
3012 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
3013 | @@ -532,8 +510,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
3014 | "5:\n\t" |
3015 | : /* gcc asm goto does not allow outputs */ |
3016 | : [cpu_id] "r" (cpu), |
3017 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3018 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3019 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3020 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3021 | /* cmp2 input */ |
3022 | [v2] "m" (*v2), |
3023 | [expect2] "r" (expect2), |
3024 | @@ -549,14 +527,11 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
3025 | , error1, error2, error3 |
3026 | #endif |
3027 | ); |
3028 | - rseq_workaround_gcc_asm_size_guess(); |
3029 | return 0; |
3030 | abort: |
3031 | - rseq_workaround_gcc_asm_size_guess(); |
3032 | RSEQ_INJECT_FAILED |
3033 | return -1; |
3034 | cmpfail: |
3035 | - rseq_workaround_gcc_asm_size_guess(); |
3036 | return 1; |
3037 | #ifdef RSEQ_COMPARE_TWICE |
3038 | error1: |
3039 | @@ -577,7 +552,6 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
3040 | |
3041 | RSEQ_INJECT_C(9) |
3042 | |
3043 | - rseq_workaround_gcc_asm_size_guess(); |
3044 | __asm__ __volatile__ goto ( |
3045 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
3046 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
3047 | @@ -649,8 +623,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
3048 | "8:\n\t" |
3049 | : /* gcc asm goto does not allow outputs */ |
3050 | : [cpu_id] "r" (cpu), |
3051 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3052 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3053 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3054 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3055 | /* final store input */ |
3056 | [v] "m" (*v), |
3057 | [expect] "r" (expect), |
3058 | @@ -670,21 +644,16 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
3059 | , error1, error2 |
3060 | #endif |
3061 | ); |
3062 | - rseq_workaround_gcc_asm_size_guess(); |
3063 | return 0; |
3064 | abort: |
3065 | - rseq_workaround_gcc_asm_size_guess(); |
3066 | RSEQ_INJECT_FAILED |
3067 | return -1; |
3068 | cmpfail: |
3069 | - rseq_workaround_gcc_asm_size_guess(); |
3070 | return 1; |
3071 | #ifdef RSEQ_COMPARE_TWICE |
3072 | error1: |
3073 | - rseq_workaround_gcc_asm_size_guess(); |
3074 | rseq_bug("cpu_id comparison failed"); |
3075 | error2: |
3076 | - rseq_workaround_gcc_asm_size_guess(); |
3077 | rseq_bug("expected value comparison failed"); |
3078 | #endif |
3079 | } |
3080 | @@ -698,7 +667,6 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
3081 | |
3082 | RSEQ_INJECT_C(9) |
3083 | |
3084 | - rseq_workaround_gcc_asm_size_guess(); |
3085 | __asm__ __volatile__ goto ( |
3086 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
3087 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
3088 | @@ -771,8 +739,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
3089 | "8:\n\t" |
3090 | : /* gcc asm goto does not allow outputs */ |
3091 | : [cpu_id] "r" (cpu), |
3092 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3093 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3094 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3095 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3096 | /* final store input */ |
3097 | [v] "m" (*v), |
3098 | [expect] "r" (expect), |
3099 | @@ -792,21 +760,16 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
3100 | , error1, error2 |
3101 | #endif |
3102 | ); |
3103 | - rseq_workaround_gcc_asm_size_guess(); |
3104 | return 0; |
3105 | abort: |
3106 | - rseq_workaround_gcc_asm_size_guess(); |
3107 | RSEQ_INJECT_FAILED |
3108 | return -1; |
3109 | cmpfail: |
3110 | - rseq_workaround_gcc_asm_size_guess(); |
3111 | return 1; |
3112 | #ifdef RSEQ_COMPARE_TWICE |
3113 | error1: |
3114 | - rseq_workaround_gcc_asm_size_guess(); |
3115 | rseq_bug("cpu_id comparison failed"); |
3116 | error2: |
3117 | - rseq_workaround_gcc_asm_size_guess(); |
3118 | rseq_bug("expected value comparison failed"); |
3119 | #endif |
3120 | } |
3121 | diff --git a/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h b/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h |
3122 | new file mode 100644 |
3123 | index 0000000000000..263eee84fb760 |
3124 | --- /dev/null |
3125 | +++ b/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h |
3126 | @@ -0,0 +1,30 @@ |
3127 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
3128 | +/* |
3129 | + * rseq-ppc-thread-pointer.h |
3130 | + * |
3131 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
3132 | + */ |
3133 | + |
3134 | +#ifndef _RSEQ_PPC_THREAD_POINTER |
3135 | +#define _RSEQ_PPC_THREAD_POINTER |
3136 | + |
3137 | +#ifdef __cplusplus |
3138 | +extern "C" { |
3139 | +#endif |
3140 | + |
3141 | +static inline void *rseq_thread_pointer(void) |
3142 | +{ |
3143 | +#ifdef __powerpc64__ |
3144 | + register void *__result asm ("r13"); |
3145 | +#else |
3146 | + register void *__result asm ("r2"); |
3147 | +#endif |
3148 | + asm ("" : "=r" (__result)); |
3149 | + return __result; |
3150 | +} |
3151 | + |
3152 | +#ifdef __cplusplus |
3153 | +} |
3154 | +#endif |
3155 | + |
3156 | +#endif |
3157 | diff --git a/tools/testing/selftests/rseq/rseq-ppc.h b/tools/testing/selftests/rseq/rseq-ppc.h |
3158 | index 76be90196fe4f..bab8e0b9fb115 100644 |
3159 | --- a/tools/testing/selftests/rseq/rseq-ppc.h |
3160 | +++ b/tools/testing/selftests/rseq/rseq-ppc.h |
3161 | @@ -47,10 +47,13 @@ do { \ |
3162 | |
3163 | #ifdef __PPC64__ |
3164 | |
3165 | -#define STORE_WORD "std " |
3166 | -#define LOAD_WORD "ld " |
3167 | -#define LOADX_WORD "ldx " |
3168 | -#define CMP_WORD "cmpd " |
3169 | +#define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ |
3170 | +#define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ |
3171 | +#define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ |
3172 | +#define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ |
3173 | +#define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */ |
3174 | +#define RSEQ_CMP_LONG "cmpd " |
3175 | +#define RSEQ_CMP_LONG_INT "cmpdi " |
3176 | |
3177 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ |
3178 | start_ip, post_commit_offset, abort_ip) \ |
3179 | @@ -89,10 +92,13 @@ do { \ |
3180 | |
3181 | #else /* #ifdef __PPC64__ */ |
3182 | |
3183 | -#define STORE_WORD "stw " |
3184 | -#define LOAD_WORD "lwz " |
3185 | -#define LOADX_WORD "lwzx " |
3186 | -#define CMP_WORD "cmpw " |
3187 | +#define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ |
3188 | +#define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */ |
3189 | +#define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ |
3190 | +#define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */ |
3191 | +#define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */ |
3192 | +#define RSEQ_CMP_LONG "cmpw " |
3193 | +#define RSEQ_CMP_LONG_INT "cmpwi " |
3194 | |
3195 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ |
3196 | start_ip, post_commit_offset, abort_ip) \ |
3197 | @@ -125,7 +131,7 @@ do { \ |
3198 | RSEQ_INJECT_ASM(1) \ |
3199 | "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \ |
3200 | "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ |
3201 | - "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ |
3202 | + RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ |
3203 | __rseq_str(label) ":\n\t" |
3204 | |
3205 | #endif /* #ifdef __PPC64__ */ |
3206 | @@ -136,7 +142,7 @@ do { \ |
3207 | |
3208 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ |
3209 | RSEQ_INJECT_ASM(2) \ |
3210 | - "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ |
3211 | + RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ |
3212 | "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ |
3213 | "bne- cr7, " __rseq_str(label) "\n\t" |
3214 | |
3215 | @@ -153,25 +159,25 @@ do { \ |
3216 | * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7) |
3217 | */ |
3218 | #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ |
3219 | - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ |
3220 | - CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ |
3221 | + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
3222 | + RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ |
3223 | "bne- cr7, " __rseq_str(label) "\n\t" |
3224 | |
3225 | #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \ |
3226 | - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ |
3227 | - CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ |
3228 | + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
3229 | + RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ |
3230 | "beq- cr7, " __rseq_str(label) "\n\t" |
3231 | |
3232 | #define RSEQ_ASM_OP_STORE(value, var) \ |
3233 | - STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" |
3234 | + RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" |
3235 | |
3236 | /* Load @var to r17 */ |
3237 | #define RSEQ_ASM_OP_R_LOAD(var) \ |
3238 | - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" |
3239 | + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" |
3240 | |
3241 | /* Store r17 to @var */ |
3242 | #define RSEQ_ASM_OP_R_STORE(var) \ |
3243 | - STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" |
3244 | + RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" |
3245 | |
3246 | /* Add @count to r17 */ |
3247 | #define RSEQ_ASM_OP_R_ADD(count) \ |
3248 | @@ -179,11 +185,11 @@ do { \ |
3249 | |
3250 | /* Load (r17 + voffp) to r17 */ |
3251 | #define RSEQ_ASM_OP_R_LOADX(voffp) \ |
3252 | - LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" |
3253 | + RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" |
3254 | |
3255 | /* TODO: implement a faster memcpy. */ |
3256 | #define RSEQ_ASM_OP_R_MEMCPY() \ |
3257 | - "cmpdi %%r19, 0\n\t" \ |
3258 | + RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ |
3259 | "beq 333f\n\t" \ |
3260 | "addi %%r20, %%r20, -1\n\t" \ |
3261 | "addi %%r21, %%r21, -1\n\t" \ |
3262 | @@ -191,16 +197,16 @@ do { \ |
3263 | "lbzu %%r18, 1(%%r20)\n\t" \ |
3264 | "stbu %%r18, 1(%%r21)\n\t" \ |
3265 | "addi %%r19, %%r19, -1\n\t" \ |
3266 | - "cmpdi %%r19, 0\n\t" \ |
3267 | + RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ |
3268 | "bne 222b\n\t" \ |
3269 | "333:\n\t" \ |
3270 | |
3271 | #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ |
3272 | - STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ |
3273 | + RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
3274 | __rseq_str(post_commit_label) ":\n\t" |
3275 | |
3276 | #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ |
3277 | - STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ |
3278 | + RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ |
3279 | __rseq_str(post_commit_label) ":\n\t" |
3280 | |
3281 | static inline __attribute__((always_inline)) |
3282 | @@ -235,8 +241,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
3283 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
3284 | : /* gcc asm goto does not allow outputs */ |
3285 | : [cpu_id] "r" (cpu), |
3286 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3287 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3288 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3289 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3290 | [v] "m" (*v), |
3291 | [expect] "r" (expect), |
3292 | [newv] "r" (newv) |
3293 | @@ -248,23 +254,28 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
3294 | , error1, error2 |
3295 | #endif |
3296 | ); |
3297 | + rseq_after_asm_goto(); |
3298 | return 0; |
3299 | abort: |
3300 | + rseq_after_asm_goto(); |
3301 | RSEQ_INJECT_FAILED |
3302 | return -1; |
3303 | cmpfail: |
3304 | + rseq_after_asm_goto(); |
3305 | return 1; |
3306 | #ifdef RSEQ_COMPARE_TWICE |
3307 | error1: |
3308 | + rseq_after_asm_goto(); |
3309 | rseq_bug("cpu_id comparison failed"); |
3310 | error2: |
3311 | + rseq_after_asm_goto(); |
3312 | rseq_bug("expected value comparison failed"); |
3313 | #endif |
3314 | } |
3315 | |
3316 | static inline __attribute__((always_inline)) |
3317 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3318 | - off_t voffp, intptr_t *load, int cpu) |
3319 | + long voffp, intptr_t *load, int cpu) |
3320 | { |
3321 | RSEQ_INJECT_C(9) |
3322 | |
3323 | @@ -301,8 +312,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3324 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
3325 | : /* gcc asm goto does not allow outputs */ |
3326 | : [cpu_id] "r" (cpu), |
3327 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3328 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3329 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3330 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3331 | /* final store input */ |
3332 | [v] "m" (*v), |
3333 | [expectnot] "r" (expectnot), |
3334 | @@ -316,16 +327,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3335 | , error1, error2 |
3336 | #endif |
3337 | ); |
3338 | + rseq_after_asm_goto(); |
3339 | return 0; |
3340 | abort: |
3341 | + rseq_after_asm_goto(); |
3342 | RSEQ_INJECT_FAILED |
3343 | return -1; |
3344 | cmpfail: |
3345 | + rseq_after_asm_goto(); |
3346 | return 1; |
3347 | #ifdef RSEQ_COMPARE_TWICE |
3348 | error1: |
3349 | + rseq_after_asm_goto(); |
3350 | rseq_bug("cpu_id comparison failed"); |
3351 | error2: |
3352 | + rseq_after_asm_goto(); |
3353 | rseq_bug("expected value comparison failed"); |
3354 | #endif |
3355 | } |
3356 | @@ -359,8 +375,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
3357 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
3358 | : /* gcc asm goto does not allow outputs */ |
3359 | : [cpu_id] "r" (cpu), |
3360 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3361 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3362 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3363 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3364 | /* final store input */ |
3365 | [v] "m" (*v), |
3366 | [count] "r" (count) |
3367 | @@ -372,12 +388,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
3368 | , error1 |
3369 | #endif |
3370 | ); |
3371 | + rseq_after_asm_goto(); |
3372 | return 0; |
3373 | abort: |
3374 | + rseq_after_asm_goto(); |
3375 | RSEQ_INJECT_FAILED |
3376 | return -1; |
3377 | #ifdef RSEQ_COMPARE_TWICE |
3378 | error1: |
3379 | + rseq_after_asm_goto(); |
3380 | rseq_bug("cpu_id comparison failed"); |
3381 | #endif |
3382 | } |
3383 | @@ -419,8 +438,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
3384 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
3385 | : /* gcc asm goto does not allow outputs */ |
3386 | : [cpu_id] "r" (cpu), |
3387 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3388 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3389 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3390 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3391 | /* try store input */ |
3392 | [v2] "m" (*v2), |
3393 | [newv2] "r" (newv2), |
3394 | @@ -436,16 +455,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
3395 | , error1, error2 |
3396 | #endif |
3397 | ); |
3398 | + rseq_after_asm_goto(); |
3399 | return 0; |
3400 | abort: |
3401 | + rseq_after_asm_goto(); |
3402 | RSEQ_INJECT_FAILED |
3403 | return -1; |
3404 | cmpfail: |
3405 | + rseq_after_asm_goto(); |
3406 | return 1; |
3407 | #ifdef RSEQ_COMPARE_TWICE |
3408 | error1: |
3409 | + rseq_after_asm_goto(); |
3410 | rseq_bug("cpu_id comparison failed"); |
3411 | error2: |
3412 | + rseq_after_asm_goto(); |
3413 | rseq_bug("expected value comparison failed"); |
3414 | #endif |
3415 | } |
3416 | @@ -489,8 +513,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
3417 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
3418 | : /* gcc asm goto does not allow outputs */ |
3419 | : [cpu_id] "r" (cpu), |
3420 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3421 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3422 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3423 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3424 | /* try store input */ |
3425 | [v2] "m" (*v2), |
3426 | [newv2] "r" (newv2), |
3427 | @@ -506,16 +530,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
3428 | , error1, error2 |
3429 | #endif |
3430 | ); |
3431 | + rseq_after_asm_goto(); |
3432 | return 0; |
3433 | abort: |
3434 | + rseq_after_asm_goto(); |
3435 | RSEQ_INJECT_FAILED |
3436 | return -1; |
3437 | cmpfail: |
3438 | + rseq_after_asm_goto(); |
3439 | return 1; |
3440 | #ifdef RSEQ_COMPARE_TWICE |
3441 | error1: |
3442 | + rseq_after_asm_goto(); |
3443 | rseq_bug("cpu_id comparison failed"); |
3444 | error2: |
3445 | + rseq_after_asm_goto(); |
3446 | rseq_bug("expected value comparison failed"); |
3447 | #endif |
3448 | } |
3449 | @@ -560,8 +589,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
3450 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
3451 | : /* gcc asm goto does not allow outputs */ |
3452 | : [cpu_id] "r" (cpu), |
3453 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3454 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3455 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3456 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3457 | /* cmp2 input */ |
3458 | [v2] "m" (*v2), |
3459 | [expect2] "r" (expect2), |
3460 | @@ -577,18 +606,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
3461 | , error1, error2, error3 |
3462 | #endif |
3463 | ); |
3464 | + rseq_after_asm_goto(); |
3465 | return 0; |
3466 | abort: |
3467 | + rseq_after_asm_goto(); |
3468 | RSEQ_INJECT_FAILED |
3469 | return -1; |
3470 | cmpfail: |
3471 | + rseq_after_asm_goto(); |
3472 | return 1; |
3473 | #ifdef RSEQ_COMPARE_TWICE |
3474 | error1: |
3475 | + rseq_after_asm_goto(); |
3476 | rseq_bug("cpu_id comparison failed"); |
3477 | error2: |
3478 | + rseq_after_asm_goto(); |
3479 | rseq_bug("1st expected value comparison failed"); |
3480 | error3: |
3481 | + rseq_after_asm_goto(); |
3482 | rseq_bug("2nd expected value comparison failed"); |
3483 | #endif |
3484 | } |
3485 | @@ -635,8 +670,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
3486 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
3487 | : /* gcc asm goto does not allow outputs */ |
3488 | : [cpu_id] "r" (cpu), |
3489 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3490 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3491 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3492 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3493 | /* final store input */ |
3494 | [v] "m" (*v), |
3495 | [expect] "r" (expect), |
3496 | @@ -653,16 +688,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
3497 | , error1, error2 |
3498 | #endif |
3499 | ); |
3500 | + rseq_after_asm_goto(); |
3501 | return 0; |
3502 | abort: |
3503 | + rseq_after_asm_goto(); |
3504 | RSEQ_INJECT_FAILED |
3505 | return -1; |
3506 | cmpfail: |
3507 | + rseq_after_asm_goto(); |
3508 | return 1; |
3509 | #ifdef RSEQ_COMPARE_TWICE |
3510 | error1: |
3511 | + rseq_after_asm_goto(); |
3512 | rseq_bug("cpu_id comparison failed"); |
3513 | error2: |
3514 | + rseq_after_asm_goto(); |
3515 | rseq_bug("expected value comparison failed"); |
3516 | #endif |
3517 | } |
3518 | @@ -711,8 +751,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
3519 | RSEQ_ASM_DEFINE_ABORT(4, abort) |
3520 | : /* gcc asm goto does not allow outputs */ |
3521 | : [cpu_id] "r" (cpu), |
3522 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3523 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3524 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3525 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3526 | /* final store input */ |
3527 | [v] "m" (*v), |
3528 | [expect] "r" (expect), |
3529 | @@ -729,23 +769,23 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
3530 | , error1, error2 |
3531 | #endif |
3532 | ); |
3533 | + rseq_after_asm_goto(); |
3534 | return 0; |
3535 | abort: |
3536 | + rseq_after_asm_goto(); |
3537 | RSEQ_INJECT_FAILED |
3538 | return -1; |
3539 | cmpfail: |
3540 | + rseq_after_asm_goto(); |
3541 | return 1; |
3542 | #ifdef RSEQ_COMPARE_TWICE |
3543 | error1: |
3544 | + rseq_after_asm_goto(); |
3545 | rseq_bug("cpu_id comparison failed"); |
3546 | error2: |
3547 | + rseq_after_asm_goto(); |
3548 | rseq_bug("expected value comparison failed"); |
3549 | #endif |
3550 | } |
3551 | |
3552 | -#undef STORE_WORD |
3553 | -#undef LOAD_WORD |
3554 | -#undef LOADX_WORD |
3555 | -#undef CMP_WORD |
3556 | - |
3557 | #endif /* !RSEQ_SKIP_FASTPATH */ |
3558 | diff --git a/tools/testing/selftests/rseq/rseq-s390.h b/tools/testing/selftests/rseq/rseq-s390.h |
3559 | index 8ef94ad1cbb45..4e6dc5f0cb429 100644 |
3560 | --- a/tools/testing/selftests/rseq/rseq-s390.h |
3561 | +++ b/tools/testing/selftests/rseq/rseq-s390.h |
3562 | @@ -165,8 +165,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
3563 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
3564 | : /* gcc asm goto does not allow outputs */ |
3565 | : [cpu_id] "r" (cpu), |
3566 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3567 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3568 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3569 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3570 | [v] "m" (*v), |
3571 | [expect] "r" (expect), |
3572 | [newv] "r" (newv) |
3573 | @@ -178,16 +178,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
3574 | , error1, error2 |
3575 | #endif |
3576 | ); |
3577 | + rseq_after_asm_goto(); |
3578 | return 0; |
3579 | abort: |
3580 | + rseq_after_asm_goto(); |
3581 | RSEQ_INJECT_FAILED |
3582 | return -1; |
3583 | cmpfail: |
3584 | + rseq_after_asm_goto(); |
3585 | return 1; |
3586 | #ifdef RSEQ_COMPARE_TWICE |
3587 | error1: |
3588 | + rseq_after_asm_goto(); |
3589 | rseq_bug("cpu_id comparison failed"); |
3590 | error2: |
3591 | + rseq_after_asm_goto(); |
3592 | rseq_bug("expected value comparison failed"); |
3593 | #endif |
3594 | } |
3595 | @@ -198,7 +203,7 @@ error2: |
3596 | */ |
3597 | static inline __attribute__((always_inline)) |
3598 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3599 | - off_t voffp, intptr_t *load, int cpu) |
3600 | + long voffp, intptr_t *load, int cpu) |
3601 | { |
3602 | RSEQ_INJECT_C(9) |
3603 | |
3604 | @@ -233,8 +238,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3605 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
3606 | : /* gcc asm goto does not allow outputs */ |
3607 | : [cpu_id] "r" (cpu), |
3608 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3609 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3610 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3611 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3612 | /* final store input */ |
3613 | [v] "m" (*v), |
3614 | [expectnot] "r" (expectnot), |
3615 | @@ -248,16 +253,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3616 | , error1, error2 |
3617 | #endif |
3618 | ); |
3619 | + rseq_after_asm_goto(); |
3620 | return 0; |
3621 | abort: |
3622 | + rseq_after_asm_goto(); |
3623 | RSEQ_INJECT_FAILED |
3624 | return -1; |
3625 | cmpfail: |
3626 | + rseq_after_asm_goto(); |
3627 | return 1; |
3628 | #ifdef RSEQ_COMPARE_TWICE |
3629 | error1: |
3630 | + rseq_after_asm_goto(); |
3631 | rseq_bug("cpu_id comparison failed"); |
3632 | error2: |
3633 | + rseq_after_asm_goto(); |
3634 | rseq_bug("expected value comparison failed"); |
3635 | #endif |
3636 | } |
3637 | @@ -288,8 +298,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
3638 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
3639 | : /* gcc asm goto does not allow outputs */ |
3640 | : [cpu_id] "r" (cpu), |
3641 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3642 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3643 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3644 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3645 | /* final store input */ |
3646 | [v] "m" (*v), |
3647 | [count] "r" (count) |
3648 | @@ -301,12 +311,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
3649 | , error1 |
3650 | #endif |
3651 | ); |
3652 | + rseq_after_asm_goto(); |
3653 | return 0; |
3654 | abort: |
3655 | + rseq_after_asm_goto(); |
3656 | RSEQ_INJECT_FAILED |
3657 | return -1; |
3658 | #ifdef RSEQ_COMPARE_TWICE |
3659 | error1: |
3660 | + rseq_after_asm_goto(); |
3661 | rseq_bug("cpu_id comparison failed"); |
3662 | #endif |
3663 | } |
3664 | @@ -347,8 +360,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
3665 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
3666 | : /* gcc asm goto does not allow outputs */ |
3667 | : [cpu_id] "r" (cpu), |
3668 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3669 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3670 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3671 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3672 | /* try store input */ |
3673 | [v2] "m" (*v2), |
3674 | [newv2] "r" (newv2), |
3675 | @@ -364,16 +377,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
3676 | , error1, error2 |
3677 | #endif |
3678 | ); |
3679 | + rseq_after_asm_goto(); |
3680 | return 0; |
3681 | abort: |
3682 | + rseq_after_asm_goto(); |
3683 | RSEQ_INJECT_FAILED |
3684 | return -1; |
3685 | cmpfail: |
3686 | + rseq_after_asm_goto(); |
3687 | return 1; |
3688 | #ifdef RSEQ_COMPARE_TWICE |
3689 | error1: |
3690 | + rseq_after_asm_goto(); |
3691 | rseq_bug("cpu_id comparison failed"); |
3692 | error2: |
3693 | + rseq_after_asm_goto(); |
3694 | rseq_bug("expected value comparison failed"); |
3695 | #endif |
3696 | } |
3697 | @@ -426,8 +444,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
3698 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
3699 | : /* gcc asm goto does not allow outputs */ |
3700 | : [cpu_id] "r" (cpu), |
3701 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3702 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3703 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3704 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3705 | /* cmp2 input */ |
3706 | [v2] "m" (*v2), |
3707 | [expect2] "r" (expect2), |
3708 | @@ -443,18 +461,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
3709 | , error1, error2, error3 |
3710 | #endif |
3711 | ); |
3712 | + rseq_after_asm_goto(); |
3713 | return 0; |
3714 | abort: |
3715 | + rseq_after_asm_goto(); |
3716 | RSEQ_INJECT_FAILED |
3717 | return -1; |
3718 | cmpfail: |
3719 | + rseq_after_asm_goto(); |
3720 | return 1; |
3721 | #ifdef RSEQ_COMPARE_TWICE |
3722 | error1: |
3723 | + rseq_after_asm_goto(); |
3724 | rseq_bug("cpu_id comparison failed"); |
3725 | error2: |
3726 | + rseq_after_asm_goto(); |
3727 | rseq_bug("1st expected value comparison failed"); |
3728 | error3: |
3729 | + rseq_after_asm_goto(); |
3730 | rseq_bug("2nd expected value comparison failed"); |
3731 | #endif |
3732 | } |
3733 | @@ -534,8 +558,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
3734 | #endif |
3735 | : /* gcc asm goto does not allow outputs */ |
3736 | : [cpu_id] "r" (cpu), |
3737 | - [current_cpu_id] "m" (__rseq_abi.cpu_id), |
3738 | - [rseq_cs] "m" (__rseq_abi.rseq_cs), |
3739 | + [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
3740 | + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), |
3741 | /* final store input */ |
3742 | [v] "m" (*v), |
3743 | [expect] "r" (expect), |
3744 | @@ -555,16 +579,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
3745 | , error1, error2 |
3746 | #endif |
3747 | ); |
3748 | + rseq_after_asm_goto(); |
3749 | return 0; |
3750 | abort: |
3751 | + rseq_after_asm_goto(); |
3752 | RSEQ_INJECT_FAILED |
3753 | return -1; |
3754 | cmpfail: |
3755 | + rseq_after_asm_goto(); |
3756 | return 1; |
3757 | #ifdef RSEQ_COMPARE_TWICE |
3758 | error1: |
3759 | + rseq_after_asm_goto(); |
3760 | rseq_bug("cpu_id comparison failed"); |
3761 | error2: |
3762 | + rseq_after_asm_goto(); |
3763 | rseq_bug("expected value comparison failed"); |
3764 | #endif |
3765 | } |
3766 | diff --git a/tools/testing/selftests/rseq/rseq-skip.h b/tools/testing/selftests/rseq/rseq-skip.h |
3767 | index 72750b5905a96..7b53dac1fcdd9 100644 |
3768 | --- a/tools/testing/selftests/rseq/rseq-skip.h |
3769 | +++ b/tools/testing/selftests/rseq/rseq-skip.h |
3770 | @@ -13,7 +13,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
3771 | |
3772 | static inline __attribute__((always_inline)) |
3773 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3774 | - off_t voffp, intptr_t *load, int cpu) |
3775 | + long voffp, intptr_t *load, int cpu) |
3776 | { |
3777 | return -1; |
3778 | } |
3779 | diff --git a/tools/testing/selftests/rseq/rseq-thread-pointer.h b/tools/testing/selftests/rseq/rseq-thread-pointer.h |
3780 | new file mode 100644 |
3781 | index 0000000000000..977c25d758b2a |
3782 | --- /dev/null |
3783 | +++ b/tools/testing/selftests/rseq/rseq-thread-pointer.h |
3784 | @@ -0,0 +1,19 @@ |
3785 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
3786 | +/* |
3787 | + * rseq-thread-pointer.h |
3788 | + * |
3789 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
3790 | + */ |
3791 | + |
3792 | +#ifndef _RSEQ_THREAD_POINTER |
3793 | +#define _RSEQ_THREAD_POINTER |
3794 | + |
3795 | +#if defined(__x86_64__) || defined(__i386__) |
3796 | +#include "rseq-x86-thread-pointer.h" |
3797 | +#elif defined(__PPC__) |
3798 | +#include "rseq-ppc-thread-pointer.h" |
3799 | +#else |
3800 | +#include "rseq-generic-thread-pointer.h" |
3801 | +#endif |
3802 | + |
3803 | +#endif |
3804 | diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h |
3805 | new file mode 100644 |
3806 | index 0000000000000..d3133587d9968 |
3807 | --- /dev/null |
3808 | +++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h |
3809 | @@ -0,0 +1,40 @@ |
3810 | +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
3811 | +/* |
3812 | + * rseq-x86-thread-pointer.h |
3813 | + * |
3814 | + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
3815 | + */ |
3816 | + |
3817 | +#ifndef _RSEQ_X86_THREAD_POINTER |
3818 | +#define _RSEQ_X86_THREAD_POINTER |
3819 | + |
3820 | +#include <features.h> |
3821 | + |
3822 | +#ifdef __cplusplus |
3823 | +extern "C" { |
3824 | +#endif |
3825 | + |
3826 | +#if __GNUC_PREREQ (11, 1) |
3827 | +static inline void *rseq_thread_pointer(void) |
3828 | +{ |
3829 | + return __builtin_thread_pointer(); |
3830 | +} |
3831 | +#else |
3832 | +static inline void *rseq_thread_pointer(void) |
3833 | +{ |
3834 | + void *__result; |
3835 | + |
3836 | +# ifdef __x86_64__ |
3837 | + __asm__ ("mov %%fs:0, %0" : "=r" (__result)); |
3838 | +# else |
3839 | + __asm__ ("mov %%gs:0, %0" : "=r" (__result)); |
3840 | +# endif |
3841 | + return __result; |
3842 | +} |
3843 | +#endif /* !GCC 11 */ |
3844 | + |
3845 | +#ifdef __cplusplus |
3846 | +} |
3847 | +#endif |
3848 | + |
3849 | +#endif |
3850 | diff --git a/tools/testing/selftests/rseq/rseq-x86.h b/tools/testing/selftests/rseq/rseq-x86.h |
3851 | index b2da6004fe307..bd01dc41ca130 100644 |
3852 | --- a/tools/testing/selftests/rseq/rseq-x86.h |
3853 | +++ b/tools/testing/selftests/rseq/rseq-x86.h |
3854 | @@ -28,6 +28,8 @@ |
3855 | |
3856 | #ifdef __x86_64__ |
3857 | |
3858 | +#define RSEQ_ASM_TP_SEGMENT %%fs |
3859 | + |
3860 | #define rseq_smp_mb() \ |
3861 | __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") |
3862 | #define rseq_smp_rmb() rseq_barrier() |
3863 | @@ -123,14 +125,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
3864 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
3865 | #endif |
3866 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
3867 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
3868 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
3869 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
3870 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
3871 | RSEQ_INJECT_ASM(3) |
3872 | "cmpq %[v], %[expect]\n\t" |
3873 | "jnz %l[cmpfail]\n\t" |
3874 | RSEQ_INJECT_ASM(4) |
3875 | #ifdef RSEQ_COMPARE_TWICE |
3876 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
3877 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
3878 | "cmpq %[v], %[expect]\n\t" |
3879 | "jnz %l[error2]\n\t" |
3880 | #endif |
3881 | @@ -141,7 +143,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
3882 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
3883 | : /* gcc asm goto does not allow outputs */ |
3884 | : [cpu_id] "r" (cpu), |
3885 | - [rseq_abi] "r" (&__rseq_abi), |
3886 | + [rseq_offset] "r" (rseq_offset), |
3887 | [v] "m" (*v), |
3888 | [expect] "r" (expect), |
3889 | [newv] "r" (newv) |
3890 | @@ -152,16 +154,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
3891 | , error1, error2 |
3892 | #endif |
3893 | ); |
3894 | + rseq_after_asm_goto(); |
3895 | return 0; |
3896 | abort: |
3897 | + rseq_after_asm_goto(); |
3898 | RSEQ_INJECT_FAILED |
3899 | return -1; |
3900 | cmpfail: |
3901 | + rseq_after_asm_goto(); |
3902 | return 1; |
3903 | #ifdef RSEQ_COMPARE_TWICE |
3904 | error1: |
3905 | + rseq_after_asm_goto(); |
3906 | rseq_bug("cpu_id comparison failed"); |
3907 | error2: |
3908 | + rseq_after_asm_goto(); |
3909 | rseq_bug("expected value comparison failed"); |
3910 | #endif |
3911 | } |
3912 | @@ -172,7 +179,7 @@ error2: |
3913 | */ |
3914 | static inline __attribute__((always_inline)) |
3915 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3916 | - off_t voffp, intptr_t *load, int cpu) |
3917 | + long voffp, intptr_t *load, int cpu) |
3918 | { |
3919 | RSEQ_INJECT_C(9) |
3920 | |
3921 | @@ -184,15 +191,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3922 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
3923 | #endif |
3924 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
3925 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
3926 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
3927 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
3928 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
3929 | RSEQ_INJECT_ASM(3) |
3930 | "movq %[v], %%rbx\n\t" |
3931 | "cmpq %%rbx, %[expectnot]\n\t" |
3932 | "je %l[cmpfail]\n\t" |
3933 | RSEQ_INJECT_ASM(4) |
3934 | #ifdef RSEQ_COMPARE_TWICE |
3935 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
3936 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
3937 | "movq %[v], %%rbx\n\t" |
3938 | "cmpq %%rbx, %[expectnot]\n\t" |
3939 | "je %l[error2]\n\t" |
3940 | @@ -207,7 +214,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3941 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
3942 | : /* gcc asm goto does not allow outputs */ |
3943 | : [cpu_id] "r" (cpu), |
3944 | - [rseq_abi] "r" (&__rseq_abi), |
3945 | + [rseq_offset] "r" (rseq_offset), |
3946 | /* final store input */ |
3947 | [v] "m" (*v), |
3948 | [expectnot] "r" (expectnot), |
3949 | @@ -220,16 +227,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
3950 | , error1, error2 |
3951 | #endif |
3952 | ); |
3953 | + rseq_after_asm_goto(); |
3954 | return 0; |
3955 | abort: |
3956 | + rseq_after_asm_goto(); |
3957 | RSEQ_INJECT_FAILED |
3958 | return -1; |
3959 | cmpfail: |
3960 | + rseq_after_asm_goto(); |
3961 | return 1; |
3962 | #ifdef RSEQ_COMPARE_TWICE |
3963 | error1: |
3964 | + rseq_after_asm_goto(); |
3965 | rseq_bug("cpu_id comparison failed"); |
3966 | error2: |
3967 | + rseq_after_asm_goto(); |
3968 | rseq_bug("expected value comparison failed"); |
3969 | #endif |
3970 | } |
3971 | @@ -245,11 +257,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
3972 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
3973 | #endif |
3974 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
3975 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
3976 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
3977 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
3978 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
3979 | RSEQ_INJECT_ASM(3) |
3980 | #ifdef RSEQ_COMPARE_TWICE |
3981 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
3982 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
3983 | #endif |
3984 | /* final store */ |
3985 | "addq %[count], %[v]\n\t" |
3986 | @@ -258,7 +270,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
3987 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
3988 | : /* gcc asm goto does not allow outputs */ |
3989 | : [cpu_id] "r" (cpu), |
3990 | - [rseq_abi] "r" (&__rseq_abi), |
3991 | + [rseq_offset] "r" (rseq_offset), |
3992 | /* final store input */ |
3993 | [v] "m" (*v), |
3994 | [count] "er" (count) |
3995 | @@ -267,6 +279,66 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
3996 | : abort |
3997 | #ifdef RSEQ_COMPARE_TWICE |
3998 | , error1 |
3999 | +#endif |
4000 | + ); |
4001 | + rseq_after_asm_goto(); |
4002 | + return 0; |
4003 | +abort: |
4004 | + rseq_after_asm_goto(); |
4005 | + RSEQ_INJECT_FAILED |
4006 | + return -1; |
4007 | +#ifdef RSEQ_COMPARE_TWICE |
4008 | +error1: |
4009 | + rseq_after_asm_goto(); |
4010 | + rseq_bug("cpu_id comparison failed"); |
4011 | +#endif |
4012 | +} |
4013 | + |
4014 | +#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV |
4015 | + |
4016 | +/* |
4017 | + * pval = *(ptr+off) |
4018 | + * *pval += inc; |
4019 | + */ |
4020 | +static inline __attribute__((always_inline)) |
4021 | +int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu) |
4022 | +{ |
4023 | + RSEQ_INJECT_C(9) |
4024 | + |
4025 | + __asm__ __volatile__ goto ( |
4026 | + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ |
4027 | +#ifdef RSEQ_COMPARE_TWICE |
4028 | + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
4029 | +#endif |
4030 | + /* Start rseq by storing table entry pointer into rseq_cs. */ |
4031 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4032 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4033 | + RSEQ_INJECT_ASM(3) |
4034 | +#ifdef RSEQ_COMPARE_TWICE |
4035 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4036 | +#endif |
4037 | + /* get p+v */ |
4038 | + "movq %[ptr], %%rbx\n\t" |
4039 | + "addq %[off], %%rbx\n\t" |
4040 | + /* get pv */ |
4041 | + "movq (%%rbx), %%rcx\n\t" |
4042 | + /* *pv += inc */ |
4043 | + "addq %[inc], (%%rcx)\n\t" |
4044 | + "2:\n\t" |
4045 | + RSEQ_INJECT_ASM(4) |
4046 | + RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4047 | + : /* gcc asm goto does not allow outputs */ |
4048 | + : [cpu_id] "r" (cpu), |
4049 | + [rseq_offset] "r" (rseq_offset), |
4050 | + /* final store input */ |
4051 | + [ptr] "m" (*ptr), |
4052 | + [off] "er" (off), |
4053 | + [inc] "er" (inc) |
4054 | + : "memory", "cc", "rax", "rbx", "rcx" |
4055 | + RSEQ_INJECT_CLOBBER |
4056 | + : abort |
4057 | +#ifdef RSEQ_COMPARE_TWICE |
4058 | + , error1 |
4059 | #endif |
4060 | ); |
4061 | return 0; |
4062 | @@ -294,14 +366,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
4063 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
4064 | #endif |
4065 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4066 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4067 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4068 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4069 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4070 | RSEQ_INJECT_ASM(3) |
4071 | "cmpq %[v], %[expect]\n\t" |
4072 | "jnz %l[cmpfail]\n\t" |
4073 | RSEQ_INJECT_ASM(4) |
4074 | #ifdef RSEQ_COMPARE_TWICE |
4075 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
4076 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4077 | "cmpq %[v], %[expect]\n\t" |
4078 | "jnz %l[error2]\n\t" |
4079 | #endif |
4080 | @@ -315,7 +387,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
4081 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4082 | : /* gcc asm goto does not allow outputs */ |
4083 | : [cpu_id] "r" (cpu), |
4084 | - [rseq_abi] "r" (&__rseq_abi), |
4085 | + [rseq_offset] "r" (rseq_offset), |
4086 | /* try store input */ |
4087 | [v2] "m" (*v2), |
4088 | [newv2] "r" (newv2), |
4089 | @@ -330,16 +402,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
4090 | , error1, error2 |
4091 | #endif |
4092 | ); |
4093 | + rseq_after_asm_goto(); |
4094 | return 0; |
4095 | abort: |
4096 | + rseq_after_asm_goto(); |
4097 | RSEQ_INJECT_FAILED |
4098 | return -1; |
4099 | cmpfail: |
4100 | + rseq_after_asm_goto(); |
4101 | return 1; |
4102 | #ifdef RSEQ_COMPARE_TWICE |
4103 | error1: |
4104 | + rseq_after_asm_goto(); |
4105 | rseq_bug("cpu_id comparison failed"); |
4106 | error2: |
4107 | + rseq_after_asm_goto(); |
4108 | rseq_bug("expected value comparison failed"); |
4109 | #endif |
4110 | } |
4111 | @@ -369,8 +446,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
4112 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) |
4113 | #endif |
4114 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4115 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4116 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4117 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4118 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4119 | RSEQ_INJECT_ASM(3) |
4120 | "cmpq %[v], %[expect]\n\t" |
4121 | "jnz %l[cmpfail]\n\t" |
4122 | @@ -379,7 +456,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
4123 | "jnz %l[cmpfail]\n\t" |
4124 | RSEQ_INJECT_ASM(5) |
4125 | #ifdef RSEQ_COMPARE_TWICE |
4126 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
4127 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4128 | "cmpq %[v], %[expect]\n\t" |
4129 | "jnz %l[error2]\n\t" |
4130 | "cmpq %[v2], %[expect2]\n\t" |
4131 | @@ -392,7 +469,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
4132 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4133 | : /* gcc asm goto does not allow outputs */ |
4134 | : [cpu_id] "r" (cpu), |
4135 | - [rseq_abi] "r" (&__rseq_abi), |
4136 | + [rseq_offset] "r" (rseq_offset), |
4137 | /* cmp2 input */ |
4138 | [v2] "m" (*v2), |
4139 | [expect2] "r" (expect2), |
4140 | @@ -407,18 +484,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
4141 | , error1, error2, error3 |
4142 | #endif |
4143 | ); |
4144 | + rseq_after_asm_goto(); |
4145 | return 0; |
4146 | abort: |
4147 | + rseq_after_asm_goto(); |
4148 | RSEQ_INJECT_FAILED |
4149 | return -1; |
4150 | cmpfail: |
4151 | + rseq_after_asm_goto(); |
4152 | return 1; |
4153 | #ifdef RSEQ_COMPARE_TWICE |
4154 | error1: |
4155 | + rseq_after_asm_goto(); |
4156 | rseq_bug("cpu_id comparison failed"); |
4157 | error2: |
4158 | + rseq_after_asm_goto(); |
4159 | rseq_bug("1st expected value comparison failed"); |
4160 | error3: |
4161 | + rseq_after_asm_goto(); |
4162 | rseq_bug("2nd expected value comparison failed"); |
4163 | #endif |
4164 | } |
4165 | @@ -443,14 +526,14 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
4166 | "movq %[dst], %[rseq_scratch1]\n\t" |
4167 | "movq %[len], %[rseq_scratch2]\n\t" |
4168 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4169 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4170 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4171 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4172 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4173 | RSEQ_INJECT_ASM(3) |
4174 | "cmpq %[v], %[expect]\n\t" |
4175 | "jnz 5f\n\t" |
4176 | RSEQ_INJECT_ASM(4) |
4177 | #ifdef RSEQ_COMPARE_TWICE |
4178 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
4179 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) |
4180 | "cmpq %[v], %[expect]\n\t" |
4181 | "jnz 7f\n\t" |
4182 | #endif |
4183 | @@ -498,7 +581,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
4184 | #endif |
4185 | : /* gcc asm goto does not allow outputs */ |
4186 | : [cpu_id] "r" (cpu), |
4187 | - [rseq_abi] "r" (&__rseq_abi), |
4188 | + [rseq_offset] "r" (rseq_offset), |
4189 | /* final store input */ |
4190 | [v] "m" (*v), |
4191 | [expect] "r" (expect), |
4192 | @@ -517,16 +600,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
4193 | , error1, error2 |
4194 | #endif |
4195 | ); |
4196 | + rseq_after_asm_goto(); |
4197 | return 0; |
4198 | abort: |
4199 | + rseq_after_asm_goto(); |
4200 | RSEQ_INJECT_FAILED |
4201 | return -1; |
4202 | cmpfail: |
4203 | + rseq_after_asm_goto(); |
4204 | return 1; |
4205 | #ifdef RSEQ_COMPARE_TWICE |
4206 | error1: |
4207 | + rseq_after_asm_goto(); |
4208 | rseq_bug("cpu_id comparison failed"); |
4209 | error2: |
4210 | + rseq_after_asm_goto(); |
4211 | rseq_bug("expected value comparison failed"); |
4212 | #endif |
4213 | } |
4214 | @@ -543,7 +631,9 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
4215 | |
4216 | #endif /* !RSEQ_SKIP_FASTPATH */ |
4217 | |
4218 | -#elif __i386__ |
4219 | +#elif defined(__i386__) |
4220 | + |
4221 | +#define RSEQ_ASM_TP_SEGMENT %%gs |
4222 | |
4223 | #define rseq_smp_mb() \ |
4224 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") |
4225 | @@ -644,14 +734,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
4226 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
4227 | #endif |
4228 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4229 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4230 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4231 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4232 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4233 | RSEQ_INJECT_ASM(3) |
4234 | "cmpl %[v], %[expect]\n\t" |
4235 | "jnz %l[cmpfail]\n\t" |
4236 | RSEQ_INJECT_ASM(4) |
4237 | #ifdef RSEQ_COMPARE_TWICE |
4238 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
4239 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4240 | "cmpl %[v], %[expect]\n\t" |
4241 | "jnz %l[error2]\n\t" |
4242 | #endif |
4243 | @@ -662,7 +752,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
4244 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4245 | : /* gcc asm goto does not allow outputs */ |
4246 | : [cpu_id] "r" (cpu), |
4247 | - [rseq_abi] "r" (&__rseq_abi), |
4248 | + [rseq_offset] "r" (rseq_offset), |
4249 | [v] "m" (*v), |
4250 | [expect] "r" (expect), |
4251 | [newv] "r" (newv) |
4252 | @@ -673,16 +763,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) |
4253 | , error1, error2 |
4254 | #endif |
4255 | ); |
4256 | + rseq_after_asm_goto(); |
4257 | return 0; |
4258 | abort: |
4259 | + rseq_after_asm_goto(); |
4260 | RSEQ_INJECT_FAILED |
4261 | return -1; |
4262 | cmpfail: |
4263 | + rseq_after_asm_goto(); |
4264 | return 1; |
4265 | #ifdef RSEQ_COMPARE_TWICE |
4266 | error1: |
4267 | + rseq_after_asm_goto(); |
4268 | rseq_bug("cpu_id comparison failed"); |
4269 | error2: |
4270 | + rseq_after_asm_goto(); |
4271 | rseq_bug("expected value comparison failed"); |
4272 | #endif |
4273 | } |
4274 | @@ -693,7 +788,7 @@ error2: |
4275 | */ |
4276 | static inline __attribute__((always_inline)) |
4277 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
4278 | - off_t voffp, intptr_t *load, int cpu) |
4279 | + long voffp, intptr_t *load, int cpu) |
4280 | { |
4281 | RSEQ_INJECT_C(9) |
4282 | |
4283 | @@ -705,15 +800,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
4284 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
4285 | #endif |
4286 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4287 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4288 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4289 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4290 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4291 | RSEQ_INJECT_ASM(3) |
4292 | "movl %[v], %%ebx\n\t" |
4293 | "cmpl %%ebx, %[expectnot]\n\t" |
4294 | "je %l[cmpfail]\n\t" |
4295 | RSEQ_INJECT_ASM(4) |
4296 | #ifdef RSEQ_COMPARE_TWICE |
4297 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
4298 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4299 | "movl %[v], %%ebx\n\t" |
4300 | "cmpl %%ebx, %[expectnot]\n\t" |
4301 | "je %l[error2]\n\t" |
4302 | @@ -728,7 +823,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
4303 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4304 | : /* gcc asm goto does not allow outputs */ |
4305 | : [cpu_id] "r" (cpu), |
4306 | - [rseq_abi] "r" (&__rseq_abi), |
4307 | + [rseq_offset] "r" (rseq_offset), |
4308 | /* final store input */ |
4309 | [v] "m" (*v), |
4310 | [expectnot] "r" (expectnot), |
4311 | @@ -741,16 +836,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, |
4312 | , error1, error2 |
4313 | #endif |
4314 | ); |
4315 | + rseq_after_asm_goto(); |
4316 | return 0; |
4317 | abort: |
4318 | + rseq_after_asm_goto(); |
4319 | RSEQ_INJECT_FAILED |
4320 | return -1; |
4321 | cmpfail: |
4322 | + rseq_after_asm_goto(); |
4323 | return 1; |
4324 | #ifdef RSEQ_COMPARE_TWICE |
4325 | error1: |
4326 | + rseq_after_asm_goto(); |
4327 | rseq_bug("cpu_id comparison failed"); |
4328 | error2: |
4329 | + rseq_after_asm_goto(); |
4330 | rseq_bug("expected value comparison failed"); |
4331 | #endif |
4332 | } |
4333 | @@ -766,11 +866,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
4334 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) |
4335 | #endif |
4336 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4337 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4338 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4339 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4340 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4341 | RSEQ_INJECT_ASM(3) |
4342 | #ifdef RSEQ_COMPARE_TWICE |
4343 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
4344 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4345 | #endif |
4346 | /* final store */ |
4347 | "addl %[count], %[v]\n\t" |
4348 | @@ -779,7 +879,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
4349 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4350 | : /* gcc asm goto does not allow outputs */ |
4351 | : [cpu_id] "r" (cpu), |
4352 | - [rseq_abi] "r" (&__rseq_abi), |
4353 | + [rseq_offset] "r" (rseq_offset), |
4354 | /* final store input */ |
4355 | [v] "m" (*v), |
4356 | [count] "ir" (count) |
4357 | @@ -790,12 +890,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) |
4358 | , error1 |
4359 | #endif |
4360 | ); |
4361 | + rseq_after_asm_goto(); |
4362 | return 0; |
4363 | abort: |
4364 | + rseq_after_asm_goto(); |
4365 | RSEQ_INJECT_FAILED |
4366 | return -1; |
4367 | #ifdef RSEQ_COMPARE_TWICE |
4368 | error1: |
4369 | + rseq_after_asm_goto(); |
4370 | rseq_bug("cpu_id comparison failed"); |
4371 | #endif |
4372 | } |
4373 | @@ -815,14 +918,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
4374 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
4375 | #endif |
4376 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4377 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4378 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4379 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4380 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4381 | RSEQ_INJECT_ASM(3) |
4382 | "cmpl %[v], %[expect]\n\t" |
4383 | "jnz %l[cmpfail]\n\t" |
4384 | RSEQ_INJECT_ASM(4) |
4385 | #ifdef RSEQ_COMPARE_TWICE |
4386 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
4387 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4388 | "cmpl %[v], %[expect]\n\t" |
4389 | "jnz %l[error2]\n\t" |
4390 | #endif |
4391 | @@ -837,7 +940,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
4392 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4393 | : /* gcc asm goto does not allow outputs */ |
4394 | : [cpu_id] "r" (cpu), |
4395 | - [rseq_abi] "r" (&__rseq_abi), |
4396 | + [rseq_offset] "r" (rseq_offset), |
4397 | /* try store input */ |
4398 | [v2] "m" (*v2), |
4399 | [newv2] "m" (newv2), |
4400 | @@ -852,16 +955,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, |
4401 | , error1, error2 |
4402 | #endif |
4403 | ); |
4404 | + rseq_after_asm_goto(); |
4405 | return 0; |
4406 | abort: |
4407 | + rseq_after_asm_goto(); |
4408 | RSEQ_INJECT_FAILED |
4409 | return -1; |
4410 | cmpfail: |
4411 | + rseq_after_asm_goto(); |
4412 | return 1; |
4413 | #ifdef RSEQ_COMPARE_TWICE |
4414 | error1: |
4415 | + rseq_after_asm_goto(); |
4416 | rseq_bug("cpu_id comparison failed"); |
4417 | error2: |
4418 | + rseq_after_asm_goto(); |
4419 | rseq_bug("expected value comparison failed"); |
4420 | #endif |
4421 | } |
4422 | @@ -881,15 +989,15 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
4423 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) |
4424 | #endif |
4425 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4426 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4427 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4428 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4429 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4430 | RSEQ_INJECT_ASM(3) |
4431 | "movl %[expect], %%eax\n\t" |
4432 | "cmpl %[v], %%eax\n\t" |
4433 | "jnz %l[cmpfail]\n\t" |
4434 | RSEQ_INJECT_ASM(4) |
4435 | #ifdef RSEQ_COMPARE_TWICE |
4436 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
4437 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4438 | "movl %[expect], %%eax\n\t" |
4439 | "cmpl %[v], %%eax\n\t" |
4440 | "jnz %l[error2]\n\t" |
4441 | @@ -905,7 +1013,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
4442 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4443 | : /* gcc asm goto does not allow outputs */ |
4444 | : [cpu_id] "r" (cpu), |
4445 | - [rseq_abi] "r" (&__rseq_abi), |
4446 | + [rseq_offset] "r" (rseq_offset), |
4447 | /* try store input */ |
4448 | [v2] "m" (*v2), |
4449 | [newv2] "r" (newv2), |
4450 | @@ -920,16 +1028,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, |
4451 | , error1, error2 |
4452 | #endif |
4453 | ); |
4454 | + rseq_after_asm_goto(); |
4455 | return 0; |
4456 | abort: |
4457 | + rseq_after_asm_goto(); |
4458 | RSEQ_INJECT_FAILED |
4459 | return -1; |
4460 | cmpfail: |
4461 | + rseq_after_asm_goto(); |
4462 | return 1; |
4463 | #ifdef RSEQ_COMPARE_TWICE |
4464 | error1: |
4465 | + rseq_after_asm_goto(); |
4466 | rseq_bug("cpu_id comparison failed"); |
4467 | error2: |
4468 | + rseq_after_asm_goto(); |
4469 | rseq_bug("expected value comparison failed"); |
4470 | #endif |
4471 | |
4472 | @@ -951,8 +1064,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
4473 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) |
4474 | #endif |
4475 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4476 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4477 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4478 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4479 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4480 | RSEQ_INJECT_ASM(3) |
4481 | "cmpl %[v], %[expect]\n\t" |
4482 | "jnz %l[cmpfail]\n\t" |
4483 | @@ -961,7 +1074,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
4484 | "jnz %l[cmpfail]\n\t" |
4485 | RSEQ_INJECT_ASM(5) |
4486 | #ifdef RSEQ_COMPARE_TWICE |
4487 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
4488 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
4489 | "cmpl %[v], %[expect]\n\t" |
4490 | "jnz %l[error2]\n\t" |
4491 | "cmpl %[expect2], %[v2]\n\t" |
4492 | @@ -975,7 +1088,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
4493 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) |
4494 | : /* gcc asm goto does not allow outputs */ |
4495 | : [cpu_id] "r" (cpu), |
4496 | - [rseq_abi] "r" (&__rseq_abi), |
4497 | + [rseq_offset] "r" (rseq_offset), |
4498 | /* cmp2 input */ |
4499 | [v2] "m" (*v2), |
4500 | [expect2] "r" (expect2), |
4501 | @@ -990,18 +1103,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, |
4502 | , error1, error2, error3 |
4503 | #endif |
4504 | ); |
4505 | + rseq_after_asm_goto(); |
4506 | return 0; |
4507 | abort: |
4508 | + rseq_after_asm_goto(); |
4509 | RSEQ_INJECT_FAILED |
4510 | return -1; |
4511 | cmpfail: |
4512 | + rseq_after_asm_goto(); |
4513 | return 1; |
4514 | #ifdef RSEQ_COMPARE_TWICE |
4515 | error1: |
4516 | + rseq_after_asm_goto(); |
4517 | rseq_bug("cpu_id comparison failed"); |
4518 | error2: |
4519 | + rseq_after_asm_goto(); |
4520 | rseq_bug("1st expected value comparison failed"); |
4521 | error3: |
4522 | + rseq_after_asm_goto(); |
4523 | rseq_bug("2nd expected value comparison failed"); |
4524 | #endif |
4525 | } |
4526 | @@ -1027,15 +1146,15 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
4527 | "movl %[dst], %[rseq_scratch1]\n\t" |
4528 | "movl %[len], %[rseq_scratch2]\n\t" |
4529 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4530 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4531 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4532 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4533 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4534 | RSEQ_INJECT_ASM(3) |
4535 | "movl %[expect], %%eax\n\t" |
4536 | "cmpl %%eax, %[v]\n\t" |
4537 | "jnz 5f\n\t" |
4538 | RSEQ_INJECT_ASM(4) |
4539 | #ifdef RSEQ_COMPARE_TWICE |
4540 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
4541 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) |
4542 | "movl %[expect], %%eax\n\t" |
4543 | "cmpl %%eax, %[v]\n\t" |
4544 | "jnz 7f\n\t" |
4545 | @@ -1085,7 +1204,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
4546 | #endif |
4547 | : /* gcc asm goto does not allow outputs */ |
4548 | : [cpu_id] "r" (cpu), |
4549 | - [rseq_abi] "r" (&__rseq_abi), |
4550 | + [rseq_offset] "r" (rseq_offset), |
4551 | /* final store input */ |
4552 | [v] "m" (*v), |
4553 | [expect] "m" (expect), |
4554 | @@ -1104,16 +1223,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, |
4555 | , error1, error2 |
4556 | #endif |
4557 | ); |
4558 | + rseq_after_asm_goto(); |
4559 | return 0; |
4560 | abort: |
4561 | + rseq_after_asm_goto(); |
4562 | RSEQ_INJECT_FAILED |
4563 | return -1; |
4564 | cmpfail: |
4565 | + rseq_after_asm_goto(); |
4566 | return 1; |
4567 | #ifdef RSEQ_COMPARE_TWICE |
4568 | error1: |
4569 | + rseq_after_asm_goto(); |
4570 | rseq_bug("cpu_id comparison failed"); |
4571 | error2: |
4572 | + rseq_after_asm_goto(); |
4573 | rseq_bug("expected value comparison failed"); |
4574 | #endif |
4575 | } |
4576 | @@ -1139,15 +1263,15 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
4577 | "movl %[dst], %[rseq_scratch1]\n\t" |
4578 | "movl %[len], %[rseq_scratch2]\n\t" |
4579 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
4580 | - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
4581 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) |
4582 | + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
4583 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) |
4584 | RSEQ_INJECT_ASM(3) |
4585 | "movl %[expect], %%eax\n\t" |
4586 | "cmpl %%eax, %[v]\n\t" |
4587 | "jnz 5f\n\t" |
4588 | RSEQ_INJECT_ASM(4) |
4589 | #ifdef RSEQ_COMPARE_TWICE |
4590 | - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
4591 | + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) |
4592 | "movl %[expect], %%eax\n\t" |
4593 | "cmpl %%eax, %[v]\n\t" |
4594 | "jnz 7f\n\t" |
4595 | @@ -1198,7 +1322,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
4596 | #endif |
4597 | : /* gcc asm goto does not allow outputs */ |
4598 | : [cpu_id] "r" (cpu), |
4599 | - [rseq_abi] "r" (&__rseq_abi), |
4600 | + [rseq_offset] "r" (rseq_offset), |
4601 | /* final store input */ |
4602 | [v] "m" (*v), |
4603 | [expect] "m" (expect), |
4604 | @@ -1217,16 +1341,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, |
4605 | , error1, error2 |
4606 | #endif |
4607 | ); |
4608 | + rseq_after_asm_goto(); |
4609 | return 0; |
4610 | abort: |
4611 | + rseq_after_asm_goto(); |
4612 | RSEQ_INJECT_FAILED |
4613 | return -1; |
4614 | cmpfail: |
4615 | + rseq_after_asm_goto(); |
4616 | return 1; |
4617 | #ifdef RSEQ_COMPARE_TWICE |
4618 | error1: |
4619 | + rseq_after_asm_goto(); |
4620 | rseq_bug("cpu_id comparison failed"); |
4621 | error2: |
4622 | + rseq_after_asm_goto(); |
4623 | rseq_bug("expected value comparison failed"); |
4624 | #endif |
4625 | } |
4626 | diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c |
4627 | index 7159eb777fd34..986b9458efb26 100644 |
4628 | --- a/tools/testing/selftests/rseq/rseq.c |
4629 | +++ b/tools/testing/selftests/rseq/rseq.c |
4630 | @@ -26,131 +26,124 @@ |
4631 | #include <assert.h> |
4632 | #include <signal.h> |
4633 | #include <limits.h> |
4634 | +#include <dlfcn.h> |
4635 | +#include <stddef.h> |
4636 | |
4637 | +#include "../kselftest.h" |
4638 | #include "rseq.h" |
4639 | |
4640 | -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) |
4641 | +static const ptrdiff_t *libc_rseq_offset_p; |
4642 | +static const unsigned int *libc_rseq_size_p; |
4643 | +static const unsigned int *libc_rseq_flags_p; |
4644 | |
4645 | -__thread volatile struct rseq __rseq_abi = { |
4646 | - .cpu_id = RSEQ_CPU_ID_UNINITIALIZED, |
4647 | -}; |
4648 | +/* Offset from the thread pointer to the rseq area. */ |
4649 | +ptrdiff_t rseq_offset; |
4650 | |
4651 | -/* |
4652 | - * Shared with other libraries. This library may take rseq ownership if it is |
4653 | - * still 0 when executing the library constructor. Set to 1 by library |
4654 | - * constructor when handling rseq. Set to 0 in destructor if handling rseq. |
4655 | - */ |
4656 | -int __rseq_handled; |
4657 | +/* Size of the registered rseq area. 0 if the registration was |
4658 | + unsuccessful. */ |
4659 | +unsigned int rseq_size = -1U; |
4660 | + |
4661 | +/* Flags used during rseq registration. */ |
4662 | +unsigned int rseq_flags; |
4663 | |
4664 | -/* Whether this library have ownership of rseq registration. */ |
4665 | static int rseq_ownership; |
4666 | |
4667 | -static __thread volatile uint32_t __rseq_refcount; |
4668 | +static |
4669 | +__thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"))) = { |
4670 | + .cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED, |
4671 | +}; |
4672 | |
4673 | -static void signal_off_save(sigset_t *oldset) |
4674 | +static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len, |
4675 | + int flags, uint32_t sig) |
4676 | { |
4677 | - sigset_t set; |
4678 | - int ret; |
4679 | - |
4680 | - sigfillset(&set); |
4681 | - ret = pthread_sigmask(SIG_BLOCK, &set, oldset); |
4682 | - if (ret) |
4683 | - abort(); |
4684 | + return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); |
4685 | } |
4686 | |
4687 | -static void signal_restore(sigset_t oldset) |
4688 | +int rseq_available(void) |
4689 | { |
4690 | - int ret; |
4691 | + int rc; |
4692 | |
4693 | - ret = pthread_sigmask(SIG_SETMASK, &oldset, NULL); |
4694 | - if (ret) |
4695 | + rc = sys_rseq(NULL, 0, 0, 0); |
4696 | + if (rc != -1) |
4697 | abort(); |
4698 | -} |
4699 | - |
4700 | -static int sys_rseq(volatile struct rseq *rseq_abi, uint32_t rseq_len, |
4701 | - int flags, uint32_t sig) |
4702 | -{ |
4703 | - return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); |
4704 | + switch (errno) { |
4705 | + case ENOSYS: |
4706 | + return 0; |
4707 | + case EINVAL: |
4708 | + return 1; |
4709 | + default: |
4710 | + abort(); |
4711 | + } |
4712 | } |
4713 | |
4714 | int rseq_register_current_thread(void) |
4715 | { |
4716 | - int rc, ret = 0; |
4717 | - sigset_t oldset; |
4718 | + int rc; |
4719 | |
4720 | - if (!rseq_ownership) |
4721 | + if (!rseq_ownership) { |
4722 | + /* Treat libc's ownership as a successful registration. */ |
4723 | return 0; |
4724 | - signal_off_save(&oldset); |
4725 | - if (__rseq_refcount == UINT_MAX) { |
4726 | - ret = -1; |
4727 | - goto end; |
4728 | - } |
4729 | - if (__rseq_refcount++) |
4730 | - goto end; |
4731 | - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG); |
4732 | - if (!rc) { |
4733 | - assert(rseq_current_cpu_raw() >= 0); |
4734 | - goto end; |
4735 | } |
4736 | - if (errno != EBUSY) |
4737 | - __rseq_abi.cpu_id = RSEQ_CPU_ID_REGISTRATION_FAILED; |
4738 | - ret = -1; |
4739 | - __rseq_refcount--; |
4740 | -end: |
4741 | - signal_restore(oldset); |
4742 | - return ret; |
4743 | + rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), 0, RSEQ_SIG); |
4744 | + if (rc) |
4745 | + return -1; |
4746 | + assert(rseq_current_cpu_raw() >= 0); |
4747 | + return 0; |
4748 | } |
4749 | |
4750 | int rseq_unregister_current_thread(void) |
4751 | { |
4752 | - int rc, ret = 0; |
4753 | - sigset_t oldset; |
4754 | + int rc; |
4755 | |
4756 | - if (!rseq_ownership) |
4757 | + if (!rseq_ownership) { |
4758 | + /* Treat libc's ownership as a successful unregistration. */ |
4759 | return 0; |
4760 | - signal_off_save(&oldset); |
4761 | - if (!__rseq_refcount) { |
4762 | - ret = -1; |
4763 | - goto end; |
4764 | } |
4765 | - if (--__rseq_refcount) |
4766 | - goto end; |
4767 | - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), |
4768 | - RSEQ_FLAG_UNREGISTER, RSEQ_SIG); |
4769 | - if (!rc) |
4770 | - goto end; |
4771 | - __rseq_refcount = 1; |
4772 | - ret = -1; |
4773 | -end: |
4774 | - signal_restore(oldset); |
4775 | - return ret; |
4776 | + rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); |
4777 | + if (rc) |
4778 | + return -1; |
4779 | + return 0; |
4780 | } |
4781 | |
4782 | -int32_t rseq_fallback_current_cpu(void) |
4783 | +static __attribute__((constructor)) |
4784 | +void rseq_init(void) |
4785 | { |
4786 | - int32_t cpu; |
4787 | - |
4788 | - cpu = sched_getcpu(); |
4789 | - if (cpu < 0) { |
4790 | - perror("sched_getcpu()"); |
4791 | - abort(); |
4792 | + libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset"); |
4793 | + libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size"); |
4794 | + libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags"); |
4795 | + if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p) { |
4796 | + /* rseq registration owned by glibc */ |
4797 | + rseq_offset = *libc_rseq_offset_p; |
4798 | + rseq_size = *libc_rseq_size_p; |
4799 | + rseq_flags = *libc_rseq_flags_p; |
4800 | + return; |
4801 | } |
4802 | - return cpu; |
4803 | -} |
4804 | - |
4805 | -void __attribute__((constructor)) rseq_init(void) |
4806 | -{ |
4807 | - /* Check whether rseq is handled by another library. */ |
4808 | - if (__rseq_handled) |
4809 | + if (!rseq_available()) |
4810 | return; |
4811 | - __rseq_handled = 1; |
4812 | rseq_ownership = 1; |
4813 | + rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer(); |
4814 | + rseq_size = sizeof(struct rseq_abi); |
4815 | + rseq_flags = 0; |
4816 | } |
4817 | |
4818 | -void __attribute__((destructor)) rseq_fini(void) |
4819 | +static __attribute__((destructor)) |
4820 | +void rseq_exit(void) |
4821 | { |
4822 | if (!rseq_ownership) |
4823 | return; |
4824 | - __rseq_handled = 0; |
4825 | + rseq_offset = 0; |
4826 | + rseq_size = -1U; |
4827 | rseq_ownership = 0; |
4828 | } |
4829 | + |
4830 | +int32_t rseq_fallback_current_cpu(void) |
4831 | +{ |
4832 | + int32_t cpu; |
4833 | + |
4834 | + cpu = sched_getcpu(); |
4835 | + if (cpu < 0) { |
4836 | + perror("sched_getcpu()"); |
4837 | + abort(); |
4838 | + } |
4839 | + return cpu; |
4840 | +} |
4841 | diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h |
4842 | index d40d60e7499e8..c8f49b018dbb0 100644 |
4843 | --- a/tools/testing/selftests/rseq/rseq.h |
4844 | +++ b/tools/testing/selftests/rseq/rseq.h |
4845 | @@ -16,7 +16,9 @@ |
4846 | #include <errno.h> |
4847 | #include <stdio.h> |
4848 | #include <stdlib.h> |
4849 | -#include <linux/rseq.h> |
4850 | +#include <stddef.h> |
4851 | +#include "rseq-abi.h" |
4852 | +#include "compiler.h" |
4853 | |
4854 | /* |
4855 | * Empty code injection macros, override when testing. |
4856 | @@ -43,8 +45,20 @@ |
4857 | #define RSEQ_INJECT_FAILED |
4858 | #endif |
4859 | |
4860 | -extern __thread volatile struct rseq __rseq_abi; |
4861 | -extern int __rseq_handled; |
4862 | +#include "rseq-thread-pointer.h" |
4863 | + |
4864 | +/* Offset from the thread pointer to the rseq area. */ |
4865 | +extern ptrdiff_t rseq_offset; |
4866 | +/* Size of the registered rseq area. 0 if the registration was |
4867 | + unsuccessful. */ |
4868 | +extern unsigned int rseq_size; |
4869 | +/* Flags used during rseq registration. */ |
4870 | +extern unsigned int rseq_flags; |
4871 | + |
4872 | +static inline struct rseq_abi *rseq_get_abi(void) |
4873 | +{ |
4874 | + return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); |
4875 | +} |
4876 | |
4877 | #define rseq_likely(x) __builtin_expect(!!(x), 1) |
4878 | #define rseq_unlikely(x) __builtin_expect(!!(x), 0) |
4879 | @@ -108,7 +122,7 @@ int32_t rseq_fallback_current_cpu(void); |
4880 | */ |
4881 | static inline int32_t rseq_current_cpu_raw(void) |
4882 | { |
4883 | - return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id); |
4884 | + return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id); |
4885 | } |
4886 | |
4887 | /* |
4888 | @@ -124,7 +138,7 @@ static inline int32_t rseq_current_cpu_raw(void) |
4889 | */ |
4890 | static inline uint32_t rseq_cpu_start(void) |
4891 | { |
4892 | - return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id_start); |
4893 | + return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start); |
4894 | } |
4895 | |
4896 | static inline uint32_t rseq_current_cpu(void) |
4897 | @@ -139,11 +153,7 @@ static inline uint32_t rseq_current_cpu(void) |
4898 | |
4899 | static inline void rseq_clear_rseq_cs(void) |
4900 | { |
4901 | -#ifdef __LP64__ |
4902 | - __rseq_abi.rseq_cs.ptr = 0; |
4903 | -#else |
4904 | - __rseq_abi.rseq_cs.ptr.ptr32 = 0; |
4905 | -#endif |
4906 | + RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); |
4907 | } |
4908 | |
4909 | /* |