30 static void spscring_pc_signal(
struct spscring *ring);
31 static void spscring_cp_signal(
struct spscring *ring);
33 static inline size_t spscring_atomic_load(
34 const volatile spscring_atomic_t *
object);
35 static inline void spscring_atomic_store(
36 volatile spscring_atomic_t *
object,
size_t desired);
37 static inline _Bool spscring_atomic_compare_exchange_strong(
38 volatile spscring_atomic_t *
object,
size_t *expected,
40 static inline _Bool spscring_atomic_compare_exchange_weak(
41 volatile spscring_atomic_t *
object,
size_t *expected,
44 static inline void spscring_yield(
void);
50 assert(size <= SIZE_MAX / 2 + 1);
59 assert(ring->p.ctx.size == ring->c.ctx.size);
61 return ring->p.ctx.size;
68 struct spscring_ctx *ctx = &ring->p.ctx;
69 assert(ctx->size <= SIZE_MAX / 2 + 1);
70 assert(ctx->pos < ctx->size);
71 assert(ctx->pos <= ctx->end);
72 assert(ctx->end - ctx->pos <= ctx->size);
74 size_t cpos = spscring_atomic_load(&ring->c.pos) + ctx->size;
77 assert(ctx->pos <= ctx->end);
78 assert(ctx->end - ctx->pos <= ctx->size);
79 return ctx->end - ctx->pos;
86 struct spscring_ctx *ctx = &ring->p.ctx;
87 assert(ctx->size <= SIZE_MAX / 2 + 1);
88 assert(ctx->pos < ctx->size);
89 assert(ctx->pos <= ctx->end);
90 assert(ctx->end - ctx->pos <= ctx->size);
92 if (ctx->end < ctx->size)
94 return MIN(ctx->end, ctx->size) - ctx->pos;
101 struct spscring_ctx *ctx = &ring->p.ctx;
102 assert(ctx->size <= SIZE_MAX / 2 + 1);
103 assert(ctx->pos < ctx->size);
104 assert(ctx->pos <= ctx->end);
105 assert(ctx->end - ctx->pos <= ctx->size);
108 size_t capacity = ctx->end - ctx->pos;
109 if (*psize > capacity) {
111 if (*psize > capacity)
121 struct spscring_ctx *ctx = &ring->p.ctx;
122 assert(ctx->size <= SIZE_MAX / 2 + 1);
123 assert(ctx->pos < ctx->size);
124 assert(ctx->pos <= ctx->end);
125 assert(ctx->end - ctx->pos <= ctx->size);
128 size_t capacity =
MIN(ctx->end, ctx->size) - ctx->pos;
129 if (*psize > capacity) {
131 if (*psize > capacity)
141 struct spscring_ctx *ctx = &ring->p.ctx;
142 assert(ctx->size <= SIZE_MAX / 2 + 1);
143 assert(ctx->pos < ctx->size);
144 assert(ctx->pos <= ctx->end);
145 assert(ctx->end - ctx->pos <= ctx->size);
146 assert(size <= ctx->end - ctx->pos);
148 if ((ctx->pos += size) >= ctx->size) {
149 ctx->base += ctx->size;
150 ctx->pos -= ctx->size;
151 assert(ctx->end >= ctx->size);
152 ctx->end -= ctx->size;
155 spscring_atomic_store(&ring->p.pos, ctx->base + ctx->pos);
157 spscring_pc_signal(ring);
164 void (*func)(
struct spscring *ring,
void *arg),
void *arg)
167 struct spscring_ctx *ctx = &ring->p.ctx;
168 assert(ctx->size <= SIZE_MAX / 2 + 1);
169 struct spscring_sig *sig = &ring->p.sig;
171 if (size > ctx->size)
185 spscring_atomic_store(&sig->size, size);
192 if (spscring_atomic_compare_exchange_strong(
193 &sig->size, &size, 0))
197 assert(size == 0 || size == SIZE_MAX);
207 struct spscring_sig *sig = &ring->p.sig;
209 size_t size = spscring_atomic_load(&sig->size);
213 if (size == SIZE_MAX) {
217 size = spscring_atomic_load(&sig->size);
219 }
else if (spscring_atomic_compare_exchange_weak(
220 &sig->size, &size, 0)) {
232 struct spscring_ctx *ctx = &ring->c.ctx;
233 assert(ctx->size <= SIZE_MAX / 2 + 1);
234 assert(ctx->pos < ctx->size);
235 assert(ctx->pos <= ctx->end);
236 assert(ctx->end - ctx->pos <= ctx->size);
238 size_t ppos = spscring_atomic_load(&ring->p.pos);
241 assert(ctx->pos <= ppos);
242 assert(ctx->end - ctx->pos <= ctx->size);
243 return ctx->end - ctx->pos;
250 struct spscring_ctx *ctx = &ring->c.ctx;
251 assert(ctx->size <= SIZE_MAX / 2 + 1);
252 assert(ctx->pos < ctx->size);
253 assert(ctx->pos <= ctx->end);
254 assert(ctx->end - ctx->pos <= ctx->size);
256 if (ctx->end < ctx->size)
258 return MIN(ctx->end, ctx->size) - ctx->pos;
265 struct spscring_ctx *ctx = &ring->c.ctx;
266 assert(ctx->size <= SIZE_MAX / 2 + 1);
267 assert(ctx->pos < ctx->size);
268 assert(ctx->pos <= ctx->end);
269 assert(ctx->end - ctx->pos <= ctx->size);
272 size_t capacity = ctx->end - ctx->pos;
273 if (*psize > capacity) {
275 if (*psize > capacity)
285 struct spscring_ctx *ctx = &ring->c.ctx;
286 assert(ctx->size <= SIZE_MAX / 2 + 1);
287 assert(ctx->pos < ctx->size);
288 assert(ctx->pos <= ctx->end);
289 assert(ctx->end - ctx->pos <= ctx->size);
292 size_t capacity =
MIN(ctx->end, ctx->size) - ctx->pos;
293 if (*psize > capacity) {
295 if (*psize > capacity)
305 struct spscring_ctx *ctx = &ring->c.ctx;
306 assert(ctx->size <= SIZE_MAX / 2 + 1);
307 assert(ctx->pos < ctx->size);
308 assert(ctx->pos <= ctx->end);
309 assert(ctx->end - ctx->pos <= ctx->size);
310 assert(size <= ctx->end - ctx->pos);
312 if ((ctx->pos += size) >= ctx->size) {
313 ctx->base += ctx->size;
314 ctx->pos -= ctx->size;
315 assert(ctx->end >= ctx->size);
316 ctx->end -= ctx->size;
319 spscring_atomic_store(&ring->c.pos, ctx->base + ctx->pos);
321 spscring_cp_signal(ring);
328 void (*func)(
struct spscring *ring,
void *arg),
void *arg)
331 struct spscring_ctx *ctx = &ring->c.ctx;
332 assert(ctx->size <= SIZE_MAX / 2 + 1);
333 struct spscring_sig *sig = &ring->c.sig;
335 if (size > ctx->size)
349 spscring_atomic_store(&sig->size, size);
356 if (spscring_atomic_compare_exchange_strong(
357 &sig->size, &size, 0))
361 assert(size == 0 || size == SIZE_MAX);
371 struct spscring_sig *sig = &ring->c.sig;
373 size_t size = spscring_atomic_load(&sig->size);
377 if (size == SIZE_MAX) {
381 size = spscring_atomic_load(&sig->size);
383 }
else if (spscring_atomic_compare_exchange_weak(
384 &sig->size, &size, 0)) {
393 spscring_pc_signal(
struct spscring *ring)
396 struct spscring_ctx *ctx = &ring->p.ctx;
397 struct spscring_sig *sig = &ring->c.sig;
400 size_t size = spscring_atomic_load(&sig->size);
410 if (spscring_atomic_compare_exchange_weak(
411 &sig->size, &size, SIZE_MAX)) {
416 void (*func)(
struct spscring * ring,
void *arg) =
418 void *arg = sig->arg;
422 spscring_atomic_store(&sig->size, 0);
435 spscring_cp_signal(
struct spscring *ring)
438 struct spscring_ctx *ctx = &ring->c.ctx;
439 struct spscring_sig *sig = &ring->p.sig;
442 size_t size = spscring_atomic_load(&sig->size);
452 if (spscring_atomic_compare_exchange_weak(
453 &sig->size, &size, SIZE_MAX)) {
458 void (*func)(
struct spscring * ring,
void *arg) =
460 void *arg = sig->arg;
464 spscring_atomic_store(&sig->size, 0);
478 spscring_atomic_load(
const volatile spscring_atomic_t *
object)
480 #if __STDC_NO_ATOMICS__
483 return atomic_load_explicit(
object, memory_order_acquire);
488 spscring_atomic_store(
volatile spscring_atomic_t *
object,
size_t desired)
490 #if __STDC_NO_ATOMICS__
493 atomic_store_explicit(
object, desired, memory_order_release);
498 spscring_atomic_compare_exchange_strong(
volatile spscring_atomic_t *
object,
499 size_t *expected,
size_t desired)
501 #if __STDC_NO_ATOMICS__
504 size_t tmp = InterlockedCompareExchange64(
505 (
volatile LONGLONG *)
object, desired, *expected);
507 size_t tmp = InterlockedCompareExchange(
508 (
volatile LONG *)
object, desired, *expected);
510 _Bool result = tmp == *expected;
515 assert(*
object == *expected);
521 return atomic_compare_exchange_strong_explicit(
object, expected,
522 desired, memory_order_acq_rel, memory_order_acquire);
527 spscring_atomic_compare_exchange_weak(
volatile spscring_atomic_t *
object,
528 size_t *expected,
size_t desired)
530 #if __STDC_NO_ATOMICS__
531 return spscring_atomic_compare_exchange_strong(
532 object, expected, desired);
534 return atomic_compare_exchange_weak_explicit(
object, expected, desired,
535 memory_order_acq_rel, memory_order_acquire);
544 #elif (defined(__i386__) || defined(__x86_64__)) \
545 && (GNUC_PREREQ(4, 7) || __has_builtin(__builtin_ia32_pause))
546 __builtin_ia32_pause();