fix deque steal race condition
@@ -635,7 +635,9 @@ impl<T> Stealer<T> {
let task = unsafe { buffer.deref().read(f) };
// Try incrementing the front index to steal the task.
- if self
+ // If the buffer has been swapped or the increment fails, we retry.
+ if self.inner.buffer.load(Ordering::Acquire, guard) != buffer
+ || self
.inner
.front
.compare_exchange(f, f.wrapping_add(1), Ordering::SeqCst, Ordering::Relaxed)
@@ -741,7 +743,9 @@ impl<T> Stealer<T> {
}
// Try incrementing the front index to steal the batch.
- if self
+ // If the buffer has been swapped or the increment fails, we retry.
+ if self.inner.buffer.load(Ordering::Acquire, guard) != buffer
+ || self
.inner
.front
.compare_exchange(
@@ -781,10 +785,17 @@ impl<T> Stealer<T> {
let task = unsafe { buffer.deref().read(f) };
// Try incrementing the front index to steal the task.
- if self
+ // If the buffer has been swapped or the increment fails, we retry.
+ if self.inner.buffer.load(Ordering::Acquire, guard) != buffer
+ || self
.inner
.front
- .compare_exchange(f, f.wrapping_add(1), Ordering::SeqCst, Ordering::Relaxed)
+ .compare_exchange(
+ f,
+ f.wrapping_add(1),
+ Ordering::SeqCst,
+ Ordering::Relaxed,
+ )
.is_err()
{
// We didn't steal this task, forget it and break from the loop.
@@ -927,8 +938,10 @@ impl<T> Stealer<T> {
}
}
- // Try incrementing the front index to steal the batch.
- if self
+ // Try incrementing the front index to steal the task.
+ // If the buffer has been swapped or the increment fails, we retry.
+ if self.inner.buffer.load(Ordering::Acquire, guard) != buffer
+ || self
.inner
.front
.compare_exchange(
@@ -983,10 +996,17 @@ impl<T> Stealer<T> {
let tmp = unsafe { buffer.deref().read(f) };
// Try incrementing the front index to steal the task.
- if self
+ // If the buffer has been swapped or the increment fails, we retry.
+ if self.inner.buffer.load(Ordering::Acquire, guard) != buffer
+ || self
.inner
.front
- .compare_exchange(f, f.wrapping_add(1), Ordering::SeqCst, Ordering::Relaxed)
+ .compare_exchange(
+ f,
+ f.wrapping_add(1),
+ Ordering::SeqCst,
+ Ordering::Relaxed,
+ )
.is_err()
{
// We didn't steal this task, forget it and break from the loop.
