rustc_thread_pool/sleep/
counters.rs1use std::sync::atomic::{AtomicUsize, Ordering};
2
3pub(super) struct AtomicCounters {
4 value: AtomicUsize,
16}
17
18#[derive(Copy, Clone)]
19pub(super) struct Counters {
20 word: usize,
21}
22
23#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
27pub(super) struct JobsEventCounter(usize);
28
29impl JobsEventCounter {
30 pub(super) const DUMMY: JobsEventCounter = JobsEventCounter(usize::MAX);
31
32 #[inline]
33 pub(super) fn as_usize(self) -> usize {
34 self.0
35 }
36
37 #[inline]
42 pub(super) fn is_sleepy(self) -> bool {
43 (self.as_usize() & 1) == 0
44 }
45
46 #[inline]
50 pub(super) fn is_active(self) -> bool {
51 !self.is_sleepy()
52 }
53}
54
55#[cfg(target_pointer_width = "64")]
57const THREADS_BITS: usize = 16;
58
59#[cfg(target_pointer_width = "32")]
60const THREADS_BITS: usize = 8;
61
62#[allow(clippy::erasing_op)]
65const SLEEPING_SHIFT: usize = 0 * THREADS_BITS;
66
67#[allow(clippy::identity_op)]
70const INACTIVE_SHIFT: usize = 1 * THREADS_BITS;
71
72const JEC_SHIFT: usize = 2 * THREADS_BITS;
75
76pub(crate) const THREADS_MAX: usize = (1 << THREADS_BITS) - 1;
78
79const ONE_SLEEPING: usize = 1;
81
82const ONE_INACTIVE: usize = 1 << INACTIVE_SHIFT;
85
86const ONE_JEC: usize = 1 << JEC_SHIFT;
88
89impl AtomicCounters {
90 #[inline]
91 pub(super) fn new() -> AtomicCounters {
92 AtomicCounters { value: AtomicUsize::new(0) }
93 }
94
95 #[inline]
99 pub(super) fn load(&self, ordering: Ordering) -> Counters {
100 Counters::new(self.value.load(ordering))
101 }
102
103 #[inline]
104 fn try_exchange(&self, old_value: Counters, new_value: Counters, ordering: Ordering) -> bool {
105 self.value
106 .compare_exchange(old_value.word, new_value.word, ordering, Ordering::Relaxed)
107 .is_ok()
108 }
109
110 #[inline]
118 pub(super) fn add_inactive_thread(&self) {
119 self.value.fetch_add(ONE_INACTIVE, Ordering::SeqCst);
120 }
121
122 pub(super) fn increment_jobs_event_counter_if(
127 &self,
128 increment_when: impl Fn(JobsEventCounter) -> bool,
129 ) -> Counters {
130 loop {
131 let old_value = self.load(Ordering::SeqCst);
132 if increment_when(old_value.jobs_counter()) {
133 let new_value = old_value.increment_jobs_counter();
134 if self.try_exchange(old_value, new_value, Ordering::SeqCst) {
135 return new_value;
136 }
137 } else {
138 return old_value;
139 }
140 }
141 }
142
143 #[inline]
149 pub(super) fn sub_inactive_thread(&self) -> usize {
150 let old_value = Counters::new(self.value.fetch_sub(ONE_INACTIVE, Ordering::SeqCst));
151 debug_assert!(
152 old_value.inactive_threads() > 0,
153 "sub_inactive_thread: old_value {:?} has no inactive threads",
154 old_value,
155 );
156 debug_assert!(
157 old_value.sleeping_threads() <= old_value.inactive_threads(),
158 "sub_inactive_thread: old_value {:?} had {} sleeping threads and {} inactive threads",
159 old_value,
160 old_value.sleeping_threads(),
161 old_value.inactive_threads(),
162 );
163
164 let sleeping_threads = old_value.sleeping_threads();
167 Ord::min(sleeping_threads, 2)
168 }
169
170 #[inline]
175 pub(super) fn sub_sleeping_thread(&self) {
176 let old_value = Counters::new(self.value.fetch_sub(ONE_SLEEPING, Ordering::SeqCst));
177 debug_assert!(
178 old_value.sleeping_threads() > 0,
179 "sub_sleeping_thread: old_value {:?} had no sleeping threads",
180 old_value,
181 );
182 debug_assert!(
183 old_value.sleeping_threads() <= old_value.inactive_threads(),
184 "sub_sleeping_thread: old_value {:?} had {} sleeping threads and {} inactive threads",
185 old_value,
186 old_value.sleeping_threads(),
187 old_value.inactive_threads(),
188 );
189 }
190
191 #[inline]
192 pub(super) fn try_add_sleeping_thread(&self, old_value: Counters) -> bool {
193 debug_assert!(
194 old_value.inactive_threads() > 0,
195 "try_add_sleeping_thread: old_value {:?} has no inactive threads",
196 old_value,
197 );
198 debug_assert!(
199 old_value.sleeping_threads() < THREADS_MAX,
200 "try_add_sleeping_thread: old_value {:?} has too many sleeping threads",
201 old_value,
202 );
203
204 let mut new_value = old_value;
205 new_value.word += ONE_SLEEPING;
206
207 self.try_exchange(old_value, new_value, Ordering::SeqCst)
208 }
209}
210
211#[inline]
212fn select_thread(word: usize, shift: usize) -> usize {
213 (word >> shift) & THREADS_MAX
214}
215
216#[inline]
217fn select_jec(word: usize) -> usize {
218 word >> JEC_SHIFT
219}
220
221impl Counters {
222 #[inline]
223 fn new(word: usize) -> Counters {
224 Counters { word }
225 }
226
227 #[inline]
228 fn increment_jobs_counter(self) -> Counters {
229 Counters { word: self.word.wrapping_add(ONE_JEC) }
232 }
233
234 #[inline]
235 pub(super) fn jobs_counter(self) -> JobsEventCounter {
236 JobsEventCounter(select_jec(self.word))
237 }
238
239 #[inline]
242 pub(super) fn inactive_threads(self) -> usize {
243 select_thread(self.word, INACTIVE_SHIFT)
244 }
245
246 #[inline]
247 pub(super) fn awake_but_idle_threads(self) -> usize {
248 debug_assert!(
249 self.sleeping_threads() <= self.inactive_threads(),
250 "sleeping threads: {} > raw idle threads {}",
251 self.sleeping_threads(),
252 self.inactive_threads()
253 );
254 self.inactive_threads() - self.sleeping_threads()
255 }
256
257 #[inline]
258 pub(super) fn sleeping_threads(self) -> usize {
259 select_thread(self.word, SLEEPING_SHIFT)
260 }
261}
262
263impl std::fmt::Debug for Counters {
264 fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
265 let word = format!("{:016x}", self.word);
266 fmt.debug_struct("Counters")
267 .field("word", &word)
268 .field("jobs", &self.jobs_counter().0)
269 .field("inactive", &self.inactive_threads())
270 .field("sleeping", &self.sleeping_threads())
271 .finish()
272 }
273}