cargo/util/network/
retry.rs1use crate::util::errors::HttpNotSuccessful;
45use crate::{CargoResult, GlobalContext};
46use anyhow::Error;
47use rand::Rng;
48use std::cmp::min;
49use std::time::Duration;
50
51pub struct Retry<'a> {
53 gctx: &'a GlobalContext,
54 retries: u64,
58 max_retries: u64,
62}
63
64pub enum RetryResult<T> {
66 Success(T),
70 Err(anyhow::Error),
72 Retry(u64),
78}
79
80const MAX_RETRY_SLEEP_MS: u64 = 10 * 1000;
82const INITIAL_RETRY_SLEEP_BASE_MS: u64 = 500;
86const INITIAL_RETRY_JITTER_MS: u64 = 1000;
91
92impl<'a> Retry<'a> {
93 pub fn new(gctx: &'a GlobalContext) -> CargoResult<Retry<'a>> {
94 Ok(Retry {
95 gctx,
96 retries: 0,
97 max_retries: gctx.net_config()?.retry.unwrap_or(3) as u64,
98 })
99 }
100
101 pub fn r#try<T>(&mut self, f: impl FnOnce() -> CargoResult<T>) -> RetryResult<T> {
105 match f() {
106 Err(ref e) if maybe_spurious(e) && self.retries < self.max_retries => {
107 let err_msg = e
108 .downcast_ref::<HttpNotSuccessful>()
109 .map(|http_err| http_err.display_short())
110 .unwrap_or_else(|| e.root_cause().to_string());
111 let msg = format!(
112 "spurious network error ({} tries remaining): {err_msg}",
113 self.max_retries - self.retries,
114 );
115 if let Err(e) = self.gctx.shell().warn(msg) {
116 return RetryResult::Err(e);
117 }
118 self.retries += 1;
119 RetryResult::Retry(self.next_sleep_ms())
120 }
121 Err(e) => RetryResult::Err(e),
122 Ok(r) => RetryResult::Success(r),
123 }
124 }
125
126 fn next_sleep_ms(&self) -> u64 {
128 if let Ok(sleep) = self.gctx.get_env("__CARGO_TEST_FIXED_RETRY_SLEEP_MS") {
129 return sleep.parse().expect("a u64");
130 }
131
132 if self.retries == 1 {
133 let mut rng = rand::rng();
134 INITIAL_RETRY_SLEEP_BASE_MS + rng.random_range(0..INITIAL_RETRY_JITTER_MS)
135 } else {
136 min(
137 ((self.retries - 1) * 3) * 1000 + INITIAL_RETRY_SLEEP_BASE_MS,
138 MAX_RETRY_SLEEP_MS,
139 )
140 }
141 }
142}
143
144fn maybe_spurious(err: &Error) -> bool {
145 if let Some(git_err) = err.downcast_ref::<git2::Error>() {
146 match git_err.class() {
147 git2::ErrorClass::Net
148 | git2::ErrorClass::Os
149 | git2::ErrorClass::Zlib
150 | git2::ErrorClass::Http => return git_err.code() != git2::ErrorCode::Certificate,
151 _ => (),
152 }
153 }
154 if let Some(curl_err) = err.downcast_ref::<curl::Error>() {
155 if curl_err.is_couldnt_connect()
156 || curl_err.is_couldnt_resolve_proxy()
157 || curl_err.is_couldnt_resolve_host()
158 || curl_err.is_operation_timedout()
159 || curl_err.is_recv_error()
160 || curl_err.is_send_error()
161 || curl_err.is_http2_error()
162 || curl_err.is_http2_stream_error()
163 || curl_err.is_ssl_connect_error()
164 || curl_err.is_partial_file()
165 {
166 return true;
167 }
168 }
169 if let Some(not_200) = err.downcast_ref::<HttpNotSuccessful>() {
170 if 500 <= not_200.code && not_200.code < 600 {
171 return true;
172 }
173 }
174
175 use gix::protocol::transport::IsSpuriousError;
176
177 if let Some(err) = err.downcast_ref::<crate::sources::git::fetch::Error>() {
178 if err.is_spurious() {
179 return true;
180 }
181 }
182
183 false
184}
185
186pub fn with_retry<T, F>(gctx: &GlobalContext, mut callback: F) -> CargoResult<T>
203where
204 F: FnMut() -> CargoResult<T>,
205{
206 let mut retry = Retry::new(gctx)?;
207 loop {
208 match retry.r#try(&mut callback) {
209 RetryResult::Success(r) => return Ok(r),
210 RetryResult::Err(e) => return Err(e),
211 RetryResult::Retry(sleep) => std::thread::sleep(Duration::from_millis(sleep)),
212 }
213 }
214}
215
216#[test]
217fn with_retry_repeats_the_call_then_works() {
218 use crate::core::Shell;
219
220 let error1 = HttpNotSuccessful {
222 code: 501,
223 url: "Uri".to_string(),
224 ip: None,
225 body: Vec::new(),
226 headers: Vec::new(),
227 }
228 .into();
229 let error2 = HttpNotSuccessful {
230 code: 502,
231 url: "Uri".to_string(),
232 ip: None,
233 body: Vec::new(),
234 headers: Vec::new(),
235 }
236 .into();
237 let mut results: Vec<CargoResult<()>> = vec![Ok(()), Err(error1), Err(error2)];
238 let gctx = GlobalContext::default().unwrap();
239 *gctx.shell() = Shell::from_write(Box::new(Vec::new()));
240 let result = with_retry(&gctx, || results.pop().unwrap());
241 assert!(result.is_ok())
242}
243
244#[test]
245fn with_retry_finds_nested_spurious_errors() {
246 use crate::core::Shell;
247
248 let error1 = anyhow::Error::from(HttpNotSuccessful {
251 code: 501,
252 url: "Uri".to_string(),
253 ip: None,
254 body: Vec::new(),
255 headers: Vec::new(),
256 });
257 let error1 = anyhow::Error::from(error1.context("A non-spurious wrapping err"));
258 let error2 = anyhow::Error::from(HttpNotSuccessful {
259 code: 502,
260 url: "Uri".to_string(),
261 ip: None,
262 body: Vec::new(),
263 headers: Vec::new(),
264 });
265 let error2 = anyhow::Error::from(error2.context("A second chained error"));
266 let mut results: Vec<CargoResult<()>> = vec![Ok(()), Err(error1), Err(error2)];
267 let gctx = GlobalContext::default().unwrap();
268 *gctx.shell() = Shell::from_write(Box::new(Vec::new()));
269 let result = with_retry(&gctx, || results.pop().unwrap());
270 assert!(result.is_ok())
271}
272
273#[test]
274fn default_retry_schedule() {
275 use crate::core::Shell;
276
277 let spurious = || -> CargoResult<()> {
278 Err(anyhow::Error::from(HttpNotSuccessful {
279 code: 500,
280 url: "Uri".to_string(),
281 ip: None,
282 body: Vec::new(),
283 headers: Vec::new(),
284 }))
285 };
286 let gctx = GlobalContext::default().unwrap();
287 *gctx.shell() = Shell::from_write(Box::new(Vec::new()));
288 let mut retry = Retry::new(&gctx).unwrap();
289 match retry.r#try(|| spurious()) {
290 RetryResult::Retry(sleep) => {
291 assert!(
292 sleep >= INITIAL_RETRY_SLEEP_BASE_MS
293 && sleep < INITIAL_RETRY_SLEEP_BASE_MS + INITIAL_RETRY_JITTER_MS
294 );
295 }
296 _ => panic!("unexpected non-retry"),
297 }
298 match retry.r#try(|| spurious()) {
299 RetryResult::Retry(sleep) => assert_eq!(sleep, 3500),
300 _ => panic!("unexpected non-retry"),
301 }
302 match retry.r#try(|| spurious()) {
303 RetryResult::Retry(sleep) => assert_eq!(sleep, 6500),
304 _ => panic!("unexpected non-retry"),
305 }
306 match retry.r#try(|| spurious()) {
307 RetryResult::Err(_) => {}
308 _ => panic!("unexpected non-retry"),
309 }
310}
311
312#[test]
313fn curle_http2_stream_is_spurious() {
314 let code = curl_sys::CURLE_HTTP2_STREAM;
315 let err = curl::Error::new(code);
316 assert!(maybe_spurious(&err.into()));
317}