crates_io/
lib.rs

1//! > This crate is maintained by the Cargo team for use by the wider
2//! > ecosystem. This crate follows semver compatibility for its APIs.
3
4use std::collections::BTreeMap;
5use std::fs::File;
6use std::io::prelude::*;
7use std::io::{Cursor, SeekFrom};
8use std::time::Instant;
9
10use curl::easy::{Easy, List};
11use percent_encoding::{percent_encode, NON_ALPHANUMERIC};
12use serde::{Deserialize, Serialize};
13use url::Url;
14
15pub type Result<T> = std::result::Result<T, Error>;
16
17pub struct Registry {
18    /// The base URL for issuing API requests.
19    host: String,
20    /// Optional authorization token.
21    /// If None, commands requiring authorization will fail.
22    token: Option<String>,
23    /// Curl handle for issuing requests.
24    handle: Easy,
25    /// Whether to include the authorization token with all requests.
26    auth_required: bool,
27}
28
29#[derive(PartialEq, Clone, Copy)]
30pub enum Auth {
31    Authorized,
32    Unauthorized,
33}
34
35#[derive(Deserialize)]
36pub struct Crate {
37    pub name: String,
38    pub description: Option<String>,
39    pub max_version: String,
40}
41
42/// This struct is serialized as JSON and sent as metadata ahead of the crate
43/// tarball when publishing crates to a crate registry like crates.io.
44///
45/// see <https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish>
46#[derive(Serialize, Deserialize)]
47pub struct NewCrate {
48    pub name: String,
49    pub vers: String,
50    pub deps: Vec<NewCrateDependency>,
51    pub features: BTreeMap<String, Vec<String>>,
52    pub authors: Vec<String>,
53    pub description: Option<String>,
54    pub documentation: Option<String>,
55    pub homepage: Option<String>,
56    pub readme: Option<String>,
57    pub readme_file: Option<String>,
58    pub keywords: Vec<String>,
59    pub categories: Vec<String>,
60    pub license: Option<String>,
61    pub license_file: Option<String>,
62    pub repository: Option<String>,
63    pub badges: BTreeMap<String, BTreeMap<String, String>>,
64    pub links: Option<String>,
65    pub rust_version: Option<String>,
66}
67
68#[derive(Serialize, Deserialize)]
69pub struct NewCrateDependency {
70    pub optional: bool,
71    pub default_features: bool,
72    pub name: String,
73    pub features: Vec<String>,
74    pub version_req: String,
75    pub target: Option<String>,
76    pub kind: String,
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub registry: Option<String>,
79    #[serde(skip_serializing_if = "Option::is_none")]
80    pub explicit_name_in_toml: Option<String>,
81    #[serde(skip_serializing_if = "Option::is_none")]
82    pub artifact: Option<Vec<String>>,
83    #[serde(skip_serializing_if = "Option::is_none")]
84    pub bindep_target: Option<String>,
85    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
86    pub lib: bool,
87}
88
89#[derive(Deserialize)]
90pub struct User {
91    pub id: u32,
92    pub login: String,
93    pub avatar: Option<String>,
94    pub email: Option<String>,
95    pub name: Option<String>,
96}
97
98pub struct Warnings {
99    pub invalid_categories: Vec<String>,
100    pub invalid_badges: Vec<String>,
101    pub other: Vec<String>,
102}
103
104#[derive(Deserialize)]
105struct R {
106    ok: bool,
107}
108#[derive(Deserialize)]
109struct OwnerResponse {
110    ok: bool,
111    msg: String,
112}
113#[derive(Deserialize)]
114struct ApiErrorList {
115    errors: Vec<ApiError>,
116}
117#[derive(Deserialize)]
118struct ApiError {
119    detail: String,
120}
121#[derive(Serialize)]
122struct OwnersReq<'a> {
123    users: &'a [&'a str],
124}
125#[derive(Deserialize)]
126struct Users {
127    users: Vec<User>,
128}
129#[derive(Deserialize)]
130struct TotalCrates {
131    total: u32,
132}
133#[derive(Deserialize)]
134struct Crates {
135    crates: Vec<Crate>,
136    meta: TotalCrates,
137}
138
139/// Error returned when interacting with a registry.
140#[derive(Debug, thiserror::Error)]
141pub enum Error {
142    /// Error from libcurl.
143    #[error(transparent)]
144    Curl(#[from] curl::Error),
145
146    /// Error from seriailzing the request payload and deserializing the
147    /// response body (like response body didn't match expected structure).
148    #[error(transparent)]
149    Json(#[from] serde_json::Error),
150
151    /// Error from IO. Mostly from reading the tarball to upload.
152    #[error("failed to seek tarball")]
153    Io(#[from] std::io::Error),
154
155    /// Response body was not valid utf8.
156    #[error("invalid response body from server")]
157    Utf8(#[from] std::string::FromUtf8Error),
158
159    /// Error from API response containing JSON field `errors.details`.
160    #[error(
161        "the remote server responded with an error{}: {}",
162        status(*code),
163        errors.join(", "),
164    )]
165    Api {
166        code: u32,
167        headers: Vec<String>,
168        errors: Vec<String>,
169    },
170
171    /// Error from API response which didn't have pre-programmed `errors.details`.
172    #[error(
173        "failed to get a 200 OK response, got {code}\nheaders:\n\t{}\nbody:\n{body}",
174        headers.join("\n\t"),
175    )]
176    Code {
177        code: u32,
178        headers: Vec<String>,
179        body: String,
180    },
181
182    /// Reason why the token was invalid.
183    #[error("{0}")]
184    InvalidToken(&'static str),
185
186    /// Server was unavailable and timeouted. Happened when uploading a way
187    /// too large tarball to crates.io.
188    #[error(
189        "Request timed out after 30 seconds. If you're trying to \
190         upload a crate it may be too large. If the crate is under \
191         10MB in size, you can email help@crates.io for assistance.\n\
192         Total size was {0}."
193    )]
194    Timeout(u64),
195}
196
197impl Registry {
198    /// Creates a new `Registry`.
199    ///
200    /// ## Example
201    ///
202    /// ```rust
203    /// use curl::easy::Easy;
204    /// use crates_io::Registry;
205    ///
206    /// let mut handle = Easy::new();
207    /// // If connecting to crates.io, a user-agent is required.
208    /// handle.useragent("my_crawler (example.com/info)");
209    /// let mut reg = Registry::new_handle(String::from("https://crates.io"), None, handle, true);
210    /// ```
211    pub fn new_handle(
212        host: String,
213        token: Option<String>,
214        handle: Easy,
215        auth_required: bool,
216    ) -> Registry {
217        Registry {
218            host,
219            token,
220            handle,
221            auth_required,
222        }
223    }
224
225    pub fn set_token(&mut self, token: Option<String>) {
226        self.token = token;
227    }
228
229    fn token(&self) -> Result<&str> {
230        let token = self.token.as_ref().ok_or_else(|| {
231            Error::InvalidToken("no upload token found, please run `cargo login`")
232        })?;
233        check_token(token)?;
234        Ok(token)
235    }
236
237    pub fn host(&self) -> &str {
238        &self.host
239    }
240
241    pub fn host_is_crates_io(&self) -> bool {
242        is_url_crates_io(&self.host)
243    }
244
245    pub fn add_owners(&mut self, krate: &str, owners: &[&str]) -> Result<String> {
246        let body = serde_json::to_string(&OwnersReq { users: owners })?;
247        let body = self.put(&format!("/crates/{}/owners", krate), body.as_bytes())?;
248        assert!(serde_json::from_str::<OwnerResponse>(&body)?.ok);
249        Ok(serde_json::from_str::<OwnerResponse>(&body)?.msg)
250    }
251
252    pub fn remove_owners(&mut self, krate: &str, owners: &[&str]) -> Result<()> {
253        let body = serde_json::to_string(&OwnersReq { users: owners })?;
254        let body = self.delete(&format!("/crates/{}/owners", krate), Some(body.as_bytes()))?;
255        assert!(serde_json::from_str::<OwnerResponse>(&body)?.ok);
256        Ok(())
257    }
258
259    pub fn list_owners(&mut self, krate: &str) -> Result<Vec<User>> {
260        let body = self.get(&format!("/crates/{}/owners", krate))?;
261        Ok(serde_json::from_str::<Users>(&body)?.users)
262    }
263
264    pub fn publish(&mut self, krate: &NewCrate, mut tarball: &File) -> Result<Warnings> {
265        let json = serde_json::to_string(krate)?;
266        // Prepare the body. The format of the upload request is:
267        //
268        //      <le u32 of json>
269        //      <json request> (metadata for the package)
270        //      <le u32 of tarball>
271        //      <source tarball>
272
273        // NOTE: This can be replaced with `stream_len` if it is ever stabilized.
274        //
275        // This checks the length using seeking instead of metadata, because
276        // on some filesystems, getting the metadata will fail because
277        // the file was renamed in ops::package.
278        let tarball_len = tarball.seek(SeekFrom::End(0))?;
279        tarball.seek(SeekFrom::Start(0))?;
280        let header = {
281            let mut w = Vec::new();
282            w.extend(&(json.len() as u32).to_le_bytes());
283            w.extend(json.as_bytes().iter().cloned());
284            w.extend(&(tarball_len as u32).to_le_bytes());
285            w
286        };
287        let size = tarball_len as usize + header.len();
288        let mut body = Cursor::new(header).chain(tarball);
289
290        let url = format!("{}/api/v1/crates/new", self.host);
291
292        self.handle.put(true)?;
293        self.handle.url(&url)?;
294        self.handle.in_filesize(size as u64)?;
295        let mut headers = List::new();
296        headers.append("Accept: application/json")?;
297        headers.append(&format!("Authorization: {}", self.token()?))?;
298        self.handle.http_headers(headers)?;
299
300        let started = Instant::now();
301        let body = self
302            .handle(&mut |buf| body.read(buf).unwrap_or(0))
303            .map_err(|e| match e {
304                Error::Code { code, .. }
305                    if code == 503
306                        && started.elapsed().as_secs() >= 29
307                        && self.host_is_crates_io() =>
308                {
309                    Error::Timeout(tarball_len)
310                }
311                _ => e.into(),
312            })?;
313
314        let response = if body.is_empty() {
315            "{}".parse()?
316        } else {
317            body.parse::<serde_json::Value>()?
318        };
319
320        let invalid_categories: Vec<String> = response
321            .get("warnings")
322            .and_then(|j| j.get("invalid_categories"))
323            .and_then(|j| j.as_array())
324            .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
325            .unwrap_or_else(Vec::new);
326
327        let invalid_badges: Vec<String> = response
328            .get("warnings")
329            .and_then(|j| j.get("invalid_badges"))
330            .and_then(|j| j.as_array())
331            .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
332            .unwrap_or_else(Vec::new);
333
334        let other: Vec<String> = response
335            .get("warnings")
336            .and_then(|j| j.get("other"))
337            .and_then(|j| j.as_array())
338            .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
339            .unwrap_or_else(Vec::new);
340
341        Ok(Warnings {
342            invalid_categories,
343            invalid_badges,
344            other,
345        })
346    }
347
348    pub fn search(&mut self, query: &str, limit: u32) -> Result<(Vec<Crate>, u32)> {
349        let formatted_query = percent_encode(query.as_bytes(), NON_ALPHANUMERIC);
350        let body = self.req(
351            &format!("/crates?q={}&per_page={}", formatted_query, limit),
352            None,
353            Auth::Unauthorized,
354        )?;
355
356        let crates = serde_json::from_str::<Crates>(&body)?;
357        Ok((crates.crates, crates.meta.total))
358    }
359
360    pub fn yank(&mut self, krate: &str, version: &str) -> Result<()> {
361        let body = self.delete(&format!("/crates/{}/{}/yank", krate, version), None)?;
362        assert!(serde_json::from_str::<R>(&body)?.ok);
363        Ok(())
364    }
365
366    pub fn unyank(&mut self, krate: &str, version: &str) -> Result<()> {
367        let body = self.put(&format!("/crates/{}/{}/unyank", krate, version), &[])?;
368        assert!(serde_json::from_str::<R>(&body)?.ok);
369        Ok(())
370    }
371
372    fn put(&mut self, path: &str, b: &[u8]) -> Result<String> {
373        self.handle.put(true)?;
374        self.req(path, Some(b), Auth::Authorized)
375    }
376
377    fn get(&mut self, path: &str) -> Result<String> {
378        self.handle.get(true)?;
379        self.req(path, None, Auth::Authorized)
380    }
381
382    fn delete(&mut self, path: &str, b: Option<&[u8]>) -> Result<String> {
383        self.handle.custom_request("DELETE")?;
384        self.req(path, b, Auth::Authorized)
385    }
386
387    fn req(&mut self, path: &str, body: Option<&[u8]>, authorized: Auth) -> Result<String> {
388        self.handle.url(&format!("{}/api/v1{}", self.host, path))?;
389        let mut headers = List::new();
390        headers.append("Accept: application/json")?;
391        if body.is_some() {
392            headers.append("Content-Type: application/json")?;
393        }
394
395        if self.auth_required || authorized == Auth::Authorized {
396            headers.append(&format!("Authorization: {}", self.token()?))?;
397        }
398        self.handle.http_headers(headers)?;
399        match body {
400            Some(mut body) => {
401                self.handle.upload(true)?;
402                self.handle.in_filesize(body.len() as u64)?;
403                self.handle(&mut |buf| body.read(buf).unwrap_or(0))
404                    .map_err(|e| e.into())
405            }
406            None => self.handle(&mut |_| 0).map_err(|e| e.into()),
407        }
408    }
409
410    fn handle(&mut self, read: &mut dyn FnMut(&mut [u8]) -> usize) -> Result<String> {
411        let mut headers = Vec::new();
412        let mut body = Vec::new();
413        {
414            let mut handle = self.handle.transfer();
415            handle.read_function(|buf| Ok(read(buf)))?;
416            handle.write_function(|data| {
417                body.extend_from_slice(data);
418                Ok(data.len())
419            })?;
420            handle.header_function(|data| {
421                // Headers contain trailing \r\n, trim them to make it easier
422                // to work with.
423                let s = String::from_utf8_lossy(data).trim().to_string();
424                // Don't let server sneak extra lines anywhere.
425                if s.contains('\n') {
426                    return true;
427                }
428                headers.push(s);
429                true
430            })?;
431            handle.perform()?;
432        }
433
434        let body = String::from_utf8(body)?;
435        let errors = serde_json::from_str::<ApiErrorList>(&body)
436            .ok()
437            .map(|s| s.errors.into_iter().map(|s| s.detail).collect::<Vec<_>>());
438
439        match (self.handle.response_code()?, errors) {
440            (0, None) => Ok(body),
441            (code, None) if is_success(code) => Ok(body),
442            (code, Some(errors)) => Err(Error::Api {
443                code,
444                headers,
445                errors,
446            }),
447            (code, None) => Err(Error::Code {
448                code,
449                headers,
450                body,
451            }),
452        }
453    }
454}
455
456fn is_success(code: u32) -> bool {
457    code >= 200 && code < 300
458}
459
460fn status(code: u32) -> String {
461    if is_success(code) {
462        String::new()
463    } else {
464        let reason = reason(code);
465        format!(" (status {code} {reason})")
466    }
467}
468
469fn reason(code: u32) -> &'static str {
470    // Taken from https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
471    match code {
472        100 => "Continue",
473        101 => "Switching Protocol",
474        103 => "Early Hints",
475        200 => "OK",
476        201 => "Created",
477        202 => "Accepted",
478        203 => "Non-Authoritative Information",
479        204 => "No Content",
480        205 => "Reset Content",
481        206 => "Partial Content",
482        300 => "Multiple Choice",
483        301 => "Moved Permanently",
484        302 => "Found",
485        303 => "See Other",
486        304 => "Not Modified",
487        307 => "Temporary Redirect",
488        308 => "Permanent Redirect",
489        400 => "Bad Request",
490        401 => "Unauthorized",
491        402 => "Payment Required",
492        403 => "Forbidden",
493        404 => "Not Found",
494        405 => "Method Not Allowed",
495        406 => "Not Acceptable",
496        407 => "Proxy Authentication Required",
497        408 => "Request Timeout",
498        409 => "Conflict",
499        410 => "Gone",
500        411 => "Length Required",
501        412 => "Precondition Failed",
502        413 => "Payload Too Large",
503        414 => "URI Too Long",
504        415 => "Unsupported Media Type",
505        416 => "Request Range Not Satisfiable",
506        417 => "Expectation Failed",
507        429 => "Too Many Requests",
508        431 => "Request Header Fields Too Large",
509        500 => "Internal Server Error",
510        501 => "Not Implemented",
511        502 => "Bad Gateway",
512        503 => "Service Unavailable",
513        504 => "Gateway Timeout",
514        _ => "<unknown>",
515    }
516}
517
518/// Returns `true` if the host of the given URL is "crates.io".
519pub fn is_url_crates_io(url: &str) -> bool {
520    Url::parse(url)
521        .map(|u| u.host_str() == Some("crates.io"))
522        .unwrap_or(false)
523}
524
525/// Checks if a token is valid or malformed.
526///
527/// This check is necessary to prevent sending tokens which create an invalid HTTP request.
528/// It would be easier to check just for alphanumeric tokens, but we can't be sure that all
529/// registries only create tokens in that format so that is as less restricted as possible.
530pub fn check_token(token: &str) -> Result<()> {
531    if token.is_empty() {
532        return Err(Error::InvalidToken("please provide a non-empty token"));
533    }
534    if token.bytes().all(|b| {
535        // This is essentially the US-ASCII limitation of
536        // https://www.rfc-editor.org/rfc/rfc9110#name-field-values. That is,
537        // visible ASCII characters (0x21-0x7e), space, and tab. We want to be
538        // able to pass this in an HTTP header without encoding.
539        b >= 32 && b < 127 || b == b'\t'
540    }) {
541        Ok(())
542    } else {
543        Err(Error::InvalidToken(
544            "token contains invalid characters.\nOnly printable ISO-8859-1 characters \
545             are allowed as it is sent in a HTTPS header.",
546        ))
547    }
548}
crates_io/lib.rs

crates_io/
lib.rs
