ureq/
response.rs

1use std::io::{self, Cursor, Read};
2use std::net::SocketAddr;
3use std::num::NonZeroUsize;
4use std::str::FromStr;
5use std::{fmt, io::BufRead};
6
7use log::debug;
8use url::Url;
9
10use crate::body::SizedReader;
11use crate::chunked::Decoder as ChunkDecoder;
12use crate::error::{Error, ErrorKind::BadStatus};
13use crate::header::{get_all_headers, get_header, Header, HeaderLine};
14use crate::pool::{PoolReturnRead, PoolReturner};
15use crate::stream::{DeadlineStream, ReadOnlyStream, Stream};
16use crate::unit::Unit;
17use crate::{stream, Agent, ErrorKind};
18
19#[cfg(feature = "json")]
20use serde::de::DeserializeOwned;
21
22#[cfg(feature = "charset")]
23use encoding_rs::Encoding;
24
25#[cfg(feature = "gzip")]
26use flate2::read::MultiGzDecoder;
27
28#[cfg(feature = "brotli")]
29use brotli_decompressor::Decompressor as BrotliDecoder;
30
31pub const DEFAULT_CONTENT_TYPE: &str = "text/plain";
32pub const DEFAULT_CHARACTER_SET: &str = "utf-8";
33const INTO_STRING_LIMIT: usize = 10 * 1_024 * 1_024;
34// Follow the example of curl and limit a single header to 100kB:
35// https://curl.se/libcurl/c/CURLOPT_HEADERFUNCTION.html
36const MAX_HEADER_SIZE: usize = 100 * 1_024;
37const MAX_HEADER_COUNT: usize = 100;
38
39#[derive(Copy, Clone, Debug, PartialEq)]
40enum ConnectionOption {
41    KeepAlive,
42    Close,
43}
44
45#[derive(Copy, Clone, Debug, PartialEq)]
46enum BodyType {
47    LengthDelimited(usize),
48    Chunked,
49    CloseDelimited,
50}
51
52/// Response instances are created as results of firing off requests.
53///
54/// The `Response` is used to read response headers and decide what to do with the body.
55/// Note that the socket connection is open and the body not read until one of
56/// [`into_reader()`](#method.into_reader), [`into_json()`](#method.into_json), or
57/// [`into_string()`](#method.into_string) consumes the response.
58///
59/// When dropping a `Response` instance, one one of two things can happen. If
60/// the response has unread bytes, the underlying socket cannot be reused,
61/// and the connection is closed. If there are no unread bytes, the connection
62/// is returned to the [`Agent`] connection pool used (notice there is always
63/// an agent present, even when not explicitly configured by the user).
64///
65/// ```
66/// # fn main() -> Result<(), ureq::Error> {
67/// # ureq::is_test(true);
68/// let response = ureq::get("http://example.com/").call()?;
69///
70/// // socket is still open and the response body has not been read.
71///
72/// let text = response.into_string()?;
73///
74/// // response is consumed, and body has been read.
75/// # Ok(())
76/// # }
77/// ```
78pub struct Response {
79    pub(crate) url: Url,
80    pub(crate) status_line: String,
81    pub(crate) index: ResponseStatusIndex,
82    pub(crate) status: u16,
83    pub(crate) headers: Vec<Header>,
84    pub(crate) reader: Box<dyn Read + Send + Sync + 'static>,
85    /// The socket address of the server that sent the response.
86    pub(crate) remote_addr: SocketAddr,
87    /// The socket address of the client that sent the request.
88    pub(crate) local_addr: SocketAddr,
89    /// The redirect history of this response, if any. The history starts with
90    /// the first response received and ends with the response immediately
91    /// previous to this one.
92    ///
93    /// If this response was not redirected, the history is empty.
94    pub(crate) history: Vec<Url>,
95}
96
97/// index into status_line where we split: HTTP/1.1 200 OK
98#[derive(Debug, Clone, Copy, Eq, PartialEq)]
99pub(crate) struct ResponseStatusIndex {
100    pub(crate) http_version: usize,
101    pub(crate) response_code: usize,
102}
103
104impl fmt::Debug for Response {
105    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
106        write!(
107            f,
108            "Response[status: {}, status_text: {}, url: {}]",
109            self.status(),
110            self.status_text(),
111            self.url,
112        )
113    }
114}
115
116impl Response {
117    /// Construct a response with a status, status text and a string body.
118    ///
119    /// This is hopefully useful for unit tests.
120    ///
121    /// Example:
122    ///
123    /// ```
124    /// # fn main() -> Result<(), ureq::Error> {
125    /// # ureq::is_test(true);
126    /// let resp = ureq::Response::new(401, "Authorization Required", "Please log in")?;
127    ///
128    /// assert_eq!(resp.status(), 401);
129    /// # Ok(())
130    /// # }
131    /// ```
132    pub fn new(status: u16, status_text: &str, body: &str) -> Result<Response, Error> {
133        let r = format!("HTTP/1.1 {} {}\r\n\r\n{}", status, status_text, body);
134        (r.as_ref() as &str).parse()
135    }
136
137    /// The URL we ended up at. This can differ from the request url when
138    /// we have followed redirects.
139    pub fn get_url(&self) -> &str {
140        &self.url[..]
141    }
142
143    /// The http version: `HTTP/1.1`
144    pub fn http_version(&self) -> &str {
145        &self.status_line.as_str()[0..self.index.http_version]
146    }
147
148    /// The status as a u16: `200`
149    pub fn status(&self) -> u16 {
150        self.status
151    }
152
153    /// The status text: `OK`
154    ///
155    /// The HTTP spec allows for non-utf8 status texts. This uses from_utf8_lossy to
156    /// convert such lines to &str.
157    pub fn status_text(&self) -> &str {
158        self.status_line.as_str()[self.index.response_code + 1..].trim()
159    }
160
161    /// The header value for the given name, or None if not found.
162    ///
163    /// For historical reasons, the HTTP spec allows for header values
164    /// to be encoded using encodings like iso-8859-1. Such encodings
165    /// means the values are not possible to interpret as utf-8.
166    ///
167    /// In case the header value can't be read as utf-8, this function
168    /// returns `None` (while the name is visible in [`Response::headers_names()`]).
169    pub fn header(&self, name: &str) -> Option<&str> {
170        get_header(&self.headers, name)
171    }
172
173    /// A list of the header names in this response.
174    /// Lowercased to be uniform.
175    ///
176    /// It's possible for a header name to be returned by this function, and
177    /// still give a `None` value. See [`Response::header()`] for an explanation
178    /// as to why.
179    pub fn headers_names(&self) -> Vec<String> {
180        self.headers
181            .iter()
182            .map(|h| h.name().to_lowercase())
183            .collect()
184    }
185
186    /// Tells if the response has the named header.
187    pub fn has(&self, name: &str) -> bool {
188        self.header(name).is_some()
189    }
190
191    /// All headers corresponding values for the give name, or empty vector.
192    pub fn all(&self, name: &str) -> Vec<&str> {
193        get_all_headers(&self.headers, name)
194    }
195
196    /// The content type part of the "Content-Type" header without
197    /// the charset.
198    ///
199    /// Example:
200    ///
201    /// ```
202    /// # fn main() -> Result<(), ureq::Error> {
203    /// # ureq::is_test(true);
204    /// let resp = ureq::get("http://example.com/charset/iso").call()?;
205    /// assert_eq!(resp.header("content-type"), Some("text/html; charset=ISO-8859-1"));
206    /// assert_eq!("text/html", resp.content_type());
207    /// # Ok(())
208    /// # }
209    /// ```
210    pub fn content_type(&self) -> &str {
211        self.header("content-type")
212            .map(|header| {
213                header
214                    .find(';')
215                    .map(|index| &header[0..index])
216                    .unwrap_or(header)
217            })
218            .unwrap_or(DEFAULT_CONTENT_TYPE)
219    }
220
221    /// The character set part of the "Content-Type".
222    ///
223    /// Example:
224    ///
225    /// ```
226    /// # fn main() -> Result<(), ureq::Error> {
227    /// # ureq::is_test(true);
228    /// let resp = ureq::get("http://example.com/charset/iso").call()?;
229    /// assert_eq!(resp.header("content-type"), Some("text/html; charset=ISO-8859-1"));
230    /// assert_eq!("ISO-8859-1", resp.charset());
231    /// # Ok(())
232    /// # }
233    /// ```
234    pub fn charset(&self) -> &str {
235        charset_from_content_type(self.header("content-type"))
236    }
237
238    /// The socket address of the server that sent the response.
239    pub fn remote_addr(&self) -> SocketAddr {
240        self.remote_addr
241    }
242
243    /// The local address the request was made from.
244    pub fn local_addr(&self) -> SocketAddr {
245        self.local_addr
246    }
247
248    /// Turn this response into a `impl Read` of the body.
249    ///
250    /// 1. If `Transfer-Encoding: chunked`, the returned reader will unchunk it
251    ///    and any `Content-Length` header is ignored.
252    /// 2. If `Content-Length` is set, the returned reader is limited to this byte
253    ///    length regardless of how many bytes the server sends.
254    /// 3. If no length header, the reader is until server stream end.
255    ///
256    /// Note: If you use `read_to_end()` on the resulting reader, a malicious
257    /// server might return enough bytes to exhaust available memory. If you're
258    /// making requests to untrusted servers, you should use `.take()` to
259    /// limit the response bytes read.
260    ///
261    /// Example:
262    ///
263    /// ```
264    /// use std::io::Read;
265    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
266    /// # ureq::is_test(true);
267    /// let resp = ureq::get("http://httpbin.org/bytes/100")
268    ///     .call()?;
269    ///
270    /// assert!(resp.has("Content-Length"));
271    /// let len: usize = resp.header("Content-Length")
272    ///     .unwrap()
273    ///     .parse()?;
274    ///
275    /// let mut bytes: Vec<u8> = Vec::with_capacity(len);
276    /// resp.into_reader()
277    ///     .take(10_000_000)
278    ///     .read_to_end(&mut bytes)?;
279    ///
280    /// assert_eq!(bytes.len(), len);
281    /// # Ok(())
282    /// # }
283    /// ```
284    pub fn into_reader(self) -> Box<dyn Read + Send + Sync + 'static> {
285        self.reader
286    }
287
288    // Determine what to do with the connection after we've read the body.
289    fn connection_option(
290        response_version: &str,
291        connection_header: Option<&str>,
292    ) -> ConnectionOption {
293        // https://datatracker.ietf.org/doc/html/rfc9112#name-tear-down
294        // "A client that receives a "close" connection option MUST cease sending requests on that
295        // connection and close the connection after reading the response message containing the "close"
296        // connection option"
297        //
298        // Per https://www.rfc-editor.org/rfc/rfc2068#section-19.7.1, an HTTP/1.0 response can explicitly
299        // say "Connection: keep-alive" in response to a request with "Connection: keep-alive". We don't
300        // send "Connection: keep-alive" in the request but are willing to accept in the response anyhow.
301        use ConnectionOption::*;
302        let is_http10 = response_version.eq_ignore_ascii_case("HTTP/1.0");
303        match (is_http10, connection_header) {
304            (true, Some(c)) if c.eq_ignore_ascii_case("keep-alive") => KeepAlive,
305            (true, _) => Close,
306            (false, Some(c)) if c.eq_ignore_ascii_case("close") => Close,
307            (false, _) => KeepAlive,
308        }
309    }
310
311    /// Determine how the body should be read, based on
312    /// <https://datatracker.ietf.org/doc/html/rfc9112#name-message-body-length>
313    fn body_type(
314        request_method: &str,
315        response_status: u16,
316        response_version: &str,
317        headers: &[Header],
318    ) -> BodyType {
319        let is_http10 = response_version.eq_ignore_ascii_case("HTTP/1.0");
320
321        let is_head = request_method.eq_ignore_ascii_case("head");
322        let has_no_body = is_head
323            || match response_status {
324                204 | 304 => true,
325                _ => false,
326            };
327
328        if has_no_body {
329            return BodyType::LengthDelimited(0);
330        }
331
332        let is_chunked = get_header(headers, "transfer-encoding")
333            .map(|enc| !enc.is_empty()) // whatever it says, do chunked
334            .unwrap_or(false);
335
336        // https://www.rfc-editor.org/rfc/rfc2068#page-161
337        // > a persistent connection with an HTTP/1.0 client cannot make
338        // > use of the chunked transfer-coding
339        let use_chunked = !is_http10 && is_chunked;
340
341        if use_chunked {
342            return BodyType::Chunked;
343        }
344
345        let length = get_header(headers, "content-length").and_then(|v| v.parse::<usize>().ok());
346
347        match length {
348            Some(n) => BodyType::LengthDelimited(n),
349            None => BodyType::CloseDelimited,
350        }
351    }
352
353    fn stream_to_reader(
354        mut stream: DeadlineStream,
355        unit: &Unit,
356        body_type: BodyType,
357        compression: Option<Compression>,
358        connection_option: ConnectionOption,
359    ) -> Box<dyn Read + Send + Sync + 'static> {
360        if connection_option == ConnectionOption::Close {
361            stream.inner_mut().set_unpoolable();
362        }
363        let inner = stream.inner_ref();
364        let result = inner.set_read_timeout(unit.agent.config.timeout_read);
365        if let Err(e) = result {
366            return Box::new(ErrorReader(e));
367        }
368        let buffer_len = inner.buffer().len();
369
370        let body_reader: Box<dyn Read + Send + Sync> = match body_type {
371            // Chunked responses have an unknown length, but do have an end of body
372            // marker. When we encounter the marker, we can return the underlying stream
373            // to the connection pool.
374            BodyType::Chunked => {
375                debug!("Chunked body in response");
376                Box::new(PoolReturnRead::new(ChunkDecoder::new(stream)))
377            }
378            // Responses with a content-length header means we should limit the reading
379            // of the body to the number of bytes in the header. Once done, we can
380            // return the underlying stream to the connection pool.
381            BodyType::LengthDelimited(len) => {
382                match NonZeroUsize::new(len) {
383                    None => {
384                        debug!("zero-length body returning stream directly to pool");
385                        let stream: Stream = stream.into();
386                        // TODO: This expect can actually panic if we get an error when
387                        // returning the stream to the pool. We reset the read timeouts
388                        // when we do that, and since that's a syscall it can fail.
389                        stream.return_to_pool().expect("returning stream to pool");
390                        Box::new(std::io::empty())
391                    }
392                    Some(len) => {
393                        let mut limited_read = LimitedRead::new(stream, len);
394
395                        if len.get() <= buffer_len {
396                            debug!("Body entirely buffered (length: {})", len);
397                            let mut buf = vec![0; len.get()];
398                            // TODO: This expect can actually panic if we get an error when
399                            // returning the stream to the pool. We reset the read timeouts
400                            // when we do that, and since that's a syscall it can fail.
401                            limited_read
402                                .read_exact(&mut buf)
403                                .expect("failed to read exact buffer length from stream");
404                            Box::new(Cursor::new(buf))
405                        } else {
406                            debug!("Streaming body until content-length: {}", len);
407                            Box::new(limited_read)
408                        }
409                    }
410                }
411            }
412            BodyType::CloseDelimited => {
413                debug!("Body of unknown size - read until socket close");
414                Box::new(stream)
415            }
416        };
417
418        match compression {
419            None => body_reader,
420            Some(c) => c.wrap_reader(body_reader),
421        }
422    }
423
424    /// Turn this response into a String of the response body. By default uses `utf-8`,
425    /// but can work with charset, see below.
426    ///
427    /// This is potentially memory inefficient for large bodies since the
428    /// implementation first reads the reader to end into a `Vec<u8>` and then
429    /// attempts to decode it using the charset.
430    ///
431    /// If the response is larger than 10 megabytes, this will return an error.
432    ///
433    /// Example:
434    ///
435    /// ```
436    /// # fn main() -> Result<(), ureq::Error> {
437    /// # ureq::is_test(true);
438    /// let text = ureq::get("http://httpbin.org/get?success")
439    ///     .call()?
440    ///     .into_string()?;
441    ///
442    /// assert!(text.contains("success"));
443    /// # Ok(())
444    /// # }
445    /// ```
446    ///
447    /// ## Charset support
448    ///
449    /// If you enable feature `ureq = { version = "*", features = ["charset"] }`, into_string()
450    /// attempts to respect the character encoding of the `Content-Type` header. If there is no
451    /// Content-Type header, or the Content-Type header does not specify a charset, into_string()
452    /// uses `utf-8`.
453    ///
454    /// I.e. `Content-Type: text/plain; charset=iso-8859-1` would be decoded in latin-1.
455    ///
456    pub fn into_string(self) -> io::Result<String> {
457        #[cfg(feature = "charset")]
458        let encoding = Encoding::for_label(self.charset().as_bytes())
459            .or_else(|| Encoding::for_label(DEFAULT_CHARACTER_SET.as_bytes()))
460            .unwrap();
461
462        let mut buf: Vec<u8> = vec![];
463        self.into_reader()
464            .take((INTO_STRING_LIMIT + 1) as u64)
465            .read_to_end(&mut buf)?;
466        if buf.len() > INTO_STRING_LIMIT {
467            return Err(io::Error::new(
468                io::ErrorKind::Other,
469                "response too big for into_string",
470            ));
471        }
472
473        #[cfg(feature = "charset")]
474        {
475            let (text, _, _) = encoding.decode(&buf);
476            Ok(text.into_owned())
477        }
478        #[cfg(not(feature = "charset"))]
479        {
480            Ok(String::from_utf8_lossy(&buf).to_string())
481        }
482    }
483
484    /// Read the body of this response into a serde_json::Value, or any other type that
485    /// implements the [serde::Deserialize] trait.
486    ///
487    /// You must use either a type annotation as shown below (`message: Message`), or the
488    /// [turbofish operator] (`::<Type>`) so Rust knows what type you are trying to read.
489    ///
490    /// [turbofish operator]: https://matematikaadit.github.io/posts/rust-turbofish.html
491    ///
492    /// Example:
493    ///
494    /// ```
495    /// # fn main() -> Result<(), ureq::Error> {
496    /// # ureq::is_test(true);
497    /// // This import requires the `derive` feature on `serde`.
498    /// // Put this in Cargo.toml: serde = { version = "1", features = ["derive"] }
499    /// use serde::{Deserialize};
500    ///
501    /// #[derive(Deserialize)]
502    /// struct Message {
503    ///     text: String,
504    /// }
505    ///
506    /// let message: Message =
507    ///     ureq::get("http://example.com/get/hello_world.json")
508    ///         .call()?
509    ///         .into_json()?;
510    ///
511    /// assert_eq!(message.text, "Ok");
512    /// # Ok(())
513    /// # }
514    /// ```
515    ///
516    /// Or, if you don't want to define a struct to read your JSON into, you can
517    /// use the convenient `serde_json::Value` type to parse arbitrary or unknown
518    /// JSON.
519    ///
520    /// ```
521    /// # fn main() -> Result<(), ureq::Error> {
522    /// # ureq::is_test(true);
523    /// let json: serde_json::Value = ureq::get("http://example.com/get/hello_world.json")
524    ///     .call()?
525    ///     .into_json()?;
526    ///
527    /// assert_eq!(json["text"], "Ok");
528    /// # Ok(())
529    /// # }
530    /// ```
531    #[cfg(feature = "json")]
532    pub fn into_json<T: DeserializeOwned>(self) -> io::Result<T> {
533        use crate::stream::io_err_timeout;
534
535        let reader = self.into_reader();
536        serde_json::from_reader(reader).map_err(|e| {
537            // This is to unify TimedOut io::Error in the API.
538            if let Some(kind) = e.io_error_kind() {
539                if kind == io::ErrorKind::TimedOut {
540                    return io_err_timeout(e.to_string());
541                }
542            }
543
544            io::Error::new(
545                io::ErrorKind::InvalidData,
546                format!("Failed to read JSON: {}", e),
547            )
548        })
549    }
550
551    /// Create a response from a Read trait impl.
552    ///
553    /// This is hopefully useful for unit tests.
554    ///
555    /// Example:
556    ///
557    /// use std::io::Cursor;
558    ///
559    /// let text = "HTTP/1.1 401 Authorization Required\r\n\r\nPlease log in\n";
560    /// let read = Cursor::new(text.to_string().into_bytes());
561    /// let resp = ureq::Response::do_from_read(read);
562    ///
563    /// assert_eq!(resp.status(), 401);
564    pub(crate) fn do_from_stream(stream: Stream, unit: Unit) -> Result<Response, Error> {
565        let remote_addr = stream.remote_addr;
566
567        let local_addr = match stream.socket() {
568            Some(sock) => sock.local_addr().map_err(Error::from)?,
569            None => std::net::SocketAddrV4::new(std::net::Ipv4Addr::new(127, 0, 0, 1), 0).into(),
570        };
571
572        //
573        // HTTP/1.1 200 OK\r\n
574        let mut stream = stream::DeadlineStream::new(stream, unit.deadline);
575
576        // The status line we can ignore non-utf8 chars and parse as_str_lossy().
577        let status_line = read_next_line(&mut stream, "the status line")?.into_string_lossy();
578        let (index, status) = parse_status_line(status_line.as_str())?;
579        let http_version = &status_line.as_str()[0..index.http_version];
580
581        let mut headers: Vec<Header> = Vec::new();
582        while headers.len() <= MAX_HEADER_COUNT {
583            let line = read_next_line(&mut stream, "a header")?;
584            if line.is_empty() {
585                break;
586            }
587            if let Ok(header) = line.into_header() {
588                headers.push(header);
589            }
590        }
591
592        if headers.len() > MAX_HEADER_COUNT {
593            return Err(ErrorKind::BadHeader.msg(
594                format!("more than {} header fields in response", MAX_HEADER_COUNT).as_str(),
595            ));
596        }
597
598        let compression =
599            get_header(&headers, "content-encoding").and_then(Compression::from_header_value);
600
601        let connection_option =
602            Self::connection_option(http_version, get_header(&headers, "connection"));
603
604        let body_type = Self::body_type(&unit.method, status, http_version, &headers);
605
606        // remove Content-Encoding and length due to automatic decompression
607        if compression.is_some() {
608            headers.retain(|h| !h.is_name("content-encoding") && !h.is_name("content-length"));
609        }
610
611        let reader =
612            Self::stream_to_reader(stream, &unit, body_type, compression, connection_option);
613
614        let url = unit.url.clone();
615
616        let response = Response {
617            url,
618            status_line,
619            index,
620            status,
621            headers,
622            reader,
623            remote_addr,
624            local_addr,
625            history: vec![],
626        };
627        Ok(response)
628    }
629
630    #[cfg(test)]
631    pub fn set_url(&mut self, url: Url) {
632        self.url = url;
633    }
634
635    #[cfg(test)]
636    pub fn history_from_previous(&mut self, previous: Response) {
637        self.history = previous.history;
638        self.history.push(previous.url);
639    }
640}
641
642#[derive(Copy, Clone, Debug, PartialEq, Eq)]
643pub(crate) enum Compression {
644    #[cfg(feature = "brotli")]
645    Brotli,
646    #[cfg(feature = "gzip")]
647    Gzip,
648}
649
650impl Compression {
651    /// Convert a string like "br" to an enum value
652    fn from_header_value(value: &str) -> Option<Compression> {
653        match value {
654            #[cfg(feature = "brotli")]
655            "br" => Some(Compression::Brotli),
656            #[cfg(feature = "gzip")]
657            "gzip" | "x-gzip" => Some(Compression::Gzip),
658            _ => None,
659        }
660    }
661
662    /// Wrap the raw reader with a decompressing reader
663    #[allow(unused_variables)] // when no features enabled, reader is unused (unreachable)
664    pub(crate) fn wrap_reader(
665        self,
666        reader: Box<dyn Read + Send + Sync + 'static>,
667    ) -> Box<dyn Read + Send + Sync + 'static> {
668        match self {
669            #[cfg(feature = "brotli")]
670            Compression::Brotli => Box::new(BrotliDecoder::new(reader, 4096)),
671            #[cfg(feature = "gzip")]
672            Compression::Gzip => Box::new(MultiGzDecoder::new(reader)),
673        }
674    }
675}
676
677/// parse a line like: HTTP/1.1 200 OK\r\n
678fn parse_status_line(line: &str) -> Result<(ResponseStatusIndex, u16), Error> {
679    //
680
681    if !line.is_ascii() {
682        return Err(BadStatus.msg("Status line not ASCII"));
683    }
684    // https://tools.ietf.org/html/rfc7230#section-3.1.2
685    //      status-line = HTTP-version SP status-code SP reason-phrase CRLF
686    let mut split: Vec<&str> = line.splitn(3, ' ').collect();
687    if split.len() == 2 {
688        // As a special case, we are lenient parsing lines without a space after the code.
689        // This is technically against spec. "HTTP/1.1 200\r\n"
690        split.push("");
691    }
692    if split.len() != 3 {
693        return Err(BadStatus.msg("Wrong number of tokens in status line"));
694    }
695
696    // https://tools.ietf.org/html/rfc7230#appendix-B
697    //    HTTP-name = %x48.54.54.50 ; HTTP
698    //    HTTP-version = HTTP-name "/" DIGIT "." DIGIT
699    let http_version = split[0];
700    if !http_version.starts_with("HTTP/") {
701        return Err(BadStatus.msg("HTTP version did not start with HTTP/"));
702    }
703    if http_version.len() != 8 {
704        return Err(BadStatus.msg("HTTP version was wrong length"));
705    }
706    if !http_version.as_bytes()[5].is_ascii_digit() || !http_version.as_bytes()[7].is_ascii_digit()
707    {
708        return Err(BadStatus.msg("HTTP version did not match format"));
709    }
710
711    let status_str: &str = split[1];
712    //      status-code    = 3DIGIT
713    if status_str.len() != 3 {
714        return Err(BadStatus.msg("Status code was wrong length"));
715    }
716
717    let status: u16 = status_str
718        .parse()
719        .map_err(|_| BadStatus.msg(format!("unable to parse status as u16 ({})", status_str)))?;
720
721    Ok((
722        ResponseStatusIndex {
723            http_version: http_version.len(),
724            response_code: http_version.len() + status_str.len(),
725        },
726        status,
727    ))
728}
729
730impl FromStr for Response {
731    type Err = Error;
732    /// Parse a response from a string.
733    ///
734    /// Example:
735    /// ```
736    /// # fn main() -> Result<(), ureq::Error> {
737    /// let s = "HTTP/1.1 200 OK\r\n\
738    ///     X-Forwarded-For: 1.2.3.4\r\n\
739    ///     Content-Type: text/plain\r\n\
740    ///     \r\n\
741    ///     Hello World!!!";
742    /// let resp: ureq::Response = s.parse()?;
743    /// assert!(resp.has("X-Forwarded-For"));
744    /// let body = resp.into_string()?;
745    /// assert_eq!(body, "Hello World!!!");
746    /// # Ok(())
747    /// # }
748    /// ```
749    fn from_str(s: &str) -> Result<Self, Self::Err> {
750        let remote_addr = "0.0.0.0:0".parse().unwrap();
751        let stream = Stream::new(
752            ReadOnlyStream::new(s.into()),
753            remote_addr,
754            PoolReturner::none(),
755        );
756        let request_url = "https://example.com".parse().unwrap();
757        let request_reader = SizedReader {
758            size: crate::body::BodySize::Empty,
759            reader: Box::new(std::io::empty()),
760        };
761        let unit = Unit::new(
762            &Agent::new(),
763            "GET",
764            &request_url,
765            vec![],
766            &request_reader,
767            None,
768        );
769        Self::do_from_stream(stream, unit)
770    }
771}
772
773fn read_next_line(reader: &mut impl BufRead, context: &str) -> io::Result<HeaderLine> {
774    let mut buf = Vec::new();
775    let result = reader
776        .take((MAX_HEADER_SIZE + 1) as u64)
777        .read_until(b'\n', &mut buf);
778
779    match result {
780        Ok(0) => Err(io::Error::new(
781            io::ErrorKind::ConnectionAborted,
782            "Unexpected EOF",
783        )),
784        Ok(n) if n > MAX_HEADER_SIZE => Err(io::Error::new(
785            io::ErrorKind::Other,
786            format!("header field longer than {} bytes", MAX_HEADER_SIZE),
787        )),
788        Ok(_) => Ok(()),
789        Err(e) => {
790            // Provide context to errors encountered while reading the line.
791            let reason = format!("Error encountered in {}", context);
792
793            let kind = e.kind();
794
795            // Use an intermediate wrapper type which carries the error message
796            // as well as a .source() reference to the original error.
797            let wrapper = Error::new(ErrorKind::Io, Some(reason)).src(e);
798
799            Err(io::Error::new(kind, wrapper))
800        }
801    }?;
802
803    if !buf.ends_with(b"\n") {
804        return Err(io::Error::new(
805            io::ErrorKind::InvalidInput,
806            format!("Header field didn't end with \\n: {:?}", buf),
807        ));
808    }
809
810    buf.pop();
811    if buf.ends_with(b"\r") {
812        buf.pop();
813    }
814
815    Ok(buf.into())
816}
817
818/// Limits a `Read` to a content size (as set by a "Content-Length" header).
819pub(crate) struct LimitedRead<R> {
820    reader: Option<R>,
821    limit: usize,
822    position: usize,
823}
824
825impl<R: Read + Sized + Into<Stream>> LimitedRead<R> {
826    pub(crate) fn new(reader: R, limit: NonZeroUsize) -> Self {
827        LimitedRead {
828            reader: Some(reader),
829            limit: limit.get(),
830            position: 0,
831        }
832    }
833
834    pub(crate) fn remaining(&self) -> usize {
835        self.limit - self.position
836    }
837
838    fn return_stream_to_pool(&mut self) -> io::Result<()> {
839        if let Some(reader) = self.reader.take() {
840            // Convert back to a stream. If return_to_pool fails, the stream will
841            // drop and the connection will be closed.
842            let stream: Stream = reader.into();
843            stream.return_to_pool()?;
844        }
845
846        Ok(())
847    }
848}
849
850impl<R: Read + Sized + Into<Stream>> Read for LimitedRead<R> {
851    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
852        if self.remaining() == 0 {
853            return Ok(0);
854        }
855        let from = if self.remaining() < buf.len() {
856            &mut buf[0..self.remaining()]
857        } else {
858            buf
859        };
860        let reader = match self.reader.as_mut() {
861            // If the reader has already been taken, return Ok(0) to all reads.
862            None => return Ok(0),
863            Some(r) => r,
864        };
865        match reader.read(from) {
866            // https://tools.ietf.org/html/rfc7230#page-33
867            // If the sender closes the connection or
868            // the recipient times out before the indicated number of octets are
869            // received, the recipient MUST consider the message to be
870            // incomplete and close the connection.
871            // TODO: actually close the connection by dropping the stream
872            Ok(0) => Err(io::Error::new(
873                io::ErrorKind::UnexpectedEof,
874                "response body closed before all bytes were read",
875            )),
876            Ok(amount) => {
877                self.position += amount;
878                if self.remaining() == 0 {
879                    self.return_stream_to_pool()?;
880                }
881                Ok(amount)
882            }
883            Err(e) => Err(e),
884        }
885    }
886}
887
888/// Extract the charset from a "Content-Type" header.
889///
890/// "Content-Type: text/plain; charset=iso8859-1" -> "iso8859-1"
891///
892/// *Internal API*
893pub(crate) fn charset_from_content_type(header: Option<&str>) -> &str {
894    header
895        .and_then(|header| {
896            header.find(';').and_then(|semi| {
897                header[semi + 1..]
898                    .find('=')
899                    .map(|equal| header[semi + equal + 2..].trim())
900            })
901        })
902        .unwrap_or(DEFAULT_CHARACTER_SET)
903}
904
905// ErrorReader returns an error for every read.
906// The error is as close to a clone of the underlying
907// io::Error as we can get.
908struct ErrorReader(io::Error);
909
910impl Read for ErrorReader {
911    fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
912        Err(io::Error::new(self.0.kind(), self.0.to_string()))
913    }
914}
915
916#[cfg(test)]
917mod tests {
918    use std::io::Cursor;
919
920    use crate::{body::Payload, pool::PoolKey};
921
922    use super::*;
923
924    #[test]
925    fn short_read() {
926        use std::io::Cursor;
927        let test_stream = crate::test::TestStream::new(Cursor::new(vec![b'a'; 3]), std::io::sink());
928        let stream = Stream::new(
929            test_stream,
930            "1.1.1.1:4343".parse().unwrap(),
931            PoolReturner::none(),
932        );
933        let mut lr = LimitedRead::new(stream, std::num::NonZeroUsize::new(10).unwrap());
934        let mut buf = vec![0; 1000];
935        let result = lr.read_to_end(&mut buf);
936        assert!(result.err().unwrap().kind() == io::ErrorKind::UnexpectedEof);
937    }
938
939    #[test]
940    fn content_type_without_charset() {
941        let s = "HTTP/1.1 200 OK\r\n\
942                 Content-Type: application/json\r\n\
943                 \r\n\
944                 OK";
945        let resp = s.parse::<Response>().unwrap();
946        assert_eq!("application/json", resp.content_type());
947    }
948
949    #[test]
950    fn content_type_without_cr() {
951        let s = "HTTP/1.1 200 OK\r\n\
952                 Content-Type: application/json\n\
953                 \r\n\
954                 OK";
955        let resp = s.parse::<Response>().unwrap();
956        assert_eq!("application/json", resp.content_type());
957    }
958
959    #[test]
960    fn content_type_with_charset() {
961        let s = "HTTP/1.1 200 OK\r\n\
962                 Content-Type: application/json; charset=iso-8859-4\r\n\
963                 \r\n\
964                 OK";
965        let resp = s.parse::<Response>().unwrap();
966        assert_eq!("application/json", resp.content_type());
967    }
968
969    #[test]
970    fn content_type_default() {
971        let s = "HTTP/1.1 200 OK\r\n\r\nOK";
972        let resp = s.parse::<Response>().unwrap();
973        assert_eq!("text/plain", resp.content_type());
974    }
975
976    #[test]
977    fn charset() {
978        let s = "HTTP/1.1 200 OK\r\n\
979                 Content-Type: application/json; charset=iso-8859-4\r\n\
980                 \r\n\
981                 OK";
982        let resp = s.parse::<Response>().unwrap();
983        assert_eq!("iso-8859-4", resp.charset());
984    }
985
986    #[test]
987    fn charset_default() {
988        let s = "HTTP/1.1 200 OK\r\n\
989                 Content-Type: application/json\r\n\
990                 \r\n\
991                 OK";
992        let resp = s.parse::<Response>().unwrap();
993        assert_eq!("utf-8", resp.charset());
994    }
995
996    #[test]
997    fn chunked_transfer() {
998        let s = "HTTP/1.1 200 OK\r\n\
999                 Transfer-Encoding: Chunked\r\n\
1000                 \r\n\
1001                 3\r\n\
1002                 hel\r\n\
1003                 b\r\n\
1004                 lo world!!!\r\n\
1005                 0\r\n\
1006                 \r\n";
1007        let resp = s.parse::<Response>().unwrap();
1008        assert_eq!("hello world!!!", resp.into_string().unwrap());
1009    }
1010
1011    #[test]
1012    fn into_string_large() {
1013        const LEN: usize = INTO_STRING_LIMIT + 1;
1014        let s = format!(
1015            "HTTP/1.1 200 OK\r\n\
1016                 Content-Length: {}\r\n
1017                 \r\n
1018                 {}",
1019            LEN,
1020            "A".repeat(LEN),
1021        );
1022        let result = s.parse::<Response>().unwrap();
1023        let err = result
1024            .into_string()
1025            .expect_err("didn't error with too-long body");
1026        assert_eq!(err.to_string(), "response too big for into_string");
1027        assert_eq!(err.kind(), io::ErrorKind::Other);
1028    }
1029
1030    #[test]
1031    #[cfg(feature = "json")]
1032    fn parse_simple_json() {
1033        let s = "HTTP/1.1 200 OK\r\n\
1034             \r\n\
1035             {\"hello\":\"world\"}";
1036        let resp = s.parse::<Response>().unwrap();
1037        let v: serde_json::Value = resp.into_json().unwrap();
1038        let compare = "{\"hello\":\"world\"}"
1039            .parse::<serde_json::Value>()
1040            .unwrap();
1041        assert_eq!(v, compare);
1042    }
1043
1044    #[test]
1045    #[cfg(feature = "json")]
1046    fn parse_deserialize_json() {
1047        use serde::Deserialize;
1048
1049        #[derive(Deserialize)]
1050        struct Hello {
1051            hello: String,
1052        }
1053
1054        let s = "HTTP/1.1 200 OK\r\n\
1055             \r\n\
1056             {\"hello\":\"world\"}";
1057        let resp = s.parse::<Response>().unwrap();
1058        let v: Hello = resp.into_json::<Hello>().unwrap();
1059        assert_eq!(v.hello, "world");
1060    }
1061
1062    #[test]
1063    fn parse_borked_header() {
1064        let s = "HTTP/1.1 BORKED\r\n".to_string();
1065        let err = s.parse::<Response>().unwrap_err();
1066        assert_eq!(err.kind(), BadStatus);
1067    }
1068
1069    #[test]
1070    fn parse_header_without_reason() {
1071        let s = "HTTP/1.1 302\r\n\r\n".to_string();
1072        let resp = s.parse::<Response>().unwrap();
1073        assert_eq!(resp.status_text(), "");
1074    }
1075
1076    #[test]
1077    fn read_next_line_large() {
1078        const LEN: usize = MAX_HEADER_SIZE + 1;
1079        let s = format!("Long-Header: {}\r\n", "A".repeat(LEN),);
1080        let mut cursor = Cursor::new(s);
1081        let result = read_next_line(&mut cursor, "some context");
1082        let err = result.expect_err("did not error on too-large header");
1083        assert_eq!(err.kind(), io::ErrorKind::Other);
1084        assert_eq!(
1085            err.to_string(),
1086            format!("header field longer than {} bytes", MAX_HEADER_SIZE)
1087        );
1088    }
1089
1090    #[test]
1091    fn too_many_headers() {
1092        const LEN: usize = MAX_HEADER_COUNT + 1;
1093        let s = format!(
1094            "HTTP/1.1 200 OK\r\n\
1095                 {}
1096                 \r\n
1097                 hi",
1098            "Header: value\r\n".repeat(LEN),
1099        );
1100        let err = s
1101            .parse::<Response>()
1102            .expect_err("did not error on too many headers");
1103        assert_eq!(err.kind(), ErrorKind::BadHeader);
1104        assert_eq!(
1105            err.to_string(),
1106            format!(
1107                "Bad Header: more than {} header fields in response",
1108                MAX_HEADER_COUNT
1109            )
1110        );
1111    }
1112
1113    #[test]
1114    #[cfg(feature = "charset")]
1115    fn read_next_line_non_ascii_reason() {
1116        let (cow, _, _) =
1117            encoding_rs::WINDOWS_1252.encode("HTTP/1.1 302 Déplacé Temporairement\r\n");
1118        let bytes = cow.to_vec();
1119        let mut reader = io::BufReader::new(io::Cursor::new(bytes));
1120        let r = read_next_line(&mut reader, "test status line");
1121        let h = r.unwrap();
1122        assert_eq!(h.to_string(), "HTTP/1.1 302 D�plac� Temporairement");
1123    }
1124
1125    #[test]
1126    #[cfg(feature = "charset")]
1127    fn parse_header_with_non_utf8() {
1128        let (cow, _, _) = encoding_rs::WINDOWS_1252.encode(
1129            "HTTP/1.1 200 OK\r\n\
1130            x-geo-header: gött mos!\r\n\
1131            \r\n\
1132            OK",
1133        );
1134        let v = cow.to_vec();
1135        let s = Stream::new(
1136            ReadOnlyStream::new(v),
1137            crate::stream::remote_addr_for_test(),
1138            PoolReturner::none(),
1139        );
1140        let request_url = "https://example.com".parse().unwrap();
1141        let request_reader = SizedReader {
1142            size: crate::body::BodySize::Empty,
1143            reader: Box::new(std::io::empty()),
1144        };
1145        let unit = Unit::new(
1146            &Agent::new(),
1147            "GET",
1148            &request_url,
1149            vec![],
1150            &request_reader,
1151            None,
1152        );
1153        let resp = Response::do_from_stream(s.into(), unit).unwrap();
1154        assert_eq!(resp.status(), 200);
1155        assert_eq!(resp.header("x-geo-header"), None);
1156    }
1157
1158    #[test]
1159    fn history() {
1160        let mut response0 = Response::new(302, "Found", "").unwrap();
1161        response0.set_url("http://1.example.com/".parse().unwrap());
1162        assert!(response0.history.is_empty());
1163
1164        let mut response1 = Response::new(302, "Found", "").unwrap();
1165        response1.set_url("http://2.example.com/".parse().unwrap());
1166        response1.history_from_previous(response0);
1167
1168        let mut response2 = Response::new(404, "NotFound", "").unwrap();
1169        response2.set_url("http://2.example.com/".parse().unwrap());
1170        response2.history_from_previous(response1);
1171
1172        let hist: Vec<String> = response2.history.iter().map(|r| r.to_string()).collect();
1173        assert_eq!(hist, ["http://1.example.com/", "http://2.example.com/"])
1174    }
1175
1176    #[test]
1177    fn response_implements_send_and_sync() {
1178        let _response: Box<dyn Send> = Box::new(Response::new(302, "Found", "").unwrap());
1179        let _response: Box<dyn Sync> = Box::new(Response::new(302, "Found", "").unwrap());
1180    }
1181
1182    #[test]
1183    fn ensure_response_size() {
1184        // This is platform dependent, so we can't be too strict or precise.
1185        let size = std::mem::size_of::<Response>();
1186        println!("Response size: {}", size);
1187        assert!(size < 400); // 200 on Macbook M1
1188    }
1189
1190    // Test that a stream gets returned to the pool immediately for a zero-length response, and
1191    // that reads from the response's body consistently return Ok(0).
1192    #[test]
1193    fn zero_length_body_immediate_return() {
1194        use std::io::Cursor;
1195        let response_bytes = "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"
1196            .as_bytes()
1197            .to_vec();
1198        let test_stream =
1199            crate::test::TestStream::new(Cursor::new(response_bytes), std::io::sink());
1200        let agent = Agent::new();
1201        let agent2 = agent.clone();
1202        let stream = Stream::new(
1203            test_stream,
1204            "1.1.1.1:4343".parse().unwrap(),
1205            PoolReturner::new(&agent, PoolKey::from_parts("https", "example.com", 443)),
1206        );
1207        Response::do_from_stream(
1208            stream,
1209            Unit::new(
1210                &agent,
1211                "GET",
1212                &"https://example.com/".parse().unwrap(),
1213                vec![],
1214                &Payload::Empty.into_read(),
1215                None,
1216            ),
1217        )
1218        .unwrap();
1219        assert_eq!(agent2.state.pool.len(), 1);
1220    }
1221
1222    #[test]
1223    #[cfg(feature = "gzip")]
1224    fn gzip_content_length() {
1225        use std::io::Cursor;
1226        let response_bytes =
1227            b"HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: 23\r\n\r\n\
1228\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\xcb\xc8\xe4\x02\x00\x7a\x7a\x6f\xed\x03\x00\x00\x00";
1229        // Follow the response with an infinite stream of 0 bytes, so the content-length
1230        // is important.
1231        let reader = Cursor::new(response_bytes).chain(std::io::repeat(0u8));
1232        let test_stream = crate::test::TestStream::new(reader, std::io::sink());
1233        let agent = Agent::new();
1234        let stream = Stream::new(
1235            test_stream,
1236            "1.1.1.1:4343".parse().unwrap(),
1237            PoolReturner::none(),
1238        );
1239        let resp = Response::do_from_stream(
1240            stream,
1241            Unit::new(
1242                &agent,
1243                "GET",
1244                &"https://example.com/".parse().unwrap(),
1245                vec![],
1246                &Payload::Empty.into_read(),
1247                None,
1248            ),
1249        )
1250        .unwrap();
1251        let body = resp.into_string().unwrap();
1252        assert_eq!(body, "hi\n");
1253    }
1254
1255    #[test]
1256    fn connection_option() {
1257        use ConnectionOption::*;
1258        assert_eq!(Response::connection_option("HTTP/1.0", None), Close);
1259        assert_eq!(Response::connection_option("HtTp/1.0", None), Close);
1260        assert_eq!(Response::connection_option("HTTP/1.0", Some("blah")), Close);
1261        assert_eq!(
1262            Response::connection_option("HTTP/1.0", Some("keep-ALIVE")),
1263            KeepAlive
1264        );
1265        assert_eq!(
1266            Response::connection_option("http/1.0", Some("keep-alive")),
1267            KeepAlive
1268        );
1269
1270        assert_eq!(Response::connection_option("http/1.1", None), KeepAlive);
1271        assert_eq!(
1272            Response::connection_option("http/1.1", Some("blah")),
1273            KeepAlive
1274        );
1275        assert_eq!(
1276            Response::connection_option("http/1.1", Some("keep-alive")),
1277            KeepAlive
1278        );
1279        assert_eq!(
1280            Response::connection_option("http/1.1", Some("CLOSE")),
1281            Close
1282        );
1283    }
1284
1285    #[test]
1286    fn body_type() {
1287        use BodyType::*;
1288        assert_eq!(
1289            Response::body_type("GET", 200, "HTTP/1.1", &[]),
1290            CloseDelimited
1291        );
1292        assert_eq!(
1293            Response::body_type("HEAD", 200, "HTTP/1.1", &[]),
1294            LengthDelimited(0)
1295        );
1296        assert_eq!(
1297            Response::body_type("hEaD", 200, "HTTP/1.1", &[]),
1298            LengthDelimited(0)
1299        );
1300        assert_eq!(
1301            Response::body_type("head", 200, "HTTP/1.1", &[]),
1302            LengthDelimited(0)
1303        );
1304        assert_eq!(
1305            Response::body_type("GET", 304, "HTTP/1.1", &[]),
1306            LengthDelimited(0)
1307        );
1308        assert_eq!(
1309            Response::body_type("GET", 204, "HTTP/1.1", &[]),
1310            LengthDelimited(0)
1311        );
1312        assert_eq!(
1313            Response::body_type(
1314                "GET",
1315                200,
1316                "HTTP/1.1",
1317                &[Header::new("Transfer-Encoding", "chunked"),]
1318            ),
1319            Chunked
1320        );
1321        assert_eq!(
1322            Response::body_type(
1323                "GET",
1324                200,
1325                "HTTP/1.1",
1326                &[Header::new("Content-Length", "123"),]
1327            ),
1328            LengthDelimited(123)
1329        );
1330        assert_eq!(
1331            Response::body_type(
1332                "GET",
1333                200,
1334                "HTTP/1.1",
1335                &[
1336                    Header::new("Content-Length", "123"),
1337                    Header::new("Transfer-Encoding", "chunked"),
1338                ]
1339            ),
1340            Chunked
1341        );
1342        assert_eq!(
1343            Response::body_type(
1344                "GET",
1345                200,
1346                "HTTP/1.1",
1347                &[
1348                    Header::new("Transfer-Encoding", "chunked"),
1349                    Header::new("Content-Length", "123"),
1350                ]
1351            ),
1352            Chunked
1353        );
1354        assert_eq!(
1355            Response::body_type(
1356                "HEAD",
1357                200,
1358                "HTTP/1.1",
1359                &[
1360                    Header::new("Transfer-Encoding", "chunked"),
1361                    Header::new("Content-Length", "123"),
1362                ]
1363            ),
1364            LengthDelimited(0)
1365        );
1366        assert_eq!(
1367            Response::body_type(
1368                "GET",
1369                200,
1370                "HTTP/1.0",
1371                &[Header::new("Transfer-Encoding", "chunked"),]
1372            ),
1373            CloseDelimited,
1374            "HTTP/1.0 did not support chunked encoding"
1375        );
1376        assert_eq!(
1377            Response::body_type(
1378                "GET",
1379                200,
1380                "HTTP/1.0",
1381                &[Header::new("Content-Length", "123"),]
1382            ),
1383            LengthDelimited(123)
1384        );
1385    }
1386}