encoding_rs/
lib.rs

1// Copyright Mozilla Foundation. See the COPYRIGHT
2// file at the top-level directory of this distribution.
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10// The above license applies to code in this file. The label data in
11// this file is generated from WHATWG's encodings.json, which came under
12// the following license:
13
14// Copyright © WHATWG (Apple, Google, Mozilla, Microsoft).
15//
16// Redistribution and use in source and binary forms, with or without
17// modification, are permitted provided that the following conditions are met:
18//
19// 1. Redistributions of source code must retain the above copyright notice, this
20//    list of conditions and the following disclaimer.
21//
22// 2. Redistributions in binary form must reproduce the above copyright notice,
23//    this list of conditions and the following disclaimer in the documentation
24//    and/or other materials provided with the distribution.
25//
26// 3. Neither the name of the copyright holder nor the names of its
27//    contributors may be used to endorse or promote products derived from
28//    this software without specific prior written permission.
29//
30// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
31// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
33// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
34// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
35// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
36// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
37// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
38// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40
41#![cfg_attr(
42    feature = "cargo-clippy",
43    allow(doc_markdown, inline_always, new_ret_no_self)
44)]
45
46//! encoding_rs is a Gecko-oriented Free Software / Open Source implementation
47//! of the [Encoding Standard](https://encoding.spec.whatwg.org/) in Rust.
48//! Gecko-oriented means that converting to and from UTF-16 is supported in
49//! addition to converting to and from UTF-8, that the performance and
50//! streamability goals are browser-oriented, and that FFI-friendliness is a
51//! goal.
52//!
53//! Additionally, the `mem` module provides functions that are useful for
54//! applications that need to be able to deal with legacy in-memory
55//! representations of Unicode.
56//!
57//! For expectation setting, please be sure to read the sections
58//! [_UTF-16LE, UTF-16BE and Unicode Encoding Schemes_](#utf-16le-utf-16be-and-unicode-encoding-schemes),
59//! [_ISO-8859-1_](#iso-8859-1) and [_Web / Browser Focus_](#web--browser-focus) below.
60//!
61//! There is a [long-form write-up](https://hsivonen.fi/encoding_rs/) about the
62//! design and internals of the crate.
63//!
64//! # Availability
65//!
66//! The code is available under the
67//! [Apache license, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
68//! or the [MIT license](https://opensource.org/licenses/MIT), at your option.
69//! See the
70//! [`COPYRIGHT`](https://github.com/hsivonen/encoding_rs/blob/master/COPYRIGHT)
71//! file for details.
72//! The [repository is on GitHub](https://github.com/hsivonen/encoding_rs). The
73//! [crate is available on crates.io](https://crates.io/crates/encoding_rs).
74//!
75//! # Integration with `std::io`
76//!
77//! This crate doesn't implement traits from `std::io`. However, for the case of
78//! wrapping a `std::io::Read` in a decoder that implements `std::io::Read` and
79//! presents the data from the wrapped `std::io::Read` as UTF-8 is addressed by
80//! the [`encoding_rs_io`](https://docs.rs/encoding_rs_io/) crate.
81//!
82//! # Examples
83//!
84//! Example programs:
85//!
86//! * [Rust](https://github.com/hsivonen/recode_rs)
87//! * [C](https://github.com/hsivonen/recode_c)
88//! * [C++](https://github.com/hsivonen/recode_cpp)
89//!
90//! Decode using the non-streaming API:
91//!
92//! ```
93//! #[cfg(feature = "alloc")] {
94//! use encoding_rs::*;
95//!
96//! let expectation = "\u{30CF}\u{30ED}\u{30FC}\u{30FB}\u{30EF}\u{30FC}\u{30EB}\u{30C9}";
97//! let bytes = b"\x83n\x83\x8D\x81[\x81E\x83\x8F\x81[\x83\x8B\x83h";
98//!
99//! let (cow, encoding_used, had_errors) = SHIFT_JIS.decode(bytes);
100//! assert_eq!(&cow[..], expectation);
101//! assert_eq!(encoding_used, SHIFT_JIS);
102//! assert!(!had_errors);
103//! }
104//! ```
105//!
106//! Decode using the streaming API with minimal `unsafe`:
107//!
108//! ```
109//! use encoding_rs::*;
110//!
111//! let expectation = "\u{30CF}\u{30ED}\u{30FC}\u{30FB}\u{30EF}\u{30FC}\u{30EB}\u{30C9}";
112//!
113//! // Use an array of byte slices to demonstrate content arriving piece by
114//! // piece from the network.
115//! let bytes: [&'static [u8]; 4] = [b"\x83",
116//!                                  b"n\x83\x8D\x81",
117//!                                  b"[\x81E\x83\x8F\x81[\x83",
118//!                                  b"\x8B\x83h"];
119//!
120//! // Very short output buffer to demonstrate the output buffer getting full.
121//! // Normally, you'd use something like `[0u8; 2048]`.
122//! let mut buffer_bytes = [0u8; 8];
123//! let mut buffer: &mut str = std::str::from_utf8_mut(&mut buffer_bytes[..]).unwrap();
124//!
125//! // How many bytes in the buffer currently hold significant data.
126//! let mut bytes_in_buffer = 0usize;
127//!
128//! // Collect the output to a string for demonstration purposes.
129//! let mut output = String::new();
130//!
131//! // The `Decoder`
132//! let mut decoder = SHIFT_JIS.new_decoder();
133//!
134//! // Track whether we see errors.
135//! let mut total_had_errors = false;
136//!
137//! // Decode using a fixed-size intermediate buffer (for demonstrating the
138//! // use of a fixed-size buffer; normally when the output of an incremental
139//! // decode goes to a `String` one would use `Decoder.decode_to_string()` to
140//! // avoid the intermediate buffer).
141//! for input in &bytes[..] {
142//!     // The number of bytes already read from current `input` in total.
143//!     let mut total_read_from_current_input = 0usize;
144//!
145//!     loop {
146//!         let (result, read, written, had_errors) =
147//!             decoder.decode_to_str(&input[total_read_from_current_input..],
148//!                                   &mut buffer[bytes_in_buffer..],
149//!                                   false);
150//!         total_read_from_current_input += read;
151//!         bytes_in_buffer += written;
152//!         total_had_errors |= had_errors;
153//!         match result {
154//!             CoderResult::InputEmpty => {
155//!                 // We have consumed the current input buffer. Break out of
156//!                 // the inner loop to get the next input buffer from the
157//!                 // outer loop.
158//!                 break;
159//!             },
160//!             CoderResult::OutputFull => {
161//!                 // Write the current buffer out and consider the buffer
162//!                 // empty.
163//!                 output.push_str(&buffer[..bytes_in_buffer]);
164//!                 bytes_in_buffer = 0usize;
165//!                 continue;
166//!             }
167//!         }
168//!     }
169//! }
170//!
171//! // Process EOF
172//! loop {
173//!     let (result, _, written, had_errors) =
174//!         decoder.decode_to_str(b"",
175//!                               &mut buffer[bytes_in_buffer..],
176//!                               true);
177//!     bytes_in_buffer += written;
178//!     total_had_errors |= had_errors;
179//!     // Write the current buffer out and consider the buffer empty.
180//!     // Need to do this here for both `match` arms, because we exit the
181//!     // loop on `CoderResult::InputEmpty`.
182//!     output.push_str(&buffer[..bytes_in_buffer]);
183//!     bytes_in_buffer = 0usize;
184//!     match result {
185//!         CoderResult::InputEmpty => {
186//!             // Done!
187//!             break;
188//!         },
189//!         CoderResult::OutputFull => {
190//!             continue;
191//!         }
192//!     }
193//! }
194//!
195//! assert_eq!(&output[..], expectation);
196//! assert!(!total_had_errors);
197//! ```
198//!
199//! ## UTF-16LE, UTF-16BE and Unicode Encoding Schemes
200//!
201//! The Encoding Standard doesn't specify encoders for UTF-16LE and UTF-16BE,
202//! __so this crate does not provide encoders for those encodings__!
203//! Along with the replacement encoding, their _output encoding_ (i.e. the
204//! encoding used for form submission and error handling in the query string
205//! of URLs) is UTF-8, so you get an UTF-8 encoder if you request an encoder
206//! for them.
207//!
208//! Additionally, the Encoding Standard factors BOM handling into wrapper
209//! algorithms so that BOM handling isn't part of the definition of the
210//! encodings themselves. The Unicode _encoding schemes_ in the Unicode
211//! Standard define BOM handling or lack thereof as part of the encoding
212//! scheme.
213//!
214//! When used with the `_without_bom_handling` entry points, the UTF-16LE
215//! and UTF-16BE _encodings_ match the same-named _encoding schemes_ from
216//! the Unicode Standard.
217//!
218//! When used with the `_with_bom_removal` entry points, the UTF-8
219//! _encoding_ matches the UTF-8 _encoding scheme_ from the Unicode
220//! Standard.
221//!
222//! This crate does not provide a mode that matches the UTF-16 _encoding
223//! scheme_ from the Unicode Stardard. The UTF-16BE encoding used with
224//! the entry points without `_bom_` qualifiers is the closest match,
225//! but in that case, the UTF-8 BOM triggers UTF-8 decoding, which is
226//! not part of the behavior of the UTF-16 _encoding scheme_ per the
227//! Unicode Standard.
228//!
229//! The UTF-32 family of Unicode encoding schemes is not supported
230//! by this crate. The Encoding Standard doesn't define any UTF-32
231//! family encodings, since they aren't necessary for consuming Web
232//! content.
233//!
234//! While gb18030 is capable of representing U+FEFF, the Encoding
235//! Standard does not treat the gb18030 byte representation of U+FEFF
236//! as a BOM, so neither does this crate.
237//!
238//! ## ISO-8859-1
239//!
240//! ISO-8859-1 does not exist as a distinct encoding from windows-1252 in
241//! the Encoding Standard. Therefore, an encoding that maps the unsigned
242//! byte value to the same Unicode scalar value is not available via
243//! `Encoding` in this crate.
244//!
245//! However, the functions whose name starts with `convert` and contains
246//! `latin1` in the `mem` module support such conversions, which are known as
247//! [_isomorphic decode_](https://infra.spec.whatwg.org/#isomorphic-decode)
248//! and [_isomorphic encode_](https://infra.spec.whatwg.org/#isomorphic-encode)
249//! in the [Infra Standard](https://infra.spec.whatwg.org/).
250//!
251//! ## Web / Browser Focus
252//!
253//! Both in terms of scope and performance, the focus is on the Web. For scope,
254//! this means that encoding_rs implements the Encoding Standard fully and
255//! doesn't implement encodings that are not specified in the Encoding
256//! Standard. For performance, this means that decoding performance is
257//! important as well as performance for encoding into UTF-8 or encoding the
258//! Basic Latin range (ASCII) into legacy encodings. Non-Basic Latin needs to
259//! be encoded into legacy encodings in only two places in the Web platform: in
260//! the query part of URLs, in which case it's a matter of relatively rare
261//! error handling, and in form submission, in which case the user action and
262//! networking tend to hide the performance of the encoder.
263//!
264//! Deemphasizing performance of encoding non-Basic Latin text into legacy
265//! encodings enables smaller code size thanks to the encoder side using the
266//! decode-optimized data tables without having encode-optimized data tables at
267//! all. Even in decoders, smaller lookup table size is preferred over avoiding
268//! multiplication operations.
269//!
270//! Additionally, performance is a non-goal for the ASCII-incompatible
271//! ISO-2022-JP encoding, which are rarely used on the Web. Instead of
272//! performance, the decoder for ISO-2022-JP optimizes for ease/clarity
273//! of implementation.
274//!
275//! Despite the browser focus, the hope is that non-browser applications
276//! that wish to consume Web content or submit Web forms in a Web-compatible
277//! way will find encoding_rs useful. While encoding_rs does not try to match
278//! Windows behavior, many of the encodings are close enough to legacy
279//! encodings implemented by Windows that applications that need to consume
280//! data in legacy Windows encodins may find encoding_rs useful. The
281//! [codepage](https://crates.io/crates/codepage) crate maps from Windows
282//! code page identifiers onto encoding_rs `Encoding`s and vice versa.
283//!
284//! For decoding email, UTF-7 support is needed (unfortunately) in additition
285//! to the encodings defined in the Encoding Standard. The
286//! [charset](https://crates.io/crates/charset) wraps encoding_rs and adds
287//! UTF-7 decoding for email purposes.
288//!
289//! For single-byte DOS encodings beyond the ones supported by the Encoding
290//! Standard, there is the [`oem_cp`](https://crates.io/crates/oem_cp) crate.
291//!
292//! # Preparing Text for the Encoders
293//!
294//! Normalizing text into Unicode Normalization Form C prior to encoding text
295//! into a legacy encoding minimizes unmappable characters. Text can be
296//! normalized to Unicode Normalization Form C using the
297//! [`icu_normalizer`](https://crates.io/crates/icu_normalizer) crate, which
298//! is part of [ICU4X](https://icu4x.unicode.org/).
299//!
300//! The exception is windows-1258, which after normalizing to Unicode
301//! Normalization Form C requires tone marks to be decomposed in order to
302//! minimize unmappable characters. Vietnamese tone marks can be decomposed
303//! using the [`detone`](https://crates.io/crates/detone) crate.
304//!
305//! # Streaming & Non-Streaming; Rust & C/C++
306//!
307//! The API in Rust has two modes of operation: streaming and non-streaming.
308//! The streaming API is the foundation of the implementation and should be
309//! used when processing data that arrives piecemeal from an i/o stream. The
310//! streaming API has an FFI wrapper (as a [separate crate][1]) that exposes it
311//! to C callers. The non-streaming part of the API is for Rust callers only and
312//! is smart about borrowing instead of copying when possible. When
313//! streamability is not needed, the non-streaming API should be preferrer in
314//! order to avoid copying data when a borrow suffices.
315//!
316//! There is no analogous C API exposed via FFI, mainly because C doesn't have
317//! standard types for growable byte buffers and Unicode strings that know
318//! their length.
319//!
320//! The C API (header file generated at `target/include/encoding_rs.h` when
321//! building encoding_rs) can, in turn, be wrapped for use from C++. Such a
322//! C++ wrapper can re-create the non-streaming API in C++ for C++ callers.
323//! The C binding comes with a [C++17 wrapper][2] that uses standard library +
324//! [GSL][3] types and that recreates the non-streaming API in C++ on top of
325//! the streaming API. A C++ wrapper with XPCOM/MFBT types is available as
326//! [`mozilla::Encoding`][4].
327//!
328//! The `Encoding` type is common to both the streaming and non-streaming
329//! modes. In the streaming mode, decoding operations are performed with a
330//! `Decoder` and encoding operations with an `Encoder` object obtained via
331//! `Encoding`. In the non-streaming mode, decoding and encoding operations are
332//! performed using methods on `Encoding` objects themselves, so the `Decoder`
333//! and `Encoder` objects are not used at all.
334//!
335//! [1]: https://github.com/hsivonen/encoding_c
336//! [2]: https://github.com/hsivonen/encoding_c/blob/master/include/encoding_rs_cpp.h
337//! [3]: https://github.com/Microsoft/GSL/
338//! [4]: https://searchfox.org/mozilla-central/source/intl/Encoding.h
339//!
340//! # Memory management
341//!
342//! The non-streaming mode never performs heap allocations (even the methods
343//! that write into a `Vec<u8>` or a `String` by taking them as arguments do
344//! not reallocate the backing buffer of the `Vec<u8>` or the `String`). That
345//! is, the non-streaming mode uses caller-allocated buffers exclusively.
346//!
347//! The methods of the streaming mode that return a `Vec<u8>` or a `String`
348//! perform heap allocations but only to allocate the backing buffer of the
349//! `Vec<u8>` or the `String`.
350//!
351//! `Encoding` is always statically allocated. `Decoder` and `Encoder` need no
352//! `Drop` cleanup.
353//!
354//! # Buffer reading and writing behavior
355//!
356//! Based on experience gained with the `java.nio.charset` encoding converter
357//! API and with the Gecko uconv encoding converter API, the buffer reading
358//! and writing behaviors of encoding_rs are asymmetric: input buffers are
359//! fully drained but output buffers are not always fully filled.
360//!
361//! When reading from an input buffer, encoding_rs always consumes all input
362//! up to the next error or to the end of the buffer. In particular, when
363//! decoding, even if the input buffer ends in the middle of a byte sequence
364//! for a character, the decoder consumes all input. This has the benefit that
365//! the caller of the API can always fill the next buffer from the start from
366//! whatever source the bytes come from and never has to first copy the last
367//! bytes of the previous buffer to the start of the next buffer. However, when
368//! encoding, the UTF-8 input buffers have to end at a character boundary, which
369//! is a requirement for the Rust `str` type anyway, and UTF-16 input buffer
370//! boundaries falling in the middle of a surrogate pair result in both
371//! suggorates being treated individually as unpaired surrogates.
372//!
373//! Additionally, decoders guarantee that they can be fed even one byte at a
374//! time and encoders guarantee that they can be fed even one code point at a
375//! time. This has the benefit of not placing restrictions on the size of
376//! chunks the content arrives e.g. from network.
377//!
378//! When writing into an output buffer, encoding_rs makes sure that the code
379//! unit sequence for a character is never split across output buffer
380//! boundaries. This may result in wasted space at the end of an output buffer,
381//! but the advantages are that the output side of both decoders and encoders
382//! is greatly simplified compared to designs that attempt to fill output
383//! buffers exactly even when that entails splitting a code unit sequence and
384//! when encoding_rs methods return to the caller, the output produces thus
385//! far is always valid taken as whole. (In the case of encoding to ISO-2022-JP,
386//! the output needs to be considered as a whole, because the latest output
387//! buffer taken alone might not be valid taken alone if the transition away
388//! from the ASCII state occurred in an earlier output buffer. However, since
389//! the ISO-2022-JP decoder doesn't treat streams that don't end in the ASCII
390//! state as being in error despite the encoder generating a transition to the
391//! ASCII state at the end, the claim about the partial output taken as a whole
392//! being valid is true even for ISO-2022-JP.)
393//!
394//! # Error Reporting
395//!
396//! Based on experience gained with the `java.nio.charset` encoding converter
397//! API and with the Gecko uconv encoding converter API, the error reporting
398//! behaviors of encoding_rs are asymmetric: decoder errors include offsets
399//! that leave it up to the caller to extract the erroneous bytes from the
400//! input stream if the caller wishes to do so but encoder errors provide the
401//! code point associated with the error without requiring the caller to
402//! extract it from the input on its own.
403//!
404//! On the encoder side, an error is always triggered by the most recently
405//! pushed Unicode scalar, which makes it simple to pass the `char` to the
406//! caller. Also, it's very typical for the caller to wish to do something with
407//! this data: generate a numeric escape for the character. Additionally, the
408//! ISO-2022-JP encoder reports U+FFFD instead of the actual input character in
409//! certain cases, so requiring the caller to extract the character from the
410//! input buffer would require the caller to handle ISO-2022-JP details.
411//! Furthermore, requiring the caller to extract the character from the input
412//! buffer would require the caller to implement UTF-8 or UTF-16 math, which is
413//! the job of an encoding conversion library.
414//!
415//! On the decoder side, errors are triggered in more complex ways. For
416//! example, when decoding the sequence ESC, '$', _buffer boundary_, 'A' as
417//! ISO-2022-JP, the ESC byte is in error, but this is discovered only after
418//! the buffer boundary when processing 'A'. Thus, the bytes in error might not
419//! be the ones most recently pushed to the decoder and the error might not even
420//! be in the current buffer.
421//!
422//! Some encoding conversion APIs address the problem by not acknowledging
423//! trailing bytes of an input buffer as consumed if it's still possible for
424//! future bytes to cause the trailing bytes to be in error. This way, error
425//! reporting can always refer to the most recently pushed buffer. This has the
426//! problem that the caller of the API has to copy the unconsumed trailing
427//! bytes to the start of the next buffer before being able to fill the rest
428//! of the next buffer. This is annoying, error-prone and inefficient.
429//!
430//! A possible solution would be making the decoder remember recently consumed
431//! bytes in order to be able to include a copy of the erroneous bytes when
432//! reporting an error. This has two problem: First, callers a rarely
433//! interested in the erroneous bytes, so attempts to identify them are most
434//! often just overhead anyway. Second, the rare applications that are
435//! interested typically care about the location of the error in the input
436//! stream.
437//!
438//! To keep the API convenient for common uses and the overhead low while making
439//! it possible to develop applications, such as HTML validators, that care
440//! about which bytes were in error, encoding_rs reports the length of the
441//! erroneous sequence and the number of bytes consumed after the erroneous
442//! sequence. As long as the caller doesn't discard the 6 most recent bytes,
443//! this makes it possible for callers that care about the erroneous bytes to
444//! locate them.
445//!
446//! # No Convenience API for Custom Replacements
447//!
448//! The Web Platform and, therefore, the Encoding Standard supports only one
449//! error recovery mode for decoders and only one error recovery mode for
450//! encoders. The supported error recovery mode for decoders is emitting the
451//! REPLACEMENT CHARACTER on error. The supported error recovery mode for
452//! encoders is emitting an HTML decimal numeric character reference for
453//! unmappable characters.
454//!
455//! Since encoding_rs is Web-focused, these are the only error recovery modes
456//! for which convenient support is provided. Moreover, on the decoder side,
457//! there aren't really good alternatives for emitting the REPLACEMENT CHARACTER
458//! on error (other than treating errors as fatal). In particular, simply
459//! ignoring errors is a
460//! [security problem](http://www.unicode.org/reports/tr36/#Substituting_for_Ill_Formed_Subsequences),
461//! so it would be a bad idea for encoding_rs to provide a mode that encouraged
462//! callers to ignore errors.
463//!
464//! On the encoder side, there are plausible alternatives for HTML decimal
465//! numeric character references. For example, when outputting CSS, CSS-style
466//! escapes would seem to make sense. However, instead of facilitating the
467//! output of CSS, JS, etc. in non-UTF-8 encodings, encoding_rs takes the design
468//! position that you shouldn't generate output in encodings other than UTF-8,
469//! except where backward compatibility with interacting with the legacy Web
470//! requires it. The legacy Web requires it only when parsing the query strings
471//! of URLs and when submitting forms, and those two both use HTML decimal
472//! numeric character references.
473//!
474//! While encoding_rs doesn't make encoder replacements other than HTML decimal
475//! numeric character references easy, it does make them _possible_.
476//! `encode_from_utf8()`, which emits HTML decimal numeric character references
477//! for unmappable characters, is implemented on top of
478//! `encode_from_utf8_without_replacement()`. Applications that really, really
479//! want other replacement schemes for unmappable characters can likewise
480//! implement them on top of `encode_from_utf8_without_replacement()`.
481//!
482//! # No Extensibility by Design
483//!
484//! The set of encodings supported by encoding_rs is not extensible by design.
485//! That is, `Encoding`, `Decoder` and `Encoder` are intentionally `struct`s
486//! rather than `trait`s. encoding_rs takes the design position that all future
487//! text interchange should be done using UTF-8, which can represent all of
488//! Unicode. (It is, in fact, the only encoding supported by the Encoding
489//! Standard and encoding_rs that can represent all of Unicode and that has
490//! encoder support. UTF-16LE and UTF-16BE don't have encoder support, and
491//! gb18030 cannot encode U+E5E5.) The other encodings are supported merely for
492//! legacy compatibility and not due to non-UTF-8 encodings having benefits
493//! other than being able to consume legacy content.
494//!
495//! Considering that UTF-8 can represent all of Unicode and is already supported
496//! by all Web browsers, introducing a new encoding wouldn't add to the
497//! expressiveness but would add to compatibility problems. In that sense,
498//! adding new encodings to the Web Platform doesn't make sense, and, in fact,
499//! post-UTF-8 attempts at encodings, such as BOCU-1, have been rejected from
500//! the Web Platform. On the other hand, the set of legacy encodings that must
501//! be supported for a Web browser to be able to be successful is not going to
502//! expand. Empirically, the set of encodings specified in the Encoding Standard
503//! is already sufficient and the set of legacy encodings won't grow
504//! retroactively.
505//!
506//! Since extensibility doesn't make sense considering the Web focus of
507//! encoding_rs and adding encodings to Web clients would be actively harmful,
508//! it makes sense to make the set of encodings that encoding_rs supports
509//! non-extensible and to take the (admittedly small) benefits arising from
510//! that, such as the size of `Decoder` and `Encoder` objects being known ahead
511//!  of time, which enables stack allocation thereof.
512//!
513//! This does have downsides for applications that might want to put encoding_rs
514//! to non-Web uses if those non-Web uses involve legacy encodings that aren't
515//! needed for Web uses. The needs of such applications should not complicate
516//! encoding_rs itself, though. It is up to those applications to provide a
517//! framework that delegates the operations with encodings that encoding_rs
518//! supports to encoding_rs and operations with other encodings to something
519//! else (as opposed to encoding_rs itself providing an extensibility
520//! framework).
521//!
522//! # Panics
523//!
524//! Methods in encoding_rs can panic if the API is used against the requirements
525//! stated in the documentation, if a state that's supposed to be impossible
526//! is reached due to an internal bug or on integer overflow. When used
527//! according to documentation with buffer sizes that stay below integer
528//! overflow, in the absence of internal bugs, encoding_rs does not panic.
529//!
530//! Panics arising from API misuse aren't documented beyond this on individual
531//! methods.
532//!
533//! # At-Risk Parts of the API
534//!
535//! The foreseeable source of partially backward-incompatible API change is the
536//! way the instances of `Encoding` are made available.
537//!
538//! If Rust changes to allow the entries of `[&'static Encoding; N]` to be
539//! initialized with `static`s of type `&'static Encoding`, the non-reference
540//! `FOO_INIT` public `Encoding` instances will be removed from the public API.
541//!
542//! If Rust changes to make the referent of `pub const FOO: &'static Encoding`
543//! unique when the constant is used in different crates, the reference-typed
544//! `static`s for the encoding instances will be changed from `static` to
545//! `const` and the non-reference-typed `_INIT` instances will be removed.
546//!
547//! # Mapping Spec Concepts onto the API
548//!
549//! <table>
550//! <thead>
551//! <tr><th>Spec Concept</th><th>Streaming</th><th>Non-Streaming</th></tr>
552//! </thead>
553//! <tbody>
554//! <tr><td><a href="https://encoding.spec.whatwg.org/#encoding">encoding</a></td><td><code>&amp;'static Encoding</code></td><td><code>&amp;'static Encoding</code></td></tr>
555//! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8">UTF-8 encoding</a></td><td><code>UTF_8</code></td><td><code>UTF_8</code></td></tr>
556//! <tr><td><a href="https://encoding.spec.whatwg.org/#concept-encoding-get">get an encoding</a></td><td><code>Encoding::for_label(<var>label</var>)</code></td><td><code>Encoding::for_label(<var>label</var>)</code></td></tr>
557//! <tr><td><a href="https://encoding.spec.whatwg.org/#name">name</a></td><td><code><var>encoding</var>.name()</code></td><td><code><var>encoding</var>.name()</code></td></tr>
558//! <tr><td><a href="https://encoding.spec.whatwg.org/#get-an-output-encoding">get an output encoding</a></td><td><code><var>encoding</var>.output_encoding()</code></td><td><code><var>encoding</var>.output_encoding()</code></td></tr>
559//! <tr><td><a href="https://encoding.spec.whatwg.org/#decode">decode</a></td><td><code>let d = <var>encoding</var>.new_decoder();<br>let res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, false);<br>// &hellip;</br>let last_res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, true);</code></td><td><code><var>encoding</var>.decode(<var>src</var>)</code></td></tr>
560//! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8-decode">UTF-8 decode</a></td><td><code>let d = UTF_8.new_decoder_with_bom_removal();<br>let res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, false);<br>// &hellip;</br>let last_res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, true);</code></td><td><code>UTF_8.decode_with_bom_removal(<var>src</var>)</code></td></tr>
561//! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8-decode-without-bom">UTF-8 decode without BOM</a></td><td><code>let d = UTF_8.new_decoder_without_bom_handling();<br>let res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, false);<br>// &hellip;</br>let last_res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, true);</code></td><td><code>UTF_8.decode_without_bom_handling(<var>src</var>)</code></td></tr>
562//! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8-decode-without-bom-or-fail">UTF-8 decode without BOM or fail</a></td><td><code>let d = UTF_8.new_decoder_without_bom_handling();<br>let res = d.decode_to_<var>*</var>_without_replacement(<var>src</var>, <var>dst</var>, false);<br>// &hellip; (fail if malformed)</br>let last_res = d.decode_to_<var>*</var>_without_replacement(<var>src</var>, <var>dst</var>, true);<br>// (fail if malformed)</code></td><td><code>UTF_8.decode_without_bom_handling_and_without_replacement(<var>src</var>)</code></td></tr>
563//! <tr><td><a href="https://encoding.spec.whatwg.org/#encode">encode</a></td><td><code>let e = <var>encoding</var>.new_encoder();<br>let res = e.encode_to_<var>*</var>(<var>src</var>, <var>dst</var>, false);<br>// &hellip;</br>let last_res = e.encode_to_<var>*</var>(<var>src</var>, <var>dst</var>, true);</code></td><td><code><var>encoding</var>.encode(<var>src</var>)</code></td></tr>
564//! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8-encode">UTF-8 encode</a></td><td>Use the UTF-8 nature of Rust strings directly:<br><code><var>write</var>(<var>src</var>.as_bytes());<br>// refill src<br><var>write</var>(<var>src</var>.as_bytes());<br>// refill src<br><var>write</var>(<var>src</var>.as_bytes());<br>// &hellip;</code></td><td>Use the UTF-8 nature of Rust strings directly:<br><code><var>src</var>.as_bytes()</code></td></tr>
565//! </tbody>
566//! </table>
567//!
568//! # Compatibility with the rust-encoding API
569//!
570//! The crate
571//! [encoding_rs_compat](https://github.com/hsivonen/encoding_rs_compat/)
572//! is a drop-in replacement for rust-encoding 0.2.32 that implements (most of)
573//! the API of rust-encoding 0.2.32 on top of encoding_rs.
574//!
575//! # Mapping rust-encoding concepts to encoding_rs concepts
576//!
577//! The following table provides a mapping from rust-encoding constructs to
578//! encoding_rs ones.
579//!
580//! <table>
581//! <thead>
582//! <tr><th>rust-encoding</th><th>encoding_rs</th></tr>
583//! </thead>
584//! <tbody>
585//! <tr><td><code>encoding::EncodingRef</code></td><td><code>&amp;'static encoding_rs::Encoding</code></td></tr>
586//! <tr><td><code>encoding::all::<var>WINDOWS_31J</var></code> (not based on the WHATWG name for some encodings)</td><td><code>encoding_rs::<var>SHIFT_JIS</var></code> (always the WHATWG name uppercased and hyphens replaced with underscores)</td></tr>
587//! <tr><td><code>encoding::all::ERROR</code></td><td>Not available because not in the Encoding Standard</td></tr>
588//! <tr><td><code>encoding::all::ASCII</code></td><td>Not available because not in the Encoding Standard</td></tr>
589//! <tr><td><code>encoding::all::ISO_8859_1</code></td><td>Not available because not in the Encoding Standard</td></tr>
590//! <tr><td><code>encoding::all::HZ</code></td><td>Not available because not in the Encoding Standard</td></tr>
591//! <tr><td><code>encoding::label::encoding_from_whatwg_label(<var>string</var>)</code></td><td><code>encoding_rs::Encoding::for_label(<var>string</var>)</code></td></tr>
592//! <tr><td><code><var>enc</var>.whatwg_name()</code> (always lower case)</td><td><code><var>enc</var>.name()</code> (potentially mixed case)</td></tr>
593//! <tr><td><code><var>enc</var>.name()</code></td><td>Not available because not in the Encoding Standard</td></tr>
594//! <tr><td><code>encoding::decode(<var>bytes</var>, encoding::DecoderTrap::Replace, <var>enc</var>)</code></td><td><code><var>enc</var>.decode(<var>bytes</var>)</code></td></tr>
595//! <tr><td><code><var>enc</var>.decode(<var>bytes</var>, encoding::DecoderTrap::Replace)</code></td><td><code><var>enc</var>.decode_without_bom_handling(<var>bytes</var>)</code></td></tr>
596//! <tr><td><code><var>enc</var>.encode(<var>string</var>, encoding::EncoderTrap::NcrEscape)</code></td><td><code><var>enc</var>.encode(<var>string</var>)</code></td></tr>
597//! <tr><td><code><var>enc</var>.raw_decoder()</code></td><td><code><var>enc</var>.new_decoder_without_bom_handling()</code></td></tr>
598//! <tr><td><code><var>enc</var>.raw_encoder()</code></td><td><code><var>enc</var>.new_encoder()</code></td></tr>
599//! <tr><td><code>encoding::RawDecoder</code></td><td><code>encoding_rs::Decoder</code></td></tr>
600//! <tr><td><code>encoding::RawEncoder</code></td><td><code>encoding_rs::Encoder</code></td></tr>
601//! <tr><td><code><var>raw_decoder</var>.raw_feed(<var>src</var>, <var>dst_string</var>)</code></td><td><code><var>dst_string</var>.reserve(<var>decoder</var>.max_utf8_buffer_length_without_replacement(<var>src</var>.len()));<br><var>decoder</var>.decode_to_string_without_replacement(<var>src</var>, <var>dst_string</var>, false)</code></td></tr>
602//! <tr><td><code><var>raw_encoder</var>.raw_feed(<var>src</var>, <var>dst_vec</var>)</code></td><td><code><var>dst_vec</var>.reserve(<var>encoder</var>.max_buffer_length_from_utf8_without_replacement(<var>src</var>.len()));<br><var>encoder</var>.encode_from_utf8_to_vec_without_replacement(<var>src</var>, <var>dst_vec</var>, false)</code></td></tr>
603//! <tr><td><code><var>raw_decoder</var>.raw_finish(<var>dst</var>)</code></td><td><code><var>dst_string</var>.reserve(<var>decoder</var>.max_utf8_buffer_length_without_replacement(0));<br><var>decoder</var>.decode_to_string_without_replacement(b"", <var>dst</var>, true)</code></td></tr>
604//! <tr><td><code><var>raw_encoder</var>.raw_finish(<var>dst</var>)</code></td><td><code><var>dst_vec</var>.reserve(<var>encoder</var>.max_buffer_length_from_utf8_without_replacement(0));<br><var>encoder</var>.encode_from_utf8_to_vec_without_replacement("", <var>dst</var>, true)</code></td></tr>
605//! <tr><td><code>encoding::DecoderTrap::Strict</code></td><td><code>decode*</code> methods that have <code>_without_replacement</code> in their name (and treating the `Malformed` result as fatal).</td></tr>
606//! <tr><td><code>encoding::DecoderTrap::Replace</code></td><td><code>decode*</code> methods that <i>do not</i> have <code>_without_replacement</code> in their name.</td></tr>
607//! <tr><td><code>encoding::DecoderTrap::Ignore</code></td><td>It is a bad idea to ignore errors due to security issues, but this could be implemented using <code>decode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
608//! <tr><td><code>encoding::DecoderTrap::Call(DecoderTrapFunc)</code></td><td>Can be implemented using <code>decode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
609//! <tr><td><code>encoding::EncoderTrap::Strict</code></td><td><code>encode*</code> methods that have <code>_without_replacement</code> in their name (and treating the `Unmappable` result as fatal).</td></tr>
610//! <tr><td><code>encoding::EncoderTrap::Replace</code></td><td>Can be implemented using <code>encode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
611//! <tr><td><code>encoding::EncoderTrap::Ignore</code></td><td>It is a bad idea to ignore errors due to security issues, but this could be implemented using <code>encode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
612//! <tr><td><code>encoding::EncoderTrap::NcrEscape</code></td><td><code>encode*</code> methods that <i>do not</i> have <code>_without_replacement</code> in their name.</td></tr>
613//! <tr><td><code>encoding::EncoderTrap::Call(EncoderTrapFunc)</code></td><td>Can be implemented using <code>encode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
614//! </tbody>
615//! </table>
616//!
617//! # Relationship with Windows Code Pages
618//!
619//! Despite the Web and browser focus, the encodings defined by the Encoding
620//! Standard and implemented by this crate may be useful for decoding legacy
621//! data that uses Windows code pages. The following table names the single-byte
622//! encodings
623//! that have a closely related Windows code page, the number of the closest
624//! code page, a column indicating whether Windows maps unassigned code points
625//! to the Unicode Private Use Area instead of U+FFFD and a remark number
626//! indicating remarks in the list after the table.
627//!
628//! <table>
629//! <thead>
630//! <tr><th>Encoding</th><th>Code Page</th><th>PUA</th><th>Remarks</th></tr>
631//! </thead>
632//! <tbody>
633//! <tr><td>Shift_JIS</td><td>932</td><td></td><td></td></tr>
634//! <tr><td>GBK</td><td>936</td><td></td><td></td></tr>
635//! <tr><td>EUC-KR</td><td>949</td><td></td><td></td></tr>
636//! <tr><td>Big5</td><td>950</td><td></td><td></td></tr>
637//! <tr><td>IBM866</td><td>866</td><td></td><td></td></tr>
638//! <tr><td>windows-874</td><td>874</td><td>&bullet;</td><td></td></tr>
639//! <tr><td>UTF-16LE</td><td>1200</td><td></td><td></td></tr>
640//! <tr><td>UTF-16BE</td><td>1201</td><td></td><td></td></tr>
641//! <tr><td>windows-1250</td><td>1250</td><td></td><td></td></tr>
642//! <tr><td>windows-1251</td><td>1251</td><td></td><td></td></tr>
643//! <tr><td>windows-1252</td><td>1252</td><td></td><td></td></tr>
644//! <tr><td>windows-1253</td><td>1253</td><td>&bullet;</td><td></td></tr>
645//! <tr><td>windows-1254</td><td>1254</td><td></td><td></td></tr>
646//! <tr><td>windows-1255</td><td>1255</td><td>&bullet;</td><td></td></tr>
647//! <tr><td>windows-1256</td><td>1256</td><td></td><td></td></tr>
648//! <tr><td>windows-1257</td><td>1257</td><td>&bullet;</td><td></td></tr>
649//! <tr><td>windows-1258</td><td>1258</td><td></td><td></td></tr>
650//! <tr><td>macintosh</td><td>10000</td><td></td><td>1</td></tr>
651//! <tr><td>x-mac-cyrillic</td><td>10017</td><td></td><td>2</td></tr>
652//! <tr><td>KOI8-R</td><td>20866</td><td></td><td></td></tr>
653//! <tr><td>EUC-JP</td><td>20932</td><td></td><td></td></tr>
654//! <tr><td>KOI8-U</td><td>21866</td><td></td><td></td></tr>
655//! <tr><td>ISO-8859-2</td><td>28592</td><td></td><td></td></tr>
656//! <tr><td>ISO-8859-3</td><td>28593</td><td></td><td></td></tr>
657//! <tr><td>ISO-8859-4</td><td>28594</td><td></td><td></td></tr>
658//! <tr><td>ISO-8859-5</td><td>28595</td><td></td><td></td></tr>
659//! <tr><td>ISO-8859-6</td><td>28596</td><td>&bullet;</td><td></td></tr>
660//! <tr><td>ISO-8859-7</td><td>28597</td><td>&bullet;</td><td>3</td></tr>
661//! <tr><td>ISO-8859-8</td><td>28598</td><td>&bullet;</td><td>4</td></tr>
662//! <tr><td>ISO-8859-13</td><td>28603</td><td>&bullet;</td><td></td></tr>
663//! <tr><td>ISO-8859-15</td><td>28605</td><td></td><td></td></tr>
664//! <tr><td>ISO-8859-8-I</td><td>38598</td><td></td><td>5</td></tr>
665//! <tr><td>ISO-2022-JP</td><td>50220</td><td></td><td></td></tr>
666//! <tr><td>gb18030</td><td>54936</td><td></td><td></td></tr>
667//! <tr><td>UTF-8</td><td>65001</td><td></td><td></td></tr>
668//! </tbody>
669//! </table>
670//!
671//! 1. Windows decodes 0xBD to U+2126 OHM SIGN instead of U+03A9 GREEK CAPITAL LETTER OMEGA.
672//! 2. Windows decodes 0xFF to U+00A4 CURRENCY SIGN instead of U+20AC EURO SIGN.
673//! 3. Windows decodes the currency signs at 0xA4 and 0xA5 as well as 0xAA,
674//!    which should be U+037A GREEK YPOGEGRAMMENI, to PUA code points. Windows
675//!    decodes 0xA1 to U+02BD MODIFIER LETTER REVERSED COMMA instead of U+2018
676//!    LEFT SINGLE QUOTATION MARK and 0xA2 to U+02BC MODIFIER LETTER APOSTROPHE
677//!    instead of U+2019 RIGHT SINGLE QUOTATION MARK.
678//! 4. Windows decodes 0xAF to OVERLINE instead of MACRON and 0xFE and 0xFD to PUA instead
679//!    of LRM and RLM.
680//! 5. Remarks from the previous item apply.
681//!
682//! The differences between this crate and Windows in the case of multibyte encodings
683//! are not yet fully documented here. The lack of remarks above should not be taken
684//! as indication of lack of differences.
685//!
686//! # Notable Differences from IANA Naming
687//!
688//! In some cases, the Encoding Standard specifies the popular unextended encoding
689//! name where in IANA terms one of the other labels would be more precise considering
690//! the extensions that the Encoding Standard has unified into the encoding.
691//!
692//! <table>
693//! <thead>
694//! <tr><th>Encoding</th><th>IANA</th></tr>
695//! </thead>
696//! <tbody>
697//! <tr><td>Big5</td><td>Big5-HKSCS</td></tr>
698//! <tr><td>EUC-KR</td><td>windows-949</td></tr>
699//! <tr><td>Shift_JIS</td><td>windows-31j</td></tr>
700//! <tr><td>x-mac-cyrillic</td><td>x-mac-ukrainian</td></tr>
701//! </tbody>
702//! </table>
703//!
704//! In other cases where the Encoding Standard unifies unextended and extended
705//! variants of an encoding, the encoding gets the name of the extended
706//! variant.
707//!
708//! <table>
709//! <thead>
710//! <tr><th>IANA</th><th>Unified into Encoding</th></tr>
711//! </thead>
712//! <tbody>
713//! <tr><td>ISO-8859-1</td><td>windows-1252</td></tr>
714//! <tr><td>ISO-8859-9</td><td>windows-1254</td></tr>
715//! <tr><td>TIS-620</td><td>windows-874</td></tr>
716//! </tbody>
717//! </table>
718//!
719//! See the section [_UTF-16LE, UTF-16BE and Unicode Encoding Schemes_](#utf-16le-utf-16be-and-unicode-encoding-schemes)
720//! for discussion about the UTF-16 family.
721
722#![no_std]
723#![cfg_attr(feature = "simd-accel", feature(core_intrinsics, portable_simd))]
724
725#[cfg(feature = "alloc")]
726#[cfg_attr(test, macro_use)]
727extern crate alloc;
728
729extern crate core;
730#[macro_use]
731extern crate cfg_if;
732
733#[cfg(feature = "serde")]
734extern crate serde;
735
736#[cfg(all(test, feature = "serde"))]
737extern crate bincode;
738#[cfg(all(test, feature = "serde"))]
739#[macro_use]
740extern crate serde_derive;
741#[cfg(all(test, feature = "serde"))]
742extern crate serde_json;
743
744#[macro_use]
745mod macros;
746
747#[cfg(all(
748    feature = "simd-accel",
749    any(
750        target_feature = "sse2",
751        all(target_endian = "little", target_arch = "aarch64"),
752        all(target_endian = "little", target_feature = "neon")
753    )
754))]
755mod simd_funcs;
756
757#[cfg(all(test, feature = "alloc"))]
758mod testing;
759
760mod big5;
761mod euc_jp;
762mod euc_kr;
763mod gb18030;
764mod gb18030_2022;
765mod iso_2022_jp;
766mod replacement;
767mod shift_jis;
768mod single_byte;
769mod utf_16;
770mod utf_8;
771mod x_user_defined;
772
773mod ascii;
774mod data;
775mod handles;
776mod variant;
777
778pub mod mem;
779
780use crate::ascii::ascii_valid_up_to;
781use crate::ascii::iso_2022_jp_ascii_valid_up_to;
782use crate::utf_8::utf8_valid_up_to;
783use crate::variant::*;
784
785#[cfg(feature = "alloc")]
786use alloc::borrow::Cow;
787#[cfg(feature = "alloc")]
788use alloc::string::String;
789#[cfg(feature = "alloc")]
790use alloc::vec::Vec;
791use core::cmp::Ordering;
792use core::hash::Hash;
793use core::hash::Hasher;
794
795#[cfg(feature = "serde")]
796use serde::de::Visitor;
797#[cfg(feature = "serde")]
798use serde::{Deserialize, Deserializer, Serialize, Serializer};
799
800/// This has to be the max length of an NCR instead of max
801/// minus one, because we can't rely on getting the minus
802/// one from the space reserved for the current unmappable,
803/// because the ISO-2022-JP encoder can fill up that space
804/// with a state transition escape.
805const NCR_EXTRA: usize = 10; // &#1114111;
806
807// BEGIN GENERATED CODE. PLEASE DO NOT EDIT.
808// Instead, please regenerate using generate-encoding-data.py
809
810const LONGEST_LABEL_LENGTH: usize = 19; // cseucpkdfmtjapanese
811
812/// The initializer for the [Big5](static.BIG5.html) encoding.
813///
814/// For use only for taking the address of this form when
815/// Rust prohibits the use of the non-`_INIT` form directly,
816/// such as in initializers of other `static`s. If in doubt,
817/// use the corresponding non-`_INIT` reference-typed `static`.
818///
819/// This part of the public API will go away if Rust changes
820/// to make the referent of `pub const FOO: &'static Encoding`
821/// unique cross-crate or if Rust starts allowing static arrays
822/// to be initialized with `pub static FOO: &'static Encoding`
823/// items.
824pub static BIG5_INIT: Encoding = Encoding {
825    name: "Big5",
826    variant: VariantEncoding::Big5,
827};
828
829/// The Big5 encoding.
830///
831/// This is Big5 with HKSCS with mappings to more recent Unicode assignments
832/// instead of the Private Use Area code points that have been used historically.
833/// It is believed to be able to decode existing Web content in a way that makes
834/// sense.
835///
836/// To avoid form submissions generating data that Web servers don't understand,
837/// the encoder doesn't use the HKSCS byte sequences that precede the unextended
838/// Big5 in the lexical order.
839///
840/// [Index visualization](https://encoding.spec.whatwg.org/big5.html),
841/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/big5-bmp.html)
842///
843/// This encoding is designed to be suited for decoding the Windows code page 950
844/// and its HKSCS patched "951" variant such that the text makes sense, given
845/// assignments that Unicode has made after those encodings used Private Use
846/// Area characters.
847///
848/// This will change from `static` to `const` if Rust changes
849/// to make the referent of `pub const FOO: &'static Encoding`
850/// unique cross-crate, so don't take the address of this
851/// `static`.
852pub static BIG5: &'static Encoding = &BIG5_INIT;
853
854/// The initializer for the [EUC-JP](static.EUC_JP.html) encoding.
855///
856/// For use only for taking the address of this form when
857/// Rust prohibits the use of the non-`_INIT` form directly,
858/// such as in initializers of other `static`s. If in doubt,
859/// use the corresponding non-`_INIT` reference-typed `static`.
860///
861/// This part of the public API will go away if Rust changes
862/// to make the referent of `pub const FOO: &'static Encoding`
863/// unique cross-crate or if Rust starts allowing static arrays
864/// to be initialized with `pub static FOO: &'static Encoding`
865/// items.
866pub static EUC_JP_INIT: Encoding = Encoding {
867    name: "EUC-JP",
868    variant: VariantEncoding::EucJp,
869};
870
871/// The EUC-JP encoding.
872///
873/// This is the legacy Unix encoding for Japanese.
874///
875/// For compatibility with Web servers that don't expect three-byte sequences
876/// in form submissions, the encoder doesn't generate three-byte sequences.
877/// That is, the JIS X 0212 support is decode-only.
878///
879/// [Index visualization](https://encoding.spec.whatwg.org/euc-jp.html),
880/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/euc-jp-bmp.html)
881///
882/// This encoding roughly matches the Windows code page 20932. There are error
883/// handling differences and a handful of 2-byte sequences that decode differently.
884/// Additionall, Windows doesn't support 3-byte sequences.
885///
886/// This will change from `static` to `const` if Rust changes
887/// to make the referent of `pub const FOO: &'static Encoding`
888/// unique cross-crate, so don't take the address of this
889/// `static`.
890pub static EUC_JP: &'static Encoding = &EUC_JP_INIT;
891
892/// The initializer for the [EUC-KR](static.EUC_KR.html) encoding.
893///
894/// For use only for taking the address of this form when
895/// Rust prohibits the use of the non-`_INIT` form directly,
896/// such as in initializers of other `static`s. If in doubt,
897/// use the corresponding non-`_INIT` reference-typed `static`.
898///
899/// This part of the public API will go away if Rust changes
900/// to make the referent of `pub const FOO: &'static Encoding`
901/// unique cross-crate or if Rust starts allowing static arrays
902/// to be initialized with `pub static FOO: &'static Encoding`
903/// items.
904pub static EUC_KR_INIT: Encoding = Encoding {
905    name: "EUC-KR",
906    variant: VariantEncoding::EucKr,
907};
908
909/// The EUC-KR encoding.
910///
911/// This is the Korean encoding for Windows. It extends the Unix legacy encoding
912/// for Korean, based on KS X 1001 (which also formed the base of MacKorean on Mac OS
913/// Classic), with all the characters from the Hangul Syllables block of Unicode.
914///
915/// [Index visualization](https://encoding.spec.whatwg.org/euc-kr.html),
916/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/euc-kr-bmp.html)
917///
918/// This encoding matches the Windows code page 949, except Windows decodes byte 0x80
919/// to U+0080 and some byte sequences that are error per the Encoding Standard to
920/// the question mark or the Private Use Area.
921///
922/// This will change from `static` to `const` if Rust changes
923/// to make the referent of `pub const FOO: &'static Encoding`
924/// unique cross-crate, so don't take the address of this
925/// `static`.
926pub static EUC_KR: &'static Encoding = &EUC_KR_INIT;
927
928/// The initializer for the [GBK](static.GBK.html) encoding.
929///
930/// For use only for taking the address of this form when
931/// Rust prohibits the use of the non-`_INIT` form directly,
932/// such as in initializers of other `static`s. If in doubt,
933/// use the corresponding non-`_INIT` reference-typed `static`.
934///
935/// This part of the public API will go away if Rust changes
936/// to make the referent of `pub const FOO: &'static Encoding`
937/// unique cross-crate or if Rust starts allowing static arrays
938/// to be initialized with `pub static FOO: &'static Encoding`
939/// items.
940pub static GBK_INIT: Encoding = Encoding {
941    name: "GBK",
942    variant: VariantEncoding::Gbk,
943};
944
945/// The GBK encoding.
946///
947/// The decoder for this encoding is the same as the decoder for gb18030.
948/// The encoder side of this encoding is GBK with Windows code page 936 euro
949/// sign behavior and with the changes to two-byte sequences made in GB18030-2022.
950/// GBK extends GB2312-80 to cover the CJK Unified Ideographs Unicode block as
951/// well as a handful of ideographs from the CJK Unified Ideographs Extension A
952/// and CJK Compatibility Ideographs blocks.
953///
954/// Unlike e.g. in the case of ISO-8859-1 and windows-1252, GBK encoder wasn't
955/// unified with the gb18030 encoder in the Encoding Standard out of concern
956/// that servers that expect GBK form submissions might not be able to handle
957/// the four-byte sequences.
958///
959/// [Index visualization for the two-byte sequences](https://encoding.spec.whatwg.org/gb18030.html),
960/// [Visualization of BMP coverage of the two-byte index](https://encoding.spec.whatwg.org/gb18030-bmp.html)
961///
962/// The encoder of this encoding roughly matches the Windows code page 936.
963/// The decoder side is a superset.
964///
965/// This will change from `static` to `const` if Rust changes
966/// to make the referent of `pub const FOO: &'static Encoding`
967/// unique cross-crate, so don't take the address of this
968/// `static`.
969pub static GBK: &'static Encoding = &GBK_INIT;
970
971/// The initializer for the [IBM866](static.IBM866.html) encoding.
972///
973/// For use only for taking the address of this form when
974/// Rust prohibits the use of the non-`_INIT` form directly,
975/// such as in initializers of other `static`s. If in doubt,
976/// use the corresponding non-`_INIT` reference-typed `static`.
977///
978/// This part of the public API will go away if Rust changes
979/// to make the referent of `pub const FOO: &'static Encoding`
980/// unique cross-crate or if Rust starts allowing static arrays
981/// to be initialized with `pub static FOO: &'static Encoding`
982/// items.
983pub static IBM866_INIT: Encoding = Encoding {
984    name: "IBM866",
985    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.ibm866, 0x0440, 96, 16),
986};
987
988/// The IBM866 encoding.
989///
990/// This the most notable one of the DOS Cyrillic code pages. It has the same
991/// box drawing characters as code page 437, so it can be used for decoding
992/// DOS-era ASCII + box drawing data.
993///
994/// [Index visualization](https://encoding.spec.whatwg.org/ibm866.html),
995/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/ibm866-bmp.html)
996///
997/// This encoding matches the Windows code page 866.
998///
999/// This will change from `static` to `const` if Rust changes
1000/// to make the referent of `pub const FOO: &'static Encoding`
1001/// unique cross-crate, so don't take the address of this
1002/// `static`.
1003pub static IBM866: &'static Encoding = &IBM866_INIT;
1004
1005/// The initializer for the [ISO-2022-JP](static.ISO_2022_JP.html) encoding.
1006///
1007/// For use only for taking the address of this form when
1008/// Rust prohibits the use of the non-`_INIT` form directly,
1009/// such as in initializers of other `static`s. If in doubt,
1010/// use the corresponding non-`_INIT` reference-typed `static`.
1011///
1012/// This part of the public API will go away if Rust changes
1013/// to make the referent of `pub const FOO: &'static Encoding`
1014/// unique cross-crate or if Rust starts allowing static arrays
1015/// to be initialized with `pub static FOO: &'static Encoding`
1016/// items.
1017pub static ISO_2022_JP_INIT: Encoding = Encoding {
1018    name: "ISO-2022-JP",
1019    variant: VariantEncoding::Iso2022Jp,
1020};
1021
1022/// The ISO-2022-JP encoding.
1023///
1024/// This the primary pre-UTF-8 encoding for Japanese email. It uses the ASCII
1025/// byte range to encode non-Basic Latin characters. It's the only encoding
1026/// supported by this crate whose encoder is stateful.
1027///
1028/// [Index visualization](https://encoding.spec.whatwg.org/jis0208.html),
1029/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/jis0208-bmp.html)
1030///
1031/// This encoding roughly matches the Windows code page 50220. Notably, Windows
1032/// uses U+30FB in place of the REPLACEMENT CHARACTER and otherwise differs in
1033/// error handling.
1034///
1035/// This will change from `static` to `const` if Rust changes
1036/// to make the referent of `pub const FOO: &'static Encoding`
1037/// unique cross-crate, so don't take the address of this
1038/// `static`.
1039pub static ISO_2022_JP: &'static Encoding = &ISO_2022_JP_INIT;
1040
1041/// The initializer for the [ISO-8859-10](static.ISO_8859_10.html) encoding.
1042///
1043/// For use only for taking the address of this form when
1044/// Rust prohibits the use of the non-`_INIT` form directly,
1045/// such as in initializers of other `static`s. If in doubt,
1046/// use the corresponding non-`_INIT` reference-typed `static`.
1047///
1048/// This part of the public API will go away if Rust changes
1049/// to make the referent of `pub const FOO: &'static Encoding`
1050/// unique cross-crate or if Rust starts allowing static arrays
1051/// to be initialized with `pub static FOO: &'static Encoding`
1052/// items.
1053pub static ISO_8859_10_INIT: Encoding = Encoding {
1054    name: "ISO-8859-10",
1055    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_10, 0x00DA, 90, 6),
1056};
1057
1058/// The ISO-8859-10 encoding.
1059///
1060/// This is the Nordic part of the ISO/IEC 8859 encoding family. This encoding
1061/// is also known as Latin 6.
1062///
1063/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-10.html),
1064/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-10-bmp.html)
1065///
1066/// The Windows code page number for this encoding is 28600, but kernel32.dll
1067/// does not support this encoding.
1068///
1069/// This will change from `static` to `const` if Rust changes
1070/// to make the referent of `pub const FOO: &'static Encoding`
1071/// unique cross-crate, so don't take the address of this
1072/// `static`.
1073pub static ISO_8859_10: &'static Encoding = &ISO_8859_10_INIT;
1074
1075/// The initializer for the [ISO-8859-13](static.ISO_8859_13.html) encoding.
1076///
1077/// For use only for taking the address of this form when
1078/// Rust prohibits the use of the non-`_INIT` form directly,
1079/// such as in initializers of other `static`s. If in doubt,
1080/// use the corresponding non-`_INIT` reference-typed `static`.
1081///
1082/// This part of the public API will go away if Rust changes
1083/// to make the referent of `pub const FOO: &'static Encoding`
1084/// unique cross-crate or if Rust starts allowing static arrays
1085/// to be initialized with `pub static FOO: &'static Encoding`
1086/// items.
1087pub static ISO_8859_13_INIT: Encoding = Encoding {
1088    name: "ISO-8859-13",
1089    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_13, 0x00DF, 95, 1),
1090};
1091
1092/// The ISO-8859-13 encoding.
1093///
1094/// This is the Baltic part of the ISO/IEC 8859 encoding family. This encoding
1095/// is also known as Latin 7.
1096///
1097/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-13.html),
1098/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-13-bmp.html)
1099///
1100/// This encoding matches the Windows code page 28603, except Windows decodes
1101/// unassigned code points to the Private Use Area of Unicode.
1102///
1103/// This will change from `static` to `const` if Rust changes
1104/// to make the referent of `pub const FOO: &'static Encoding`
1105/// unique cross-crate, so don't take the address of this
1106/// `static`.
1107pub static ISO_8859_13: &'static Encoding = &ISO_8859_13_INIT;
1108
1109/// The initializer for the [ISO-8859-14](static.ISO_8859_14.html) encoding.
1110///
1111/// For use only for taking the address of this form when
1112/// Rust prohibits the use of the non-`_INIT` form directly,
1113/// such as in initializers of other `static`s. If in doubt,
1114/// use the corresponding non-`_INIT` reference-typed `static`.
1115///
1116/// This part of the public API will go away if Rust changes
1117/// to make the referent of `pub const FOO: &'static Encoding`
1118/// unique cross-crate or if Rust starts allowing static arrays
1119/// to be initialized with `pub static FOO: &'static Encoding`
1120/// items.
1121pub static ISO_8859_14_INIT: Encoding = Encoding {
1122    name: "ISO-8859-14",
1123    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_14, 0x00DF, 95, 17),
1124};
1125
1126/// The ISO-8859-14 encoding.
1127///
1128/// This is the Celtic part of the ISO/IEC 8859 encoding family. This encoding
1129/// is also known as Latin 8.
1130///
1131/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-14.html),
1132/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-14-bmp.html)
1133///
1134/// The Windows code page number for this encoding is 28604, but kernel32.dll
1135/// does not support this encoding.
1136///
1137/// This will change from `static` to `const` if Rust changes
1138/// to make the referent of `pub const FOO: &'static Encoding`
1139/// unique cross-crate, so don't take the address of this
1140/// `static`.
1141pub static ISO_8859_14: &'static Encoding = &ISO_8859_14_INIT;
1142
1143/// The initializer for the [ISO-8859-15](static.ISO_8859_15.html) encoding.
1144///
1145/// For use only for taking the address of this form when
1146/// Rust prohibits the use of the non-`_INIT` form directly,
1147/// such as in initializers of other `static`s. If in doubt,
1148/// use the corresponding non-`_INIT` reference-typed `static`.
1149///
1150/// This part of the public API will go away if Rust changes
1151/// to make the referent of `pub const FOO: &'static Encoding`
1152/// unique cross-crate or if Rust starts allowing static arrays
1153/// to be initialized with `pub static FOO: &'static Encoding`
1154/// items.
1155pub static ISO_8859_15_INIT: Encoding = Encoding {
1156    name: "ISO-8859-15",
1157    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_15, 0x00BF, 63, 65),
1158};
1159
1160/// The ISO-8859-15 encoding.
1161///
1162/// This is the revised Western European part of the ISO/IEC 8859 encoding
1163/// family. This encoding is also known as Latin 9.
1164///
1165/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-15.html),
1166/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-15-bmp.html)
1167///
1168/// This encoding matches the Windows code page 28605.
1169///
1170/// This will change from `static` to `const` if Rust changes
1171/// to make the referent of `pub const FOO: &'static Encoding`
1172/// unique cross-crate, so don't take the address of this
1173/// `static`.
1174pub static ISO_8859_15: &'static Encoding = &ISO_8859_15_INIT;
1175
1176/// The initializer for the [ISO-8859-16](static.ISO_8859_16.html) encoding.
1177///
1178/// For use only for taking the address of this form when
1179/// Rust prohibits the use of the non-`_INIT` form directly,
1180/// such as in initializers of other `static`s. If in doubt,
1181/// use the corresponding non-`_INIT` reference-typed `static`.
1182///
1183/// This part of the public API will go away if Rust changes
1184/// to make the referent of `pub const FOO: &'static Encoding`
1185/// unique cross-crate or if Rust starts allowing static arrays
1186/// to be initialized with `pub static FOO: &'static Encoding`
1187/// items.
1188pub static ISO_8859_16_INIT: Encoding = Encoding {
1189    name: "ISO-8859-16",
1190    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_16, 0x00DF, 95, 4),
1191};
1192
1193/// The ISO-8859-16 encoding.
1194///
1195/// This is the South-Eastern European part of the ISO/IEC 8859 encoding
1196/// family. This encoding is also known as Latin 10.
1197///
1198/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-16.html),
1199/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-16-bmp.html)
1200///
1201/// The Windows code page number for this encoding is 28606, but kernel32.dll
1202/// does not support this encoding.
1203///
1204/// This will change from `static` to `const` if Rust changes
1205/// to make the referent of `pub const FOO: &'static Encoding`
1206/// unique cross-crate, so don't take the address of this
1207/// `static`.
1208pub static ISO_8859_16: &'static Encoding = &ISO_8859_16_INIT;
1209
1210/// The initializer for the [ISO-8859-2](static.ISO_8859_2.html) encoding.
1211///
1212/// For use only for taking the address of this form when
1213/// Rust prohibits the use of the non-`_INIT` form directly,
1214/// such as in initializers of other `static`s. If in doubt,
1215/// use the corresponding non-`_INIT` reference-typed `static`.
1216///
1217/// This part of the public API will go away if Rust changes
1218/// to make the referent of `pub const FOO: &'static Encoding`
1219/// unique cross-crate or if Rust starts allowing static arrays
1220/// to be initialized with `pub static FOO: &'static Encoding`
1221/// items.
1222pub static ISO_8859_2_INIT: Encoding = Encoding {
1223    name: "ISO-8859-2",
1224    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_2, 0x00DF, 95, 1),
1225};
1226
1227/// The ISO-8859-2 encoding.
1228///
1229/// This is the Central European part of the ISO/IEC 8859 encoding family. This encoding is also known as Latin 2.
1230///
1231/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-2.html),
1232/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-2-bmp.html)
1233///
1234/// This encoding matches the Windows code page 28592.
1235///
1236/// This will change from `static` to `const` if Rust changes
1237/// to make the referent of `pub const FOO: &'static Encoding`
1238/// unique cross-crate, so don't take the address of this
1239/// `static`.
1240pub static ISO_8859_2: &'static Encoding = &ISO_8859_2_INIT;
1241
1242/// The initializer for the [ISO-8859-3](static.ISO_8859_3.html) encoding.
1243///
1244/// For use only for taking the address of this form when
1245/// Rust prohibits the use of the non-`_INIT` form directly,
1246/// such as in initializers of other `static`s. If in doubt,
1247/// use the corresponding non-`_INIT` reference-typed `static`.
1248///
1249/// This part of the public API will go away if Rust changes
1250/// to make the referent of `pub const FOO: &'static Encoding`
1251/// unique cross-crate or if Rust starts allowing static arrays
1252/// to be initialized with `pub static FOO: &'static Encoding`
1253/// items.
1254pub static ISO_8859_3_INIT: Encoding = Encoding {
1255    name: "ISO-8859-3",
1256    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_3, 0x00DF, 95, 4),
1257};
1258
1259/// The ISO-8859-3 encoding.
1260///
1261/// This is the South European part of the ISO/IEC 8859 encoding family. This encoding is also known as Latin 3.
1262///
1263/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-3.html),
1264/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-3-bmp.html)
1265///
1266/// This encoding matches the Windows code page 28593.
1267///
1268/// This will change from `static` to `const` if Rust changes
1269/// to make the referent of `pub const FOO: &'static Encoding`
1270/// unique cross-crate, so don't take the address of this
1271/// `static`.
1272pub static ISO_8859_3: &'static Encoding = &ISO_8859_3_INIT;
1273
1274/// The initializer for the [ISO-8859-4](static.ISO_8859_4.html) encoding.
1275///
1276/// For use only for taking the address of this form when
1277/// Rust prohibits the use of the non-`_INIT` form directly,
1278/// such as in initializers of other `static`s. If in doubt,
1279/// use the corresponding non-`_INIT` reference-typed `static`.
1280///
1281/// This part of the public API will go away if Rust changes
1282/// to make the referent of `pub const FOO: &'static Encoding`
1283/// unique cross-crate or if Rust starts allowing static arrays
1284/// to be initialized with `pub static FOO: &'static Encoding`
1285/// items.
1286pub static ISO_8859_4_INIT: Encoding = Encoding {
1287    name: "ISO-8859-4",
1288    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_4, 0x00DF, 95, 1),
1289};
1290
1291/// The ISO-8859-4 encoding.
1292///
1293/// This is the North European part of the ISO/IEC 8859 encoding family. This encoding is also known as Latin 4.
1294///
1295/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-4.html),
1296/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-4-bmp.html)
1297///
1298/// This encoding matches the Windows code page 28594.
1299///
1300/// This will change from `static` to `const` if Rust changes
1301/// to make the referent of `pub const FOO: &'static Encoding`
1302/// unique cross-crate, so don't take the address of this
1303/// `static`.
1304pub static ISO_8859_4: &'static Encoding = &ISO_8859_4_INIT;
1305
1306/// The initializer for the [ISO-8859-5](static.ISO_8859_5.html) encoding.
1307///
1308/// For use only for taking the address of this form when
1309/// Rust prohibits the use of the non-`_INIT` form directly,
1310/// such as in initializers of other `static`s. If in doubt,
1311/// use the corresponding non-`_INIT` reference-typed `static`.
1312///
1313/// This part of the public API will go away if Rust changes
1314/// to make the referent of `pub const FOO: &'static Encoding`
1315/// unique cross-crate or if Rust starts allowing static arrays
1316/// to be initialized with `pub static FOO: &'static Encoding`
1317/// items.
1318pub static ISO_8859_5_INIT: Encoding = Encoding {
1319    name: "ISO-8859-5",
1320    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_5, 0x040E, 46, 66),
1321};
1322
1323/// The ISO-8859-5 encoding.
1324///
1325/// This is the Cyrillic part of the ISO/IEC 8859 encoding family.
1326///
1327/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-5.html),
1328/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-5-bmp.html)
1329///
1330/// This encoding matches the Windows code page 28595.
1331///
1332/// This will change from `static` to `const` if Rust changes
1333/// to make the referent of `pub const FOO: &'static Encoding`
1334/// unique cross-crate, so don't take the address of this
1335/// `static`.
1336pub static ISO_8859_5: &'static Encoding = &ISO_8859_5_INIT;
1337
1338/// The initializer for the [ISO-8859-6](static.ISO_8859_6.html) encoding.
1339///
1340/// For use only for taking the address of this form when
1341/// Rust prohibits the use of the non-`_INIT` form directly,
1342/// such as in initializers of other `static`s. If in doubt,
1343/// use the corresponding non-`_INIT` reference-typed `static`.
1344///
1345/// This part of the public API will go away if Rust changes
1346/// to make the referent of `pub const FOO: &'static Encoding`
1347/// unique cross-crate or if Rust starts allowing static arrays
1348/// to be initialized with `pub static FOO: &'static Encoding`
1349/// items.
1350pub static ISO_8859_6_INIT: Encoding = Encoding {
1351    name: "ISO-8859-6",
1352    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_6, 0x0621, 65, 26),
1353};
1354
1355/// The ISO-8859-6 encoding.
1356///
1357/// This is the Arabic part of the ISO/IEC 8859 encoding family.
1358///
1359/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-6.html),
1360/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-6-bmp.html)
1361///
1362/// This encoding matches the Windows code page 28596, except Windows decodes
1363/// unassigned code points to the Private Use Area of Unicode.
1364///
1365/// This will change from `static` to `const` if Rust changes
1366/// to make the referent of `pub const FOO: &'static Encoding`
1367/// unique cross-crate, so don't take the address of this
1368/// `static`.
1369pub static ISO_8859_6: &'static Encoding = &ISO_8859_6_INIT;
1370
1371/// The initializer for the [ISO-8859-7](static.ISO_8859_7.html) encoding.
1372///
1373/// For use only for taking the address of this form when
1374/// Rust prohibits the use of the non-`_INIT` form directly,
1375/// such as in initializers of other `static`s. If in doubt,
1376/// use the corresponding non-`_INIT` reference-typed `static`.
1377///
1378/// This part of the public API will go away if Rust changes
1379/// to make the referent of `pub const FOO: &'static Encoding`
1380/// unique cross-crate or if Rust starts allowing static arrays
1381/// to be initialized with `pub static FOO: &'static Encoding`
1382/// items.
1383pub static ISO_8859_7_INIT: Encoding = Encoding {
1384    name: "ISO-8859-7",
1385    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_7, 0x03A3, 83, 44),
1386};
1387
1388/// The ISO-8859-7 encoding.
1389///
1390/// This is the Greek part of the ISO/IEC 8859 encoding family.
1391///
1392/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-7.html),
1393/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-7-bmp.html)
1394///
1395/// This encoding roughly matches the Windows code page 28597. Windows decodes
1396/// unassigned code points, the currency signs at 0xA4 and 0xA5 as well as
1397/// 0xAA, which should be U+037A GREEK YPOGEGRAMMENI, to the Private Use Area
1398/// of Unicode. Windows decodes 0xA1 to U+02BD MODIFIER LETTER REVERSED COMMA
1399/// instead of U+2018 LEFT SINGLE QUOTATION MARK and 0xA2 to U+02BC MODIFIER
1400/// LETTER APOSTROPHE instead of U+2019 RIGHT SINGLE QUOTATION MARK.
1401///
1402/// This will change from `static` to `const` if Rust changes
1403/// to make the referent of `pub const FOO: &'static Encoding`
1404/// unique cross-crate, so don't take the address of this
1405/// `static`.
1406pub static ISO_8859_7: &'static Encoding = &ISO_8859_7_INIT;
1407
1408/// The initializer for the [ISO-8859-8](static.ISO_8859_8.html) encoding.
1409///
1410/// For use only for taking the address of this form when
1411/// Rust prohibits the use of the non-`_INIT` form directly,
1412/// such as in initializers of other `static`s. If in doubt,
1413/// use the corresponding non-`_INIT` reference-typed `static`.
1414///
1415/// This part of the public API will go away if Rust changes
1416/// to make the referent of `pub const FOO: &'static Encoding`
1417/// unique cross-crate or if Rust starts allowing static arrays
1418/// to be initialized with `pub static FOO: &'static Encoding`
1419/// items.
1420pub static ISO_8859_8_INIT: Encoding = Encoding {
1421    name: "ISO-8859-8",
1422    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_8, 0x05D0, 96, 27),
1423};
1424
1425/// The ISO-8859-8 encoding.
1426///
1427/// This is the Hebrew part of the ISO/IEC 8859 encoding family in visual order.
1428///
1429/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-8.html),
1430/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-8-bmp.html)
1431///
1432/// This encoding roughly matches the Windows code page 28598. Windows decodes
1433/// 0xAF to OVERLINE instead of MACRON and 0xFE and 0xFD to the Private Use
1434/// Area instead of LRM and RLM. Windows decodes unassigned code points to
1435/// the private use area.
1436///
1437/// This will change from `static` to `const` if Rust changes
1438/// to make the referent of `pub const FOO: &'static Encoding`
1439/// unique cross-crate, so don't take the address of this
1440/// `static`.
1441pub static ISO_8859_8: &'static Encoding = &ISO_8859_8_INIT;
1442
1443/// The initializer for the [ISO-8859-8-I](static.ISO_8859_8_I.html) encoding.
1444///
1445/// For use only for taking the address of this form when
1446/// Rust prohibits the use of the non-`_INIT` form directly,
1447/// such as in initializers of other `static`s. If in doubt,
1448/// use the corresponding non-`_INIT` reference-typed `static`.
1449///
1450/// This part of the public API will go away if Rust changes
1451/// to make the referent of `pub const FOO: &'static Encoding`
1452/// unique cross-crate or if Rust starts allowing static arrays
1453/// to be initialized with `pub static FOO: &'static Encoding`
1454/// items.
1455pub static ISO_8859_8_I_INIT: Encoding = Encoding {
1456    name: "ISO-8859-8-I",
1457    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_8, 0x05D0, 96, 27),
1458};
1459
1460/// The ISO-8859-8-I encoding.
1461///
1462/// This is the Hebrew part of the ISO/IEC 8859 encoding family in logical order.
1463///
1464/// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-8.html),
1465/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-8-bmp.html)
1466///
1467/// This encoding roughly matches the Windows code page 38598. Windows decodes
1468/// 0xAF to OVERLINE instead of MACRON and 0xFE and 0xFD to the Private Use
1469/// Area instead of LRM and RLM. Windows decodes unassigned code points to
1470/// the private use area.
1471///
1472/// This will change from `static` to `const` if Rust changes
1473/// to make the referent of `pub const FOO: &'static Encoding`
1474/// unique cross-crate, so don't take the address of this
1475/// `static`.
1476pub static ISO_8859_8_I: &'static Encoding = &ISO_8859_8_I_INIT;
1477
1478/// The initializer for the [KOI8-R](static.KOI8_R.html) encoding.
1479///
1480/// For use only for taking the address of this form when
1481/// Rust prohibits the use of the non-`_INIT` form directly,
1482/// such as in initializers of other `static`s. If in doubt,
1483/// use the corresponding non-`_INIT` reference-typed `static`.
1484///
1485/// This part of the public API will go away if Rust changes
1486/// to make the referent of `pub const FOO: &'static Encoding`
1487/// unique cross-crate or if Rust starts allowing static arrays
1488/// to be initialized with `pub static FOO: &'static Encoding`
1489/// items.
1490pub static KOI8_R_INIT: Encoding = Encoding {
1491    name: "KOI8-R",
1492    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.koi8_r, 0x044E, 64, 1),
1493};
1494
1495/// The KOI8-R encoding.
1496///
1497/// This is an encoding for Russian from [RFC 1489](https://tools.ietf.org/html/rfc1489).
1498///
1499/// [Index visualization](https://encoding.spec.whatwg.org/koi8-r.html),
1500/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/koi8-r-bmp.html)
1501///
1502/// This encoding matches the Windows code page 20866.
1503///
1504/// This will change from `static` to `const` if Rust changes
1505/// to make the referent of `pub const FOO: &'static Encoding`
1506/// unique cross-crate, so don't take the address of this
1507/// `static`.
1508pub static KOI8_R: &'static Encoding = &KOI8_R_INIT;
1509
1510/// The initializer for the [KOI8-U](static.KOI8_U.html) encoding.
1511///
1512/// For use only for taking the address of this form when
1513/// Rust prohibits the use of the non-`_INIT` form directly,
1514/// such as in initializers of other `static`s. If in doubt,
1515/// use the corresponding non-`_INIT` reference-typed `static`.
1516///
1517/// This part of the public API will go away if Rust changes
1518/// to make the referent of `pub const FOO: &'static Encoding`
1519/// unique cross-crate or if Rust starts allowing static arrays
1520/// to be initialized with `pub static FOO: &'static Encoding`
1521/// items.
1522pub static KOI8_U_INIT: Encoding = Encoding {
1523    name: "KOI8-U",
1524    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.koi8_u, 0x044E, 64, 1),
1525};
1526
1527/// The KOI8-U encoding.
1528///
1529/// This is an encoding for Ukrainian adapted from KOI8-R.
1530///
1531/// [Index visualization](https://encoding.spec.whatwg.org/koi8-u.html),
1532/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/koi8-u-bmp.html)
1533///
1534/// This encoding matches the Windows code page 21866.
1535///
1536/// This will change from `static` to `const` if Rust changes
1537/// to make the referent of `pub const FOO: &'static Encoding`
1538/// unique cross-crate, so don't take the address of this
1539/// `static`.
1540pub static KOI8_U: &'static Encoding = &KOI8_U_INIT;
1541
1542/// The initializer for the [Shift_JIS](static.SHIFT_JIS.html) encoding.
1543///
1544/// For use only for taking the address of this form when
1545/// Rust prohibits the use of the non-`_INIT` form directly,
1546/// such as in initializers of other `static`s. If in doubt,
1547/// use the corresponding non-`_INIT` reference-typed `static`.
1548///
1549/// This part of the public API will go away if Rust changes
1550/// to make the referent of `pub const FOO: &'static Encoding`
1551/// unique cross-crate or if Rust starts allowing static arrays
1552/// to be initialized with `pub static FOO: &'static Encoding`
1553/// items.
1554pub static SHIFT_JIS_INIT: Encoding = Encoding {
1555    name: "Shift_JIS",
1556    variant: VariantEncoding::ShiftJis,
1557};
1558
1559/// The Shift_JIS encoding.
1560///
1561/// This is the Japanese encoding for Windows.
1562///
1563/// [Index visualization](https://encoding.spec.whatwg.org/shift_jis.html),
1564/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/shift_jis-bmp.html)
1565///
1566/// This encoding matches the Windows code page 932, except Windows decodes some byte
1567/// sequences that are error per the Encoding Standard to the question mark or the
1568/// Private Use Area and generally uses U+30FB in place of the REPLACEMENT CHARACTER.
1569///
1570/// This will change from `static` to `const` if Rust changes
1571/// to make the referent of `pub const FOO: &'static Encoding`
1572/// unique cross-crate, so don't take the address of this
1573/// `static`.
1574pub static SHIFT_JIS: &'static Encoding = &SHIFT_JIS_INIT;
1575
1576/// The initializer for the [UTF-16BE](static.UTF_16BE.html) encoding.
1577///
1578/// For use only for taking the address of this form when
1579/// Rust prohibits the use of the non-`_INIT` form directly,
1580/// such as in initializers of other `static`s. If in doubt,
1581/// use the corresponding non-`_INIT` reference-typed `static`.
1582///
1583/// This part of the public API will go away if Rust changes
1584/// to make the referent of `pub const FOO: &'static Encoding`
1585/// unique cross-crate or if Rust starts allowing static arrays
1586/// to be initialized with `pub static FOO: &'static Encoding`
1587/// items.
1588pub static UTF_16BE_INIT: Encoding = Encoding {
1589    name: "UTF-16BE",
1590    variant: VariantEncoding::Utf16Be,
1591};
1592
1593/// The UTF-16BE encoding.
1594///
1595/// This decode-only encoding uses 16-bit code units due to Unicode originally
1596/// having been designed as a 16-bit reportoire. In the absence of a byte order
1597/// mark the big endian byte order is assumed.
1598///
1599/// There is no corresponding encoder in this crate or in the Encoding
1600/// Standard. The output encoding of this encoding is UTF-8.
1601///
1602/// This encoding matches the Windows code page 1201.
1603///
1604/// This will change from `static` to `const` if Rust changes
1605/// to make the referent of `pub const FOO: &'static Encoding`
1606/// unique cross-crate, so don't take the address of this
1607/// `static`.
1608pub static UTF_16BE: &'static Encoding = &UTF_16BE_INIT;
1609
1610/// The initializer for the [UTF-16LE](static.UTF_16LE.html) encoding.
1611///
1612/// For use only for taking the address of this form when
1613/// Rust prohibits the use of the non-`_INIT` form directly,
1614/// such as in initializers of other `static`s. If in doubt,
1615/// use the corresponding non-`_INIT` reference-typed `static`.
1616///
1617/// This part of the public API will go away if Rust changes
1618/// to make the referent of `pub const FOO: &'static Encoding`
1619/// unique cross-crate or if Rust starts allowing static arrays
1620/// to be initialized with `pub static FOO: &'static Encoding`
1621/// items.
1622pub static UTF_16LE_INIT: Encoding = Encoding {
1623    name: "UTF-16LE",
1624    variant: VariantEncoding::Utf16Le,
1625};
1626
1627/// The UTF-16LE encoding.
1628///
1629/// This decode-only encoding uses 16-bit code units due to Unicode originally
1630/// having been designed as a 16-bit reportoire. In the absence of a byte order
1631/// mark the little endian byte order is assumed.
1632///
1633/// There is no corresponding encoder in this crate or in the Encoding
1634/// Standard. The output encoding of this encoding is UTF-8.
1635///
1636/// This encoding matches the Windows code page 1200.
1637///
1638/// This will change from `static` to `const` if Rust changes
1639/// to make the referent of `pub const FOO: &'static Encoding`
1640/// unique cross-crate, so don't take the address of this
1641/// `static`.
1642pub static UTF_16LE: &'static Encoding = &UTF_16LE_INIT;
1643
1644/// The initializer for the [UTF-8](static.UTF_8.html) encoding.
1645///
1646/// For use only for taking the address of this form when
1647/// Rust prohibits the use of the non-`_INIT` form directly,
1648/// such as in initializers of other `static`s. If in doubt,
1649/// use the corresponding non-`_INIT` reference-typed `static`.
1650///
1651/// This part of the public API will go away if Rust changes
1652/// to make the referent of `pub const FOO: &'static Encoding`
1653/// unique cross-crate or if Rust starts allowing static arrays
1654/// to be initialized with `pub static FOO: &'static Encoding`
1655/// items.
1656pub static UTF_8_INIT: Encoding = Encoding {
1657    name: "UTF-8",
1658    variant: VariantEncoding::Utf8,
1659};
1660
1661/// The UTF-8 encoding.
1662///
1663/// This is the encoding that should be used for all new development it can
1664/// represent all of Unicode.
1665///
1666/// This encoding matches the Windows code page 65001, except Windows differs
1667/// in the number of errors generated for some erroneous byte sequences.
1668///
1669/// This will change from `static` to `const` if Rust changes
1670/// to make the referent of `pub const FOO: &'static Encoding`
1671/// unique cross-crate, so don't take the address of this
1672/// `static`.
1673pub static UTF_8: &'static Encoding = &UTF_8_INIT;
1674
1675/// The initializer for the [gb18030](static.GB18030.html) encoding.
1676///
1677/// For use only for taking the address of this form when
1678/// Rust prohibits the use of the non-`_INIT` form directly,
1679/// such as in initializers of other `static`s. If in doubt,
1680/// use the corresponding non-`_INIT` reference-typed `static`.
1681///
1682/// This part of the public API will go away if Rust changes
1683/// to make the referent of `pub const FOO: &'static Encoding`
1684/// unique cross-crate or if Rust starts allowing static arrays
1685/// to be initialized with `pub static FOO: &'static Encoding`
1686/// items.
1687pub static GB18030_INIT: Encoding = Encoding {
1688    name: "gb18030",
1689    variant: VariantEncoding::Gb18030,
1690};
1691
1692/// The gb18030 encoding.
1693///
1694/// This encoding matches GB18030-2022 except the two-byte sequence 0xA3 0xA0
1695/// maps to U+3000 for compatibility with existing Web content and the four-byte
1696/// sequences for the non-PUA characters that got two-byte sequences still decode
1697/// to the same non-PUA characters as in GB18030-2005. As a result, this encoding
1698/// can represent all of Unicode except for 19 private-use characters.
1699///
1700/// [Index visualization for the two-byte sequences](https://encoding.spec.whatwg.org/gb18030.html),
1701/// [Visualization of BMP coverage of the two-byte index](https://encoding.spec.whatwg.org/gb18030-bmp.html)
1702///
1703/// This encoding matches the Windows code page 54936.
1704///
1705/// This will change from `static` to `const` if Rust changes
1706/// to make the referent of `pub const FOO: &'static Encoding`
1707/// unique cross-crate, so don't take the address of this
1708/// `static`.
1709pub static GB18030: &'static Encoding = &GB18030_INIT;
1710
1711/// The initializer for the [macintosh](static.MACINTOSH.html) encoding.
1712///
1713/// For use only for taking the address of this form when
1714/// Rust prohibits the use of the non-`_INIT` form directly,
1715/// such as in initializers of other `static`s. If in doubt,
1716/// use the corresponding non-`_INIT` reference-typed `static`.
1717///
1718/// This part of the public API will go away if Rust changes
1719/// to make the referent of `pub const FOO: &'static Encoding`
1720/// unique cross-crate or if Rust starts allowing static arrays
1721/// to be initialized with `pub static FOO: &'static Encoding`
1722/// items.
1723pub static MACINTOSH_INIT: Encoding = Encoding {
1724    name: "macintosh",
1725    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.macintosh, 0x00CD, 106, 3),
1726};
1727
1728/// The macintosh encoding.
1729///
1730/// This is the MacRoman encoding from Mac OS Classic.
1731///
1732/// [Index visualization](https://encoding.spec.whatwg.org/macintosh.html),
1733/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/macintosh-bmp.html)
1734///
1735/// This encoding matches the Windows code page 10000, except Windows decodes
1736/// 0xBD to U+2126 OHM SIGN instead of U+03A9 GREEK CAPITAL LETTER OMEGA.
1737///
1738/// This will change from `static` to `const` if Rust changes
1739/// to make the referent of `pub const FOO: &'static Encoding`
1740/// unique cross-crate, so don't take the address of this
1741/// `static`.
1742pub static MACINTOSH: &'static Encoding = &MACINTOSH_INIT;
1743
1744/// The initializer for the [replacement](static.REPLACEMENT.html) encoding.
1745///
1746/// For use only for taking the address of this form when
1747/// Rust prohibits the use of the non-`_INIT` form directly,
1748/// such as in initializers of other `static`s. If in doubt,
1749/// use the corresponding non-`_INIT` reference-typed `static`.
1750///
1751/// This part of the public API will go away if Rust changes
1752/// to make the referent of `pub const FOO: &'static Encoding`
1753/// unique cross-crate or if Rust starts allowing static arrays
1754/// to be initialized with `pub static FOO: &'static Encoding`
1755/// items.
1756pub static REPLACEMENT_INIT: Encoding = Encoding {
1757    name: "replacement",
1758    variant: VariantEncoding::Replacement,
1759};
1760
1761/// The replacement encoding.
1762///
1763/// This decode-only encoding decodes all non-zero-length streams to a single
1764/// REPLACEMENT CHARACTER. Its purpose is to avoid the use of an
1765/// ASCII-compatible fallback encoding (typically windows-1252) for some
1766/// encodings that are no longer supported by the Web Platform and that
1767/// would be dangerous to treat as ASCII-compatible.
1768///
1769/// There is no corresponding encoder. The output encoding of this encoding
1770/// is UTF-8.
1771///
1772/// This encoding does not have a Windows code page number.
1773///
1774/// This will change from `static` to `const` if Rust changes
1775/// to make the referent of `pub const FOO: &'static Encoding`
1776/// unique cross-crate, so don't take the address of this
1777/// `static`.
1778pub static REPLACEMENT: &'static Encoding = &REPLACEMENT_INIT;
1779
1780/// The initializer for the [windows-1250](static.WINDOWS_1250.html) encoding.
1781///
1782/// For use only for taking the address of this form when
1783/// Rust prohibits the use of the non-`_INIT` form directly,
1784/// such as in initializers of other `static`s. If in doubt,
1785/// use the corresponding non-`_INIT` reference-typed `static`.
1786///
1787/// This part of the public API will go away if Rust changes
1788/// to make the referent of `pub const FOO: &'static Encoding`
1789/// unique cross-crate or if Rust starts allowing static arrays
1790/// to be initialized with `pub static FOO: &'static Encoding`
1791/// items.
1792pub static WINDOWS_1250_INIT: Encoding = Encoding {
1793    name: "windows-1250",
1794    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1250, 0x00DC, 92, 2),
1795};
1796
1797/// The windows-1250 encoding.
1798///
1799/// This is the Central European encoding for Windows.
1800///
1801/// [Index visualization](https://encoding.spec.whatwg.org/windows-1250.html),
1802/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1250-bmp.html)
1803///
1804/// This encoding matches the Windows code page 1250.
1805///
1806/// This will change from `static` to `const` if Rust changes
1807/// to make the referent of `pub const FOO: &'static Encoding`
1808/// unique cross-crate, so don't take the address of this
1809/// `static`.
1810pub static WINDOWS_1250: &'static Encoding = &WINDOWS_1250_INIT;
1811
1812/// The initializer for the [windows-1251](static.WINDOWS_1251.html) encoding.
1813///
1814/// For use only for taking the address of this form when
1815/// Rust prohibits the use of the non-`_INIT` form directly,
1816/// such as in initializers of other `static`s. If in doubt,
1817/// use the corresponding non-`_INIT` reference-typed `static`.
1818///
1819/// This part of the public API will go away if Rust changes
1820/// to make the referent of `pub const FOO: &'static Encoding`
1821/// unique cross-crate or if Rust starts allowing static arrays
1822/// to be initialized with `pub static FOO: &'static Encoding`
1823/// items.
1824pub static WINDOWS_1251_INIT: Encoding = Encoding {
1825    name: "windows-1251",
1826    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1251, 0x0410, 64, 64),
1827};
1828
1829/// The windows-1251 encoding.
1830///
1831/// This is the Cyrillic encoding for Windows.
1832///
1833/// [Index visualization](https://encoding.spec.whatwg.org/windows-1251.html),
1834/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1251-bmp.html)
1835///
1836/// This encoding matches the Windows code page 1251.
1837///
1838/// This will change from `static` to `const` if Rust changes
1839/// to make the referent of `pub const FOO: &'static Encoding`
1840/// unique cross-crate, so don't take the address of this
1841/// `static`.
1842pub static WINDOWS_1251: &'static Encoding = &WINDOWS_1251_INIT;
1843
1844/// The initializer for the [windows-1252](static.WINDOWS_1252.html) encoding.
1845///
1846/// For use only for taking the address of this form when
1847/// Rust prohibits the use of the non-`_INIT` form directly,
1848/// such as in initializers of other `static`s. If in doubt,
1849/// use the corresponding non-`_INIT` reference-typed `static`.
1850///
1851/// This part of the public API will go away if Rust changes
1852/// to make the referent of `pub const FOO: &'static Encoding`
1853/// unique cross-crate or if Rust starts allowing static arrays
1854/// to be initialized with `pub static FOO: &'static Encoding`
1855/// items.
1856pub static WINDOWS_1252_INIT: Encoding = Encoding {
1857    name: "windows-1252",
1858    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1252, 0x00A0, 32, 96),
1859};
1860
1861/// The windows-1252 encoding.
1862///
1863/// This is the Western encoding for Windows. It is an extension of ISO-8859-1,
1864/// which is known as Latin 1.
1865///
1866/// [Index visualization](https://encoding.spec.whatwg.org/windows-1252.html),
1867/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1252-bmp.html)
1868///
1869/// This encoding matches the Windows code page 1252.
1870///
1871/// This will change from `static` to `const` if Rust changes
1872/// to make the referent of `pub const FOO: &'static Encoding`
1873/// unique cross-crate, so don't take the address of this
1874/// `static`.
1875pub static WINDOWS_1252: &'static Encoding = &WINDOWS_1252_INIT;
1876
1877/// The initializer for the [windows-1253](static.WINDOWS_1253.html) encoding.
1878///
1879/// For use only for taking the address of this form when
1880/// Rust prohibits the use of the non-`_INIT` form directly,
1881/// such as in initializers of other `static`s. If in doubt,
1882/// use the corresponding non-`_INIT` reference-typed `static`.
1883///
1884/// This part of the public API will go away if Rust changes
1885/// to make the referent of `pub const FOO: &'static Encoding`
1886/// unique cross-crate or if Rust starts allowing static arrays
1887/// to be initialized with `pub static FOO: &'static Encoding`
1888/// items.
1889pub static WINDOWS_1253_INIT: Encoding = Encoding {
1890    name: "windows-1253",
1891    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1253, 0x03A3, 83, 44),
1892};
1893
1894/// The windows-1253 encoding.
1895///
1896/// This is the Greek encoding for Windows. It is mostly an extension of
1897/// ISO-8859-7, but U+0386 is mapped to a different byte.
1898///
1899/// [Index visualization](https://encoding.spec.whatwg.org/windows-1253.html),
1900/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1253-bmp.html)
1901///
1902/// This encoding matches the Windows code page 1253, except Windows decodes
1903/// unassigned code points to the Private Use Area of Unicode.
1904///
1905/// This will change from `static` to `const` if Rust changes
1906/// to make the referent of `pub const FOO: &'static Encoding`
1907/// unique cross-crate, so don't take the address of this
1908/// `static`.
1909pub static WINDOWS_1253: &'static Encoding = &WINDOWS_1253_INIT;
1910
1911/// The initializer for the [windows-1254](static.WINDOWS_1254.html) encoding.
1912///
1913/// For use only for taking the address of this form when
1914/// Rust prohibits the use of the non-`_INIT` form directly,
1915/// such as in initializers of other `static`s. If in doubt,
1916/// use the corresponding non-`_INIT` reference-typed `static`.
1917///
1918/// This part of the public API will go away if Rust changes
1919/// to make the referent of `pub const FOO: &'static Encoding`
1920/// unique cross-crate or if Rust starts allowing static arrays
1921/// to be initialized with `pub static FOO: &'static Encoding`
1922/// items.
1923pub static WINDOWS_1254_INIT: Encoding = Encoding {
1924    name: "windows-1254",
1925    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1254, 0x00DF, 95, 17),
1926};
1927
1928/// The windows-1254 encoding.
1929///
1930/// This is the Turkish encoding for Windows. It is an extension of ISO-8859-9,
1931/// which is known as Latin 5.
1932///
1933/// [Index visualization](https://encoding.spec.whatwg.org/windows-1254.html),
1934/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1254-bmp.html)
1935///
1936/// This encoding matches the Windows code page 1254.
1937///
1938/// This will change from `static` to `const` if Rust changes
1939/// to make the referent of `pub const FOO: &'static Encoding`
1940/// unique cross-crate, so don't take the address of this
1941/// `static`.
1942pub static WINDOWS_1254: &'static Encoding = &WINDOWS_1254_INIT;
1943
1944/// The initializer for the [windows-1255](static.WINDOWS_1255.html) encoding.
1945///
1946/// For use only for taking the address of this form when
1947/// Rust prohibits the use of the non-`_INIT` form directly,
1948/// such as in initializers of other `static`s. If in doubt,
1949/// use the corresponding non-`_INIT` reference-typed `static`.
1950///
1951/// This part of the public API will go away if Rust changes
1952/// to make the referent of `pub const FOO: &'static Encoding`
1953/// unique cross-crate or if Rust starts allowing static arrays
1954/// to be initialized with `pub static FOO: &'static Encoding`
1955/// items.
1956pub static WINDOWS_1255_INIT: Encoding = Encoding {
1957    name: "windows-1255",
1958    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1255, 0x05D0, 96, 27),
1959};
1960
1961/// The windows-1255 encoding.
1962///
1963/// This is the Hebrew encoding for Windows. It is an extension of ISO-8859-8-I,
1964/// except for a currency sign swap.
1965///
1966/// [Index visualization](https://encoding.spec.whatwg.org/windows-1255.html),
1967/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1255-bmp.html)
1968///
1969/// This encoding matches the Windows code page 1255, except Windows decodes
1970/// unassigned code points to the Private Use Area of Unicode.
1971///
1972/// This will change from `static` to `const` if Rust changes
1973/// to make the referent of `pub const FOO: &'static Encoding`
1974/// unique cross-crate, so don't take the address of this
1975/// `static`.
1976pub static WINDOWS_1255: &'static Encoding = &WINDOWS_1255_INIT;
1977
1978/// The initializer for the [windows-1256](static.WINDOWS_1256.html) encoding.
1979///
1980/// For use only for taking the address of this form when
1981/// Rust prohibits the use of the non-`_INIT` form directly,
1982/// such as in initializers of other `static`s. If in doubt,
1983/// use the corresponding non-`_INIT` reference-typed `static`.
1984///
1985/// This part of the public API will go away if Rust changes
1986/// to make the referent of `pub const FOO: &'static Encoding`
1987/// unique cross-crate or if Rust starts allowing static arrays
1988/// to be initialized with `pub static FOO: &'static Encoding`
1989/// items.
1990pub static WINDOWS_1256_INIT: Encoding = Encoding {
1991    name: "windows-1256",
1992    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1256, 0x0621, 65, 22),
1993};
1994
1995/// The windows-1256 encoding.
1996///
1997/// This is the Arabic encoding for Windows.
1998///
1999/// [Index visualization](https://encoding.spec.whatwg.org/windows-1256.html),
2000/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1256-bmp.html)
2001///
2002/// This encoding matches the Windows code page 1256.
2003///
2004/// This will change from `static` to `const` if Rust changes
2005/// to make the referent of `pub const FOO: &'static Encoding`
2006/// unique cross-crate, so don't take the address of this
2007/// `static`.
2008pub static WINDOWS_1256: &'static Encoding = &WINDOWS_1256_INIT;
2009
2010/// The initializer for the [windows-1257](static.WINDOWS_1257.html) encoding.
2011///
2012/// For use only for taking the address of this form when
2013/// Rust prohibits the use of the non-`_INIT` form directly,
2014/// such as in initializers of other `static`s. If in doubt,
2015/// use the corresponding non-`_INIT` reference-typed `static`.
2016///
2017/// This part of the public API will go away if Rust changes
2018/// to make the referent of `pub const FOO: &'static Encoding`
2019/// unique cross-crate or if Rust starts allowing static arrays
2020/// to be initialized with `pub static FOO: &'static Encoding`
2021/// items.
2022pub static WINDOWS_1257_INIT: Encoding = Encoding {
2023    name: "windows-1257",
2024    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1257, 0x00DF, 95, 1),
2025};
2026
2027/// The windows-1257 encoding.
2028///
2029/// This is the Baltic encoding for Windows.
2030///
2031/// [Index visualization](https://encoding.spec.whatwg.org/windows-1257.html),
2032/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1257-bmp.html)
2033///
2034/// This encoding matches the Windows code page 1257, except Windows decodes
2035/// unassigned code points to the Private Use Area of Unicode.
2036///
2037/// This will change from `static` to `const` if Rust changes
2038/// to make the referent of `pub const FOO: &'static Encoding`
2039/// unique cross-crate, so don't take the address of this
2040/// `static`.
2041pub static WINDOWS_1257: &'static Encoding = &WINDOWS_1257_INIT;
2042
2043/// The initializer for the [windows-1258](static.WINDOWS_1258.html) encoding.
2044///
2045/// For use only for taking the address of this form when
2046/// Rust prohibits the use of the non-`_INIT` form directly,
2047/// such as in initializers of other `static`s. If in doubt,
2048/// use the corresponding non-`_INIT` reference-typed `static`.
2049///
2050/// This part of the public API will go away if Rust changes
2051/// to make the referent of `pub const FOO: &'static Encoding`
2052/// unique cross-crate or if Rust starts allowing static arrays
2053/// to be initialized with `pub static FOO: &'static Encoding`
2054/// items.
2055pub static WINDOWS_1258_INIT: Encoding = Encoding {
2056    name: "windows-1258",
2057    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1258, 0x00DF, 95, 4),
2058};
2059
2060/// The windows-1258 encoding.
2061///
2062/// This is the Vietnamese encoding for Windows.
2063///
2064/// [Index visualization](https://encoding.spec.whatwg.org/windows-1258.html),
2065/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1258-bmp.html)
2066///
2067/// This encoding matches the Windows code page 1258 when used in the
2068/// non-normalizing mode. Unlike with the other single-byte encodings, the
2069/// result of decoding is not necessarily in Normalization Form C. On the
2070/// other hand, input in the Normalization Form C is not encoded without
2071/// replacement. In general, it's a bad idea to encode to encodings other
2072/// than UTF-8, but this encoding is especially hazardous to encode to.
2073///
2074/// This will change from `static` to `const` if Rust changes
2075/// to make the referent of `pub const FOO: &'static Encoding`
2076/// unique cross-crate, so don't take the address of this
2077/// `static`.
2078pub static WINDOWS_1258: &'static Encoding = &WINDOWS_1258_INIT;
2079
2080/// The initializer for the [windows-874](static.WINDOWS_874.html) encoding.
2081///
2082/// For use only for taking the address of this form when
2083/// Rust prohibits the use of the non-`_INIT` form directly,
2084/// such as in initializers of other `static`s. If in doubt,
2085/// use the corresponding non-`_INIT` reference-typed `static`.
2086///
2087/// This part of the public API will go away if Rust changes
2088/// to make the referent of `pub const FOO: &'static Encoding`
2089/// unique cross-crate or if Rust starts allowing static arrays
2090/// to be initialized with `pub static FOO: &'static Encoding`
2091/// items.
2092pub static WINDOWS_874_INIT: Encoding = Encoding {
2093    name: "windows-874",
2094    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_874, 0x0E01, 33, 58),
2095};
2096
2097/// The windows-874 encoding.
2098///
2099/// This is the Thai encoding for Windows. It is an extension of TIS-620 / ISO-8859-11.
2100///
2101/// [Index visualization](https://encoding.spec.whatwg.org/windows-874.html),
2102/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-874-bmp.html)
2103///
2104/// This encoding matches the Windows code page 874, except Windows decodes
2105/// unassigned code points to the Private Use Area of Unicode.
2106///
2107/// This will change from `static` to `const` if Rust changes
2108/// to make the referent of `pub const FOO: &'static Encoding`
2109/// unique cross-crate, so don't take the address of this
2110/// `static`.
2111pub static WINDOWS_874: &'static Encoding = &WINDOWS_874_INIT;
2112
2113/// The initializer for the [x-mac-cyrillic](static.X_MAC_CYRILLIC.html) encoding.
2114///
2115/// For use only for taking the address of this form when
2116/// Rust prohibits the use of the non-`_INIT` form directly,
2117/// such as in initializers of other `static`s. If in doubt,
2118/// use the corresponding non-`_INIT` reference-typed `static`.
2119///
2120/// This part of the public API will go away if Rust changes
2121/// to make the referent of `pub const FOO: &'static Encoding`
2122/// unique cross-crate or if Rust starts allowing static arrays
2123/// to be initialized with `pub static FOO: &'static Encoding`
2124/// items.
2125pub static X_MAC_CYRILLIC_INIT: Encoding = Encoding {
2126    name: "x-mac-cyrillic",
2127    variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.x_mac_cyrillic, 0x0430, 96, 31),
2128};
2129
2130/// The x-mac-cyrillic encoding.
2131///
2132/// This is the MacUkrainian encoding from Mac OS Classic.
2133///
2134/// [Index visualization](https://encoding.spec.whatwg.org/x-mac-cyrillic.html),
2135/// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/x-mac-cyrillic-bmp.html)
2136///
2137/// This encoding matches the Windows code page 10017.
2138///
2139/// This will change from `static` to `const` if Rust changes
2140/// to make the referent of `pub const FOO: &'static Encoding`
2141/// unique cross-crate, so don't take the address of this
2142/// `static`.
2143pub static X_MAC_CYRILLIC: &'static Encoding = &X_MAC_CYRILLIC_INIT;
2144
2145/// The initializer for the [x-user-defined](static.X_USER_DEFINED.html) encoding.
2146///
2147/// For use only for taking the address of this form when
2148/// Rust prohibits the use of the non-`_INIT` form directly,
2149/// such as in initializers of other `static`s. If in doubt,
2150/// use the corresponding non-`_INIT` reference-typed `static`.
2151///
2152/// This part of the public API will go away if Rust changes
2153/// to make the referent of `pub const FOO: &'static Encoding`
2154/// unique cross-crate or if Rust starts allowing static arrays
2155/// to be initialized with `pub static FOO: &'static Encoding`
2156/// items.
2157pub static X_USER_DEFINED_INIT: Encoding = Encoding {
2158    name: "x-user-defined",
2159    variant: VariantEncoding::UserDefined,
2160};
2161
2162/// The x-user-defined encoding.
2163///
2164/// This encoding offsets the non-ASCII bytes by `0xF700` thereby decoding
2165/// them to the Private Use Area of Unicode. It was used for loading binary
2166/// data into a JavaScript string using `XMLHttpRequest` before XHR supported
2167/// the `"arraybuffer"` response type.
2168///
2169/// This encoding does not have a Windows code page number.
2170///
2171/// This will change from `static` to `const` if Rust changes
2172/// to make the referent of `pub const FOO: &'static Encoding`
2173/// unique cross-crate, so don't take the address of this
2174/// `static`.
2175pub static X_USER_DEFINED: &'static Encoding = &X_USER_DEFINED_INIT;
2176
2177static LABELS_SORTED: [&'static str; 228] = [
2178    "l1",
2179    "l2",
2180    "l3",
2181    "l4",
2182    "l5",
2183    "l6",
2184    "l9",
2185    "866",
2186    "mac",
2187    "koi",
2188    "gbk",
2189    "big5",
2190    "utf8",
2191    "koi8",
2192    "sjis",
2193    "ucs-2",
2194    "ms932",
2195    "cp866",
2196    "utf-8",
2197    "cp819",
2198    "ascii",
2199    "x-gbk",
2200    "greek",
2201    "cp1250",
2202    "cp1251",
2203    "latin1",
2204    "gb2312",
2205    "cp1252",
2206    "latin2",
2207    "cp1253",
2208    "latin3",
2209    "cp1254",
2210    "latin4",
2211    "cp1255",
2212    "csbig5",
2213    "latin5",
2214    "utf-16",
2215    "cp1256",
2216    "ibm866",
2217    "latin6",
2218    "cp1257",
2219    "cp1258",
2220    "greek8",
2221    "ibm819",
2222    "arabic",
2223    "visual",
2224    "korean",
2225    "euc-jp",
2226    "koi8-r",
2227    "koi8_r",
2228    "euc-kr",
2229    "x-sjis",
2230    "koi8-u",
2231    "hebrew",
2232    "tis-620",
2233    "gb18030",
2234    "ksc5601",
2235    "gb_2312",
2236    "dos-874",
2237    "cn-big5",
2238    "unicode",
2239    "chinese",
2240    "logical",
2241    "cskoi8r",
2242    "cseuckr",
2243    "koi8-ru",
2244    "x-cp1250",
2245    "ksc_5601",
2246    "x-cp1251",
2247    "iso88591",
2248    "csgb2312",
2249    "x-cp1252",
2250    "iso88592",
2251    "x-cp1253",
2252    "iso88593",
2253    "ecma-114",
2254    "x-cp1254",
2255    "iso88594",
2256    "x-cp1255",
2257    "iso88595",
2258    "x-x-big5",
2259    "x-cp1256",
2260    "csibm866",
2261    "iso88596",
2262    "x-cp1257",
2263    "iso88597",
2264    "asmo-708",
2265    "ecma-118",
2266    "elot_928",
2267    "x-cp1258",
2268    "iso88598",
2269    "iso88599",
2270    "cyrillic",
2271    "utf-16be",
2272    "utf-16le",
2273    "us-ascii",
2274    "ms_kanji",
2275    "x-euc-jp",
2276    "iso885910",
2277    "iso8859-1",
2278    "iso885911",
2279    "iso8859-2",
2280    "iso8859-3",
2281    "iso885913",
2282    "iso8859-4",
2283    "iso885914",
2284    "iso8859-5",
2285    "iso885915",
2286    "iso8859-6",
2287    "iso8859-7",
2288    "iso8859-8",
2289    "iso-ir-58",
2290    "iso8859-9",
2291    "csunicode",
2292    "macintosh",
2293    "shift-jis",
2294    "shift_jis",
2295    "iso-ir-100",
2296    "iso8859-10",
2297    "iso-ir-110",
2298    "gb_2312-80",
2299    "iso-8859-1",
2300    "iso_8859-1",
2301    "iso-ir-101",
2302    "iso8859-11",
2303    "iso-8859-2",
2304    "iso_8859-2",
2305    "hz-gb-2312",
2306    "iso-8859-3",
2307    "iso_8859-3",
2308    "iso8859-13",
2309    "iso-8859-4",
2310    "iso_8859-4",
2311    "iso8859-14",
2312    "iso-ir-144",
2313    "iso-8859-5",
2314    "iso_8859-5",
2315    "iso8859-15",
2316    "iso-8859-6",
2317    "iso_8859-6",
2318    "iso-ir-126",
2319    "iso-8859-7",
2320    "iso_8859-7",
2321    "iso-ir-127",
2322    "iso-ir-157",
2323    "iso-8859-8",
2324    "iso_8859-8",
2325    "iso-ir-138",
2326    "iso-ir-148",
2327    "iso-8859-9",
2328    "iso_8859-9",
2329    "iso-ir-109",
2330    "iso-ir-149",
2331    "big5-hkscs",
2332    "csshiftjis",
2333    "iso-8859-10",
2334    "iso-8859-11",
2335    "csisolatin1",
2336    "csisolatin2",
2337    "iso-8859-13",
2338    "csisolatin3",
2339    "iso-8859-14",
2340    "windows-874",
2341    "csisolatin4",
2342    "iso-8859-15",
2343    "iso_8859-15",
2344    "csisolatin5",
2345    "iso-8859-16",
2346    "csisolatin6",
2347    "windows-949",
2348    "csisolatin9",
2349    "csiso88596e",
2350    "csiso88598e",
2351    "unicodefffe",
2352    "unicodefeff",
2353    "csmacintosh",
2354    "csiso88596i",
2355    "csiso88598i",
2356    "windows-31j",
2357    "x-mac-roman",
2358    "iso-2022-cn",
2359    "iso-2022-jp",
2360    "csiso2022jp",
2361    "iso-2022-kr",
2362    "csiso2022kr",
2363    "replacement",
2364    "windows-1250",
2365    "windows-1251",
2366    "windows-1252",
2367    "windows-1253",
2368    "windows-1254",
2369    "windows-1255",
2370    "windows-1256",
2371    "windows-1257",
2372    "windows-1258",
2373    "iso-8859-6-e",
2374    "iso-8859-8-e",
2375    "iso-8859-6-i",
2376    "iso-8859-8-i",
2377    "sun_eu_greek",
2378    "csksc56011987",
2379    "unicode20utf8",
2380    "unicode11utf8",
2381    "ks_c_5601-1987",
2382    "ansi_x3.4-1968",
2383    "ks_c_5601-1989",
2384    "x-mac-cyrillic",
2385    "x-user-defined",
2386    "csiso58gb231280",
2387    "iso-10646-ucs-2",
2388    "iso_8859-1:1987",
2389    "iso_8859-2:1987",
2390    "iso_8859-6:1987",
2391    "iso_8859-7:1987",
2392    "iso_8859-3:1988",
2393    "iso_8859-4:1988",
2394    "iso_8859-5:1988",
2395    "iso_8859-8:1988",
2396    "x-unicode20utf8",
2397    "iso_8859-9:1989",
2398    "csisolatingreek",
2399    "x-mac-ukrainian",
2400    "iso-2022-cn-ext",
2401    "csisolatinarabic",
2402    "csisolatinhebrew",
2403    "unicode-1-1-utf-8",
2404    "csisolatincyrillic",
2405    "cseucpkdfmtjapanese",
2406];
2407
2408static ENCODINGS_IN_LABEL_SORT: [&'static Encoding; 228] = [
2409    &WINDOWS_1252_INIT,
2410    &ISO_8859_2_INIT,
2411    &ISO_8859_3_INIT,
2412    &ISO_8859_4_INIT,
2413    &WINDOWS_1254_INIT,
2414    &ISO_8859_10_INIT,
2415    &ISO_8859_15_INIT,
2416    &IBM866_INIT,
2417    &MACINTOSH_INIT,
2418    &KOI8_R_INIT,
2419    &GBK_INIT,
2420    &BIG5_INIT,
2421    &UTF_8_INIT,
2422    &KOI8_R_INIT,
2423    &SHIFT_JIS_INIT,
2424    &UTF_16LE_INIT,
2425    &SHIFT_JIS_INIT,
2426    &IBM866_INIT,
2427    &UTF_8_INIT,
2428    &WINDOWS_1252_INIT,
2429    &WINDOWS_1252_INIT,
2430    &GBK_INIT,
2431    &ISO_8859_7_INIT,
2432    &WINDOWS_1250_INIT,
2433    &WINDOWS_1251_INIT,
2434    &WINDOWS_1252_INIT,
2435    &GBK_INIT,
2436    &WINDOWS_1252_INIT,
2437    &ISO_8859_2_INIT,
2438    &WINDOWS_1253_INIT,
2439    &ISO_8859_3_INIT,
2440    &WINDOWS_1254_INIT,
2441    &ISO_8859_4_INIT,
2442    &WINDOWS_1255_INIT,
2443    &BIG5_INIT,
2444    &WINDOWS_1254_INIT,
2445    &UTF_16LE_INIT,
2446    &WINDOWS_1256_INIT,
2447    &IBM866_INIT,
2448    &ISO_8859_10_INIT,
2449    &WINDOWS_1257_INIT,
2450    &WINDOWS_1258_INIT,
2451    &ISO_8859_7_INIT,
2452    &WINDOWS_1252_INIT,
2453    &ISO_8859_6_INIT,
2454    &ISO_8859_8_INIT,
2455    &EUC_KR_INIT,
2456    &EUC_JP_INIT,
2457    &KOI8_R_INIT,
2458    &KOI8_R_INIT,
2459    &EUC_KR_INIT,
2460    &SHIFT_JIS_INIT,
2461    &KOI8_U_INIT,
2462    &ISO_8859_8_INIT,
2463    &WINDOWS_874_INIT,
2464    &GB18030_INIT,
2465    &EUC_KR_INIT,
2466    &GBK_INIT,
2467    &WINDOWS_874_INIT,
2468    &BIG5_INIT,
2469    &UTF_16LE_INIT,
2470    &GBK_INIT,
2471    &ISO_8859_8_I_INIT,
2472    &KOI8_R_INIT,
2473    &EUC_KR_INIT,
2474    &KOI8_U_INIT,
2475    &WINDOWS_1250_INIT,
2476    &EUC_KR_INIT,
2477    &WINDOWS_1251_INIT,
2478    &WINDOWS_1252_INIT,
2479    &GBK_INIT,
2480    &WINDOWS_1252_INIT,
2481    &ISO_8859_2_INIT,
2482    &WINDOWS_1253_INIT,
2483    &ISO_8859_3_INIT,
2484    &ISO_8859_6_INIT,
2485    &WINDOWS_1254_INIT,
2486    &ISO_8859_4_INIT,
2487    &WINDOWS_1255_INIT,
2488    &ISO_8859_5_INIT,
2489    &BIG5_INIT,
2490    &WINDOWS_1256_INIT,
2491    &IBM866_INIT,
2492    &ISO_8859_6_INIT,
2493    &WINDOWS_1257_INIT,
2494    &ISO_8859_7_INIT,
2495    &ISO_8859_6_INIT,
2496    &ISO_8859_7_INIT,
2497    &ISO_8859_7_INIT,
2498    &WINDOWS_1258_INIT,
2499    &ISO_8859_8_INIT,
2500    &WINDOWS_1254_INIT,
2501    &ISO_8859_5_INIT,
2502    &UTF_16BE_INIT,
2503    &UTF_16LE_INIT,
2504    &WINDOWS_1252_INIT,
2505    &SHIFT_JIS_INIT,
2506    &EUC_JP_INIT,
2507    &ISO_8859_10_INIT,
2508    &WINDOWS_1252_INIT,
2509    &WINDOWS_874_INIT,
2510    &ISO_8859_2_INIT,
2511    &ISO_8859_3_INIT,
2512    &ISO_8859_13_INIT,
2513    &ISO_8859_4_INIT,
2514    &ISO_8859_14_INIT,
2515    &ISO_8859_5_INIT,
2516    &ISO_8859_15_INIT,
2517    &ISO_8859_6_INIT,
2518    &ISO_8859_7_INIT,
2519    &ISO_8859_8_INIT,
2520    &GBK_INIT,
2521    &WINDOWS_1254_INIT,
2522    &UTF_16LE_INIT,
2523    &MACINTOSH_INIT,
2524    &SHIFT_JIS_INIT,
2525    &SHIFT_JIS_INIT,
2526    &WINDOWS_1252_INIT,
2527    &ISO_8859_10_INIT,
2528    &ISO_8859_4_INIT,
2529    &GBK_INIT,
2530    &WINDOWS_1252_INIT,
2531    &WINDOWS_1252_INIT,
2532    &ISO_8859_2_INIT,
2533    &WINDOWS_874_INIT,
2534    &ISO_8859_2_INIT,
2535    &ISO_8859_2_INIT,
2536    &REPLACEMENT_INIT,
2537    &ISO_8859_3_INIT,
2538    &ISO_8859_3_INIT,
2539    &ISO_8859_13_INIT,
2540    &ISO_8859_4_INIT,
2541    &ISO_8859_4_INIT,
2542    &ISO_8859_14_INIT,
2543    &ISO_8859_5_INIT,
2544    &ISO_8859_5_INIT,
2545    &ISO_8859_5_INIT,
2546    &ISO_8859_15_INIT,
2547    &ISO_8859_6_INIT,
2548    &ISO_8859_6_INIT,
2549    &ISO_8859_7_INIT,
2550    &ISO_8859_7_INIT,
2551    &ISO_8859_7_INIT,
2552    &ISO_8859_6_INIT,
2553    &ISO_8859_10_INIT,
2554    &ISO_8859_8_INIT,
2555    &ISO_8859_8_INIT,
2556    &ISO_8859_8_INIT,
2557    &WINDOWS_1254_INIT,
2558    &WINDOWS_1254_INIT,
2559    &WINDOWS_1254_INIT,
2560    &ISO_8859_3_INIT,
2561    &EUC_KR_INIT,
2562    &BIG5_INIT,
2563    &SHIFT_JIS_INIT,
2564    &ISO_8859_10_INIT,
2565    &WINDOWS_874_INIT,
2566    &WINDOWS_1252_INIT,
2567    &ISO_8859_2_INIT,
2568    &ISO_8859_13_INIT,
2569    &ISO_8859_3_INIT,
2570    &ISO_8859_14_INIT,
2571    &WINDOWS_874_INIT,
2572    &ISO_8859_4_INIT,
2573    &ISO_8859_15_INIT,
2574    &ISO_8859_15_INIT,
2575    &WINDOWS_1254_INIT,
2576    &ISO_8859_16_INIT,
2577    &ISO_8859_10_INIT,
2578    &EUC_KR_INIT,
2579    &ISO_8859_15_INIT,
2580    &ISO_8859_6_INIT,
2581    &ISO_8859_8_INIT,
2582    &UTF_16BE_INIT,
2583    &UTF_16LE_INIT,
2584    &MACINTOSH_INIT,
2585    &ISO_8859_6_INIT,
2586    &ISO_8859_8_I_INIT,
2587    &SHIFT_JIS_INIT,
2588    &MACINTOSH_INIT,
2589    &REPLACEMENT_INIT,
2590    &ISO_2022_JP_INIT,
2591    &ISO_2022_JP_INIT,
2592    &REPLACEMENT_INIT,
2593    &REPLACEMENT_INIT,
2594    &REPLACEMENT_INIT,
2595    &WINDOWS_1250_INIT,
2596    &WINDOWS_1251_INIT,
2597    &WINDOWS_1252_INIT,
2598    &WINDOWS_1253_INIT,
2599    &WINDOWS_1254_INIT,
2600    &WINDOWS_1255_INIT,
2601    &WINDOWS_1256_INIT,
2602    &WINDOWS_1257_INIT,
2603    &WINDOWS_1258_INIT,
2604    &ISO_8859_6_INIT,
2605    &ISO_8859_8_INIT,
2606    &ISO_8859_6_INIT,
2607    &ISO_8859_8_I_INIT,
2608    &ISO_8859_7_INIT,
2609    &EUC_KR_INIT,
2610    &UTF_8_INIT,
2611    &UTF_8_INIT,
2612    &EUC_KR_INIT,
2613    &WINDOWS_1252_INIT,
2614    &EUC_KR_INIT,
2615    &X_MAC_CYRILLIC_INIT,
2616    &X_USER_DEFINED_INIT,
2617    &GBK_INIT,
2618    &UTF_16LE_INIT,
2619    &WINDOWS_1252_INIT,
2620    &ISO_8859_2_INIT,
2621    &ISO_8859_6_INIT,
2622    &ISO_8859_7_INIT,
2623    &ISO_8859_3_INIT,
2624    &ISO_8859_4_INIT,
2625    &ISO_8859_5_INIT,
2626    &ISO_8859_8_INIT,
2627    &UTF_8_INIT,
2628    &WINDOWS_1254_INIT,
2629    &ISO_8859_7_INIT,
2630    &X_MAC_CYRILLIC_INIT,
2631    &REPLACEMENT_INIT,
2632    &ISO_8859_6_INIT,
2633    &ISO_8859_8_INIT,
2634    &UTF_8_INIT,
2635    &ISO_8859_5_INIT,
2636    &EUC_JP_INIT,
2637];
2638
2639// END GENERATED CODE
2640
2641/// An encoding as defined in the [Encoding Standard][1].
2642///
2643/// An _encoding_ defines a mapping from a `u8` sequence to a `char` sequence
2644/// and, in most cases, vice versa. Each encoding has a name, an output
2645/// encoding, and one or more labels.
2646///
2647/// _Labels_ are ASCII-case-insensitive strings that are used to identify an
2648/// encoding in formats and protocols. The _name_ of the encoding is the
2649/// preferred label in the case appropriate for returning from the
2650/// [`characterSet`][2] property of the `Document` DOM interface.
2651///
2652/// The _output encoding_ is the encoding used for form submission and URL
2653/// parsing on Web pages in the encoding. This is UTF-8 for the replacement,
2654/// UTF-16LE and UTF-16BE encodings and the encoding itself for other
2655/// encodings.
2656///
2657/// [1]: https://encoding.spec.whatwg.org/
2658/// [2]: https://dom.spec.whatwg.org/#dom-document-characterset
2659///
2660/// # Streaming vs. Non-Streaming
2661///
2662/// When you have the entire input in a single buffer, you can use the
2663/// methods [`decode()`][3], [`decode_with_bom_removal()`][3],
2664/// [`decode_without_bom_handling()`][5],
2665/// [`decode_without_bom_handling_and_without_replacement()`][6] and
2666/// [`encode()`][7]. (These methods are available to Rust callers only and are
2667/// not available in the C API.) Unlike the rest of the API available to Rust,
2668/// these methods perform heap allocations. You should the `Decoder` and
2669/// `Encoder` objects when your input is split into multiple buffers or when
2670/// you want to control the allocation of the output buffers.
2671///
2672/// [3]: #method.decode
2673/// [4]: #method.decode_with_bom_removal
2674/// [5]: #method.decode_without_bom_handling
2675/// [6]: #method.decode_without_bom_handling_and_without_replacement
2676/// [7]: #method.encode
2677///
2678/// # Instances
2679///
2680/// All instances of `Encoding` are statically allocated and have the `'static`
2681/// lifetime. There is precisely one unique `Encoding` instance for each
2682/// encoding defined in the Encoding Standard.
2683///
2684/// To obtain a reference to a particular encoding whose identity you know at
2685/// compile time, use a `static` that refers to encoding. There is a `static`
2686/// for each encoding. The `static`s are named in all caps with hyphens
2687/// replaced with underscores (and in C/C++ have `_ENCODING` appended to the
2688/// name). For example, if you know at compile time that you will want to
2689/// decode using the UTF-8 encoding, use the `UTF_8` `static` (`UTF_8_ENCODING`
2690/// in C/C++).
2691///
2692/// Additionally, there are non-reference-typed forms ending with `_INIT` to
2693/// work around the problem that `static`s of the type `&'static Encoding`
2694/// cannot be used to initialize items of an array whose type is
2695/// `[&'static Encoding; N]`.
2696///
2697/// If you don't know what encoding you need at compile time and need to
2698/// dynamically get an encoding by label, use
2699/// <code>Encoding::<a href="#method.for_label">for_label</a>(<var>label</var>)</code>.
2700///
2701/// Instances of `Encoding` can be compared with `==` (in both Rust and in
2702/// C/C++).
2703pub struct Encoding {
2704    name: &'static str,
2705    variant: VariantEncoding,
2706}
2707
2708impl Encoding {
2709    /// Implements the
2710    /// [_get an encoding_](https://encoding.spec.whatwg.org/#concept-encoding-get)
2711    /// algorithm.
2712    ///
2713    /// If, after ASCII-lowercasing and removing leading and trailing
2714    /// whitespace, the argument matches a label defined in the Encoding
2715    /// Standard, `Some(&'static Encoding)` representing the corresponding
2716    /// encoding is returned. If there is no match, `None` is returned.
2717    ///
2718    /// This is the right method to use if the action upon the method returning
2719    /// `None` is to use a fallback encoding (e.g. `WINDOWS_1252`) instead.
2720    /// When the action upon the method returning `None` is not to proceed with
2721    /// a fallback but to refuse processing, `for_label_no_replacement()` is more
2722    /// appropriate.
2723    ///
2724    /// The argument is of type `&[u8]` instead of `&str` to save callers
2725    /// that are extracting the label from a non-UTF-8 protocol the trouble
2726    /// of conversion to UTF-8. (If you have a `&str`, just call `.as_bytes()`
2727    /// on it.)
2728    ///
2729    /// Available via the C wrapper.
2730    ///
2731    /// # Example
2732    /// ```
2733    /// use encoding_rs::Encoding;
2734    ///
2735    /// assert_eq!(Some(encoding_rs::UTF_8), Encoding::for_label(b"utf-8"));
2736    /// assert_eq!(Some(encoding_rs::UTF_8), Encoding::for_label(b"unicode11utf8"));
2737    ///
2738    /// assert_eq!(Some(encoding_rs::ISO_8859_2), Encoding::for_label(b"latin2"));
2739    ///
2740    /// assert_eq!(Some(encoding_rs::UTF_16BE), Encoding::for_label(b"utf-16be"));
2741    ///
2742    /// assert_eq!(None, Encoding::for_label(b"unrecognized label"));
2743    /// ```
2744    pub fn for_label(label: &[u8]) -> Option<&'static Encoding> {
2745        let mut trimmed = [0u8; LONGEST_LABEL_LENGTH];
2746        let mut trimmed_pos = 0usize;
2747        let mut iter = label.into_iter();
2748        // before
2749        loop {
2750            match iter.next() {
2751                None => {
2752                    return None;
2753                }
2754                Some(byte) => {
2755                    // The characters used in labels are:
2756                    // a-z (except q, but excluding it below seems excessive)
2757                    // 0-9
2758                    // . _ - :
2759                    match *byte {
2760                        0x09u8 | 0x0Au8 | 0x0Cu8 | 0x0Du8 | 0x20u8 => {
2761                            continue;
2762                        }
2763                        b'A'..=b'Z' => {
2764                            trimmed[trimmed_pos] = *byte + 0x20u8;
2765                            trimmed_pos = 1usize;
2766                            break;
2767                        }
2768                        b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b':' | b'.' => {
2769                            trimmed[trimmed_pos] = *byte;
2770                            trimmed_pos = 1usize;
2771                            break;
2772                        }
2773                        _ => {
2774                            return None;
2775                        }
2776                    }
2777                }
2778            }
2779        }
2780        // inside
2781        loop {
2782            match iter.next() {
2783                None => {
2784                    break;
2785                }
2786                Some(byte) => {
2787                    match *byte {
2788                        0x09u8 | 0x0Au8 | 0x0Cu8 | 0x0Du8 | 0x20u8 => {
2789                            break;
2790                        }
2791                        b'A'..=b'Z' => {
2792                            if trimmed_pos == LONGEST_LABEL_LENGTH {
2793                                // There's no encoding with a label this long
2794                                return None;
2795                            }
2796                            trimmed[trimmed_pos] = *byte + 0x20u8;
2797                            trimmed_pos += 1usize;
2798                            continue;
2799                        }
2800                        b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b':' | b'.' => {
2801                            if trimmed_pos == LONGEST_LABEL_LENGTH {
2802                                // There's no encoding with a label this long
2803                                return None;
2804                            }
2805                            trimmed[trimmed_pos] = *byte;
2806                            trimmed_pos += 1usize;
2807                            continue;
2808                        }
2809                        _ => {
2810                            return None;
2811                        }
2812                    }
2813                }
2814            }
2815        }
2816        // after
2817        loop {
2818            match iter.next() {
2819                None => {
2820                    break;
2821                }
2822                Some(byte) => {
2823                    match *byte {
2824                        0x09u8 | 0x0Au8 | 0x0Cu8 | 0x0Du8 | 0x20u8 => {
2825                            continue;
2826                        }
2827                        _ => {
2828                            // There's no label with space in the middle
2829                            return None;
2830                        }
2831                    }
2832                }
2833            }
2834        }
2835        let candidate = &trimmed[..trimmed_pos];
2836        match LABELS_SORTED.binary_search_by(|probe| {
2837            let bytes = probe.as_bytes();
2838            let c = bytes.len().cmp(&candidate.len());
2839            if c != Ordering::Equal {
2840                return c;
2841            }
2842            let probe_iter = bytes.iter().rev();
2843            let candidate_iter = candidate.iter().rev();
2844            probe_iter.cmp(candidate_iter)
2845        }) {
2846            Ok(i) => Some(ENCODINGS_IN_LABEL_SORT[i]),
2847            Err(_) => None,
2848        }
2849    }
2850
2851    /// This method behaves the same as `for_label()`, except when `for_label()`
2852    /// would return `Some(REPLACEMENT)`, this method returns `None` instead.
2853    ///
2854    /// This method is useful in scenarios where a fatal error is required
2855    /// upon invalid label, because in those cases the caller typically wishes
2856    /// to treat the labels that map to the replacement encoding as fatal
2857    /// errors, too.
2858    ///
2859    /// It is not OK to use this method when the action upon the method returning
2860    /// `None` is to use a fallback encoding (e.g. `WINDOWS_1252`). In such a
2861    /// case, the `for_label()` method should be used instead in order to avoid
2862    /// unsafe fallback for labels that `for_label()` maps to `Some(REPLACEMENT)`.
2863    ///
2864    /// Available via the C wrapper.
2865    #[inline]
2866    pub fn for_label_no_replacement(label: &[u8]) -> Option<&'static Encoding> {
2867        match Encoding::for_label(label) {
2868            None => None,
2869            Some(encoding) => {
2870                if encoding == REPLACEMENT {
2871                    None
2872                } else {
2873                    Some(encoding)
2874                }
2875            }
2876        }
2877    }
2878
2879    /// Performs non-incremental BOM sniffing.
2880    ///
2881    /// The argument must either be a buffer representing the entire input
2882    /// stream (non-streaming case) or a buffer representing at least the first
2883    /// three bytes of the input stream (streaming case).
2884    ///
2885    /// Returns `Some((UTF_8, 3))`, `Some((UTF_16LE, 2))` or
2886    /// `Some((UTF_16BE, 2))` if the argument starts with the UTF-8, UTF-16LE
2887    /// or UTF-16BE BOM or `None` otherwise.
2888    ///
2889    /// Available via the C wrapper.
2890    #[inline]
2891    pub fn for_bom(buffer: &[u8]) -> Option<(&'static Encoding, usize)> {
2892        if buffer.starts_with(b"\xEF\xBB\xBF") {
2893            Some((UTF_8, 3))
2894        } else if buffer.starts_with(b"\xFF\xFE") {
2895            Some((UTF_16LE, 2))
2896        } else if buffer.starts_with(b"\xFE\xFF") {
2897            Some((UTF_16BE, 2))
2898        } else {
2899            None
2900        }
2901    }
2902
2903    /// Returns the name of this encoding.
2904    ///
2905    /// This name is appropriate to return as-is from the DOM
2906    /// `document.characterSet` property.
2907    ///
2908    /// Available via the C wrapper.
2909    #[inline]
2910    pub fn name(&'static self) -> &'static str {
2911        self.name
2912    }
2913
2914    /// Checks whether the _output encoding_ of this encoding can encode every
2915    /// `char`. (Only true if the output encoding is UTF-8.)
2916    ///
2917    /// Available via the C wrapper.
2918    #[inline]
2919    pub fn can_encode_everything(&'static self) -> bool {
2920        self.output_encoding() == UTF_8
2921    }
2922
2923    /// Checks whether the bytes 0x00...0x7F map exclusively to the characters
2924    /// U+0000...U+007F and vice versa.
2925    ///
2926    /// Available via the C wrapper.
2927    #[inline]
2928    pub fn is_ascii_compatible(&'static self) -> bool {
2929        !(self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE || self == ISO_2022_JP)
2930    }
2931
2932    /// Checks whether this encoding maps one byte to one Basic Multilingual
2933    /// Plane code point (i.e. byte length equals decoded UTF-16 length) and
2934    /// vice versa (for mappable characters).
2935    ///
2936    /// `true` iff this encoding is on the list of [Legacy single-byte
2937    /// encodings](https://encoding.spec.whatwg.org/#legacy-single-byte-encodings)
2938    /// in the spec or x-user-defined.
2939    ///
2940    /// Available via the C wrapper.
2941    #[inline]
2942    pub fn is_single_byte(&'static self) -> bool {
2943        self.variant.is_single_byte()
2944    }
2945
2946    /// Checks whether the bytes 0x00...0x7F map mostly to the characters
2947    /// U+0000...U+007F and vice versa.
2948    #[cfg(feature = "alloc")]
2949    #[inline]
2950    fn is_potentially_borrowable(&'static self) -> bool {
2951        !(self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE)
2952    }
2953
2954    /// Returns the _output encoding_ of this encoding. This is UTF-8 for
2955    /// UTF-16BE, UTF-16LE, and replacement and the encoding itself otherwise.
2956    ///
2957    /// _Note:_ The _output encoding_ concept is needed for form submission and
2958    /// error handling in the query strings of URLs in the Web Platform.
2959    ///
2960    /// Available via the C wrapper.
2961    #[inline]
2962    pub fn output_encoding(&'static self) -> &'static Encoding {
2963        if self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE {
2964            UTF_8
2965        } else {
2966            self
2967        }
2968    }
2969
2970    /// Decode complete input to `Cow<'a, str>` _with BOM sniffing_ and with
2971    /// malformed sequences replaced with the REPLACEMENT CHARACTER when the
2972    /// entire input is available as a single buffer (i.e. the end of the
2973    /// buffer marks the end of the stream).
2974    ///
2975    /// The BOM, if any, does not appear in the output.
2976    ///
2977    /// This method implements the (non-streaming version of) the
2978    /// [_decode_](https://encoding.spec.whatwg.org/#decode) spec concept.
2979    ///
2980    /// The second item in the returned tuple is the encoding that was actually
2981    /// used (which may differ from this encoding thanks to BOM sniffing).
2982    ///
2983    /// The third item in the returned tuple indicates whether there were
2984    /// malformed sequences (that were replaced with the REPLACEMENT CHARACTER).
2985    ///
2986    /// _Note:_ It is wrong to use this when the input buffer represents only
2987    /// a segment of the input instead of the whole input. Use `new_decoder()`
2988    /// when decoding segmented input.
2989    ///
2990    /// This method performs a one or two heap allocations for the backing
2991    /// buffer of the `String` when unable to borrow. (One allocation if not
2992    /// errors and potentially another one in the presence of errors.) The
2993    /// first allocation assumes jemalloc and may not be optimal with
2994    /// allocators that do not use power-of-two buckets. A borrow is performed
2995    /// if decoding UTF-8 and the input is valid UTF-8, if decoding an
2996    /// ASCII-compatible encoding and the input is ASCII-only, or when decoding
2997    /// ISO-2022-JP and the input is entirely in the ASCII state without state
2998    /// transitions.
2999    ///
3000    /// # Panics
3001    ///
3002    /// If the size calculation for a heap-allocated backing buffer overflows
3003    /// `usize`.
3004    ///
3005    /// Available to Rust only and only with the `alloc` feature enabled (enabled
3006    /// by default).
3007    #[cfg(feature = "alloc")]
3008    #[inline]
3009    pub fn decode<'a>(&'static self, bytes: &'a [u8]) -> (Cow<'a, str>, &'static Encoding, bool) {
3010        let (encoding, without_bom) = match Encoding::for_bom(bytes) {
3011            Some((encoding, bom_length)) => (encoding, &bytes[bom_length..]),
3012            None => (self, bytes),
3013        };
3014        let (cow, had_errors) = encoding.decode_without_bom_handling(without_bom);
3015        (cow, encoding, had_errors)
3016    }
3017
3018    /// Decode complete input to `Cow<'a, str>` _with BOM removal_ and with
3019    /// malformed sequences replaced with the REPLACEMENT CHARACTER when the
3020    /// entire input is available as a single buffer (i.e. the end of the
3021    /// buffer marks the end of the stream).
3022    ///
3023    /// Only an initial byte sequence that is a BOM for this encoding is removed.
3024    ///
3025    /// When invoked on `UTF_8`, this method implements the (non-streaming
3026    /// version of) the
3027    /// [_UTF-8 decode_](https://encoding.spec.whatwg.org/#utf-8-decode) spec
3028    /// concept.
3029    ///
3030    /// The second item in the returned pair indicates whether there were
3031    /// malformed sequences (that were replaced with the REPLACEMENT CHARACTER).
3032    ///
3033    /// _Note:_ It is wrong to use this when the input buffer represents only
3034    /// a segment of the input instead of the whole input. Use
3035    /// `new_decoder_with_bom_removal()` when decoding segmented input.
3036    ///
3037    /// This method performs a one or two heap allocations for the backing
3038    /// buffer of the `String` when unable to borrow. (One allocation if not
3039    /// errors and potentially another one in the presence of errors.) The
3040    /// first allocation assumes jemalloc and may not be optimal with
3041    /// allocators that do not use power-of-two buckets. A borrow is performed
3042    /// if decoding UTF-8 and the input is valid UTF-8, if decoding an
3043    /// ASCII-compatible encoding and the input is ASCII-only, or when decoding
3044    /// ISO-2022-JP and the input is entirely in the ASCII state without state
3045    /// transitions.
3046    ///
3047    /// # Panics
3048    ///
3049    /// If the size calculation for a heap-allocated backing buffer overflows
3050    /// `usize`.
3051    ///
3052    /// Available to Rust only and only with the `alloc` feature enabled (enabled
3053    /// by default).
3054    #[cfg(feature = "alloc")]
3055    #[inline]
3056    pub fn decode_with_bom_removal<'a>(&'static self, bytes: &'a [u8]) -> (Cow<'a, str>, bool) {
3057        let without_bom = if self == UTF_8 && bytes.starts_with(b"\xEF\xBB\xBF") {
3058            &bytes[3..]
3059        } else if (self == UTF_16LE && bytes.starts_with(b"\xFF\xFE"))
3060            || (self == UTF_16BE && bytes.starts_with(b"\xFE\xFF"))
3061        {
3062            &bytes[2..]
3063        } else {
3064            bytes
3065        };
3066        self.decode_without_bom_handling(without_bom)
3067    }
3068
3069    /// Decode complete input to `Cow<'a, str>` _without BOM handling_ and
3070    /// with malformed sequences replaced with the REPLACEMENT CHARACTER when
3071    /// the entire input is available as a single buffer (i.e. the end of the
3072    /// buffer marks the end of the stream).
3073    ///
3074    /// When invoked on `UTF_8`, this method implements the (non-streaming
3075    /// version of) the
3076    /// [_UTF-8 decode without BOM_](https://encoding.spec.whatwg.org/#utf-8-decode-without-bom)
3077    /// spec concept.
3078    ///
3079    /// The second item in the returned pair indicates whether there were
3080    /// malformed sequences (that were replaced with the REPLACEMENT CHARACTER).
3081    ///
3082    /// _Note:_ It is wrong to use this when the input buffer represents only
3083    /// a segment of the input instead of the whole input. Use
3084    /// `new_decoder_without_bom_handling()` when decoding segmented input.
3085    ///
3086    /// This method performs a one or two heap allocations for the backing
3087    /// buffer of the `String` when unable to borrow. (One allocation if not
3088    /// errors and potentially another one in the presence of errors.) The
3089    /// first allocation assumes jemalloc and may not be optimal with
3090    /// allocators that do not use power-of-two buckets. A borrow is performed
3091    /// if decoding UTF-8 and the input is valid UTF-8, if decoding an
3092    /// ASCII-compatible encoding and the input is ASCII-only, or when decoding
3093    /// ISO-2022-JP and the input is entirely in the ASCII state without state
3094    /// transitions.
3095    ///
3096    /// # Panics
3097    ///
3098    /// If the size calculation for a heap-allocated backing buffer overflows
3099    /// `usize`.
3100    ///
3101    /// Available to Rust only and only with the `alloc` feature enabled (enabled
3102    /// by default).
3103    #[cfg(feature = "alloc")]
3104    pub fn decode_without_bom_handling<'a>(&'static self, bytes: &'a [u8]) -> (Cow<'a, str>, bool) {
3105        let (mut decoder, mut string, mut total_read) = if self.is_potentially_borrowable() {
3106            let valid_up_to = if self == UTF_8 {
3107                utf8_valid_up_to(bytes)
3108            } else if self == ISO_2022_JP {
3109                iso_2022_jp_ascii_valid_up_to(bytes)
3110            } else {
3111                ascii_valid_up_to(bytes)
3112            };
3113            if valid_up_to == bytes.len() {
3114                let str: &str = unsafe { core::str::from_utf8_unchecked(bytes) };
3115                return (Cow::Borrowed(str), false);
3116            }
3117            let decoder = self.new_decoder_without_bom_handling();
3118
3119            let rounded_without_replacement = checked_next_power_of_two(checked_add(
3120                valid_up_to,
3121                decoder.max_utf8_buffer_length_without_replacement(bytes.len() - valid_up_to),
3122            ));
3123            let with_replacement = checked_add(
3124                valid_up_to,
3125                decoder.max_utf8_buffer_length(bytes.len() - valid_up_to),
3126            );
3127            let mut string = String::with_capacity(
3128                checked_min(rounded_without_replacement, with_replacement).unwrap(),
3129            );
3130            unsafe {
3131                let vec = string.as_mut_vec();
3132                vec.set_len(valid_up_to);
3133                core::ptr::copy_nonoverlapping(bytes.as_ptr(), vec.as_mut_ptr(), valid_up_to);
3134            }
3135            (decoder, string, valid_up_to)
3136        } else {
3137            let decoder = self.new_decoder_without_bom_handling();
3138            let rounded_without_replacement = checked_next_power_of_two(
3139                decoder.max_utf8_buffer_length_without_replacement(bytes.len()),
3140            );
3141            let with_replacement = decoder.max_utf8_buffer_length(bytes.len());
3142            let string = String::with_capacity(
3143                checked_min(rounded_without_replacement, with_replacement).unwrap(),
3144            );
3145            (decoder, string, 0)
3146        };
3147
3148        let mut total_had_errors = false;
3149        loop {
3150            let (result, read, had_errors) =
3151                decoder.decode_to_string(&bytes[total_read..], &mut string, true);
3152            total_read += read;
3153            total_had_errors |= had_errors;
3154            match result {
3155                CoderResult::InputEmpty => {
3156                    debug_assert_eq!(total_read, bytes.len());
3157                    return (Cow::Owned(string), total_had_errors);
3158                }
3159                CoderResult::OutputFull => {
3160                    // Allocate for the worst case. That is, we should come
3161                    // here at most once per invocation of this method.
3162                    let needed = decoder.max_utf8_buffer_length(bytes.len() - total_read);
3163                    string.reserve(needed.unwrap());
3164                }
3165            }
3166        }
3167    }
3168
3169    /// Decode complete input to `Cow<'a, str>` _without BOM handling_ and
3170    /// _with malformed sequences treated as fatal_ when the entire input is
3171    /// available as a single buffer (i.e. the end of the buffer marks the end
3172    /// of the stream).
3173    ///
3174    /// When invoked on `UTF_8`, this method implements the (non-streaming
3175    /// version of) the
3176    /// [_UTF-8 decode without BOM or fail_](https://encoding.spec.whatwg.org/#utf-8-decode-without-bom-or-fail)
3177    /// spec concept.
3178    ///
3179    /// Returns `None` if a malformed sequence was encountered and the result
3180    /// of the decode as `Some(String)` otherwise.
3181    ///
3182    /// _Note:_ It is wrong to use this when the input buffer represents only
3183    /// a segment of the input instead of the whole input. Use
3184    /// `new_decoder_without_bom_handling()` when decoding segmented input.
3185    ///
3186    /// This method performs a single heap allocation for the backing
3187    /// buffer of the `String` when unable to borrow. A borrow is performed if
3188    /// decoding UTF-8 and the input is valid UTF-8, if decoding an
3189    /// ASCII-compatible encoding and the input is ASCII-only, or when decoding
3190    /// ISO-2022-JP and the input is entirely in the ASCII state without state
3191    /// transitions.
3192    ///
3193    /// # Panics
3194    ///
3195    /// If the size calculation for a heap-allocated backing buffer overflows
3196    /// `usize`.
3197    ///
3198    /// Available to Rust only and only with the `alloc` feature enabled (enabled
3199    /// by default).
3200    #[cfg(feature = "alloc")]
3201    pub fn decode_without_bom_handling_and_without_replacement<'a>(
3202        &'static self,
3203        bytes: &'a [u8],
3204    ) -> Option<Cow<'a, str>> {
3205        if self == UTF_8 {
3206            let valid_up_to = utf8_valid_up_to(bytes);
3207            if valid_up_to == bytes.len() {
3208                let str: &str = unsafe { core::str::from_utf8_unchecked(bytes) };
3209                return Some(Cow::Borrowed(str));
3210            }
3211            return None;
3212        }
3213        let (mut decoder, mut string, input) = if self.is_potentially_borrowable() {
3214            let valid_up_to = if self == ISO_2022_JP {
3215                iso_2022_jp_ascii_valid_up_to(bytes)
3216            } else {
3217                ascii_valid_up_to(bytes)
3218            };
3219            if valid_up_to == bytes.len() {
3220                let str: &str = unsafe { core::str::from_utf8_unchecked(bytes) };
3221                return Some(Cow::Borrowed(str));
3222            }
3223            let decoder = self.new_decoder_without_bom_handling();
3224            let mut string = String::with_capacity(
3225                checked_add(
3226                    valid_up_to,
3227                    decoder.max_utf8_buffer_length_without_replacement(bytes.len() - valid_up_to),
3228                )
3229                .unwrap(),
3230            );
3231            unsafe {
3232                let vec = string.as_mut_vec();
3233                vec.set_len(valid_up_to);
3234                core::ptr::copy_nonoverlapping(bytes.as_ptr(), vec.as_mut_ptr(), valid_up_to);
3235            }
3236            (decoder, string, &bytes[valid_up_to..])
3237        } else {
3238            let decoder = self.new_decoder_without_bom_handling();
3239            let string = String::with_capacity(
3240                decoder
3241                    .max_utf8_buffer_length_without_replacement(bytes.len())
3242                    .unwrap(),
3243            );
3244            (decoder, string, bytes)
3245        };
3246        let (result, read) = decoder.decode_to_string_without_replacement(input, &mut string, true);
3247        match result {
3248            DecoderResult::InputEmpty => {
3249                debug_assert_eq!(read, input.len());
3250                Some(Cow::Owned(string))
3251            }
3252            DecoderResult::Malformed(_, _) => None,
3253            DecoderResult::OutputFull => unreachable!(),
3254        }
3255    }
3256
3257    /// Encode complete input to `Cow<'a, [u8]>` using the
3258    /// [_output encoding_](Encoding::output_encoding) of this encoding with
3259    /// unmappable characters replaced with decimal numeric character references
3260    /// when the entire input is available as a single buffer (i.e. the end of
3261    /// the buffer marks the end of the stream).
3262    ///
3263    /// This method implements the (non-streaming version of) the
3264    /// [_encode_](https://encoding.spec.whatwg.org/#encode) spec concept. For
3265    /// the [_UTF-8 encode_](https://encoding.spec.whatwg.org/#utf-8-encode)
3266    /// spec concept, it is slightly more efficient to use
3267    /// <code><var>string</var>.as_bytes()</code> instead of invoking this
3268    /// method on `UTF_8`.
3269    ///
3270    /// The second item in the returned tuple is the encoding that was actually
3271    /// used (*which may differ from this encoding thanks to some encodings
3272    /// having UTF-8 as their output encoding*).
3273    ///
3274    /// The third item in the returned tuple indicates whether there were
3275    /// unmappable characters (that were replaced with HTML numeric character
3276    /// references).
3277    ///
3278    /// _Note:_ It is wrong to use this when the input buffer represents only
3279    /// a segment of the input instead of the whole input. Use `new_encoder()`
3280    /// when encoding segmented output.
3281    ///
3282    /// When encoding to UTF-8 or when encoding an ASCII-only input to a
3283    /// ASCII-compatible encoding, this method returns a borrow of the input
3284    /// without a heap allocation. Otherwise, this method performs a single
3285    /// heap allocation for the backing buffer of the `Vec<u8>` if there are no
3286    /// unmappable characters and potentially multiple heap allocations if
3287    /// there are. These allocations are tuned for jemalloc and may not be
3288    /// optimal when using a different allocator that doesn't use power-of-two
3289    /// buckets.
3290    ///
3291    /// # Panics
3292    ///
3293    /// If the size calculation for a heap-allocated backing buffer overflows
3294    /// `usize`.
3295    ///
3296    /// Available to Rust only and only with the `alloc` feature enabled (enabled
3297    /// by default).
3298    #[cfg(feature = "alloc")]
3299    pub fn encode<'a>(&'static self, string: &'a str) -> (Cow<'a, [u8]>, &'static Encoding, bool) {
3300        let output_encoding = self.output_encoding();
3301        if output_encoding == UTF_8 {
3302            return (Cow::Borrowed(string.as_bytes()), output_encoding, false);
3303        }
3304        debug_assert!(output_encoding.is_potentially_borrowable());
3305        let bytes = string.as_bytes();
3306        let valid_up_to = if output_encoding == ISO_2022_JP {
3307            iso_2022_jp_ascii_valid_up_to(bytes)
3308        } else {
3309            ascii_valid_up_to(bytes)
3310        };
3311        if valid_up_to == bytes.len() {
3312            return (Cow::Borrowed(bytes), output_encoding, false);
3313        }
3314        let mut encoder = output_encoding.new_encoder();
3315        let mut vec: Vec<u8> = Vec::with_capacity(
3316            (checked_add(
3317                valid_up_to,
3318                encoder.max_buffer_length_from_utf8_if_no_unmappables(string.len() - valid_up_to),
3319            ))
3320            .unwrap()
3321            .next_power_of_two(),
3322        );
3323        unsafe {
3324            vec.set_len(valid_up_to);
3325            core::ptr::copy_nonoverlapping(bytes.as_ptr(), vec.as_mut_ptr(), valid_up_to);
3326        }
3327        let mut total_read = valid_up_to;
3328        let mut total_had_errors = false;
3329        loop {
3330            let (result, read, had_errors) =
3331                encoder.encode_from_utf8_to_vec(&string[total_read..], &mut vec, true);
3332            total_read += read;
3333            total_had_errors |= had_errors;
3334            match result {
3335                CoderResult::InputEmpty => {
3336                    debug_assert_eq!(total_read, string.len());
3337                    return (Cow::Owned(vec), output_encoding, total_had_errors);
3338                }
3339                CoderResult::OutputFull => {
3340                    // reserve_exact wants to know how much more on top of current
3341                    // length--not current capacity.
3342                    let needed = encoder
3343                        .max_buffer_length_from_utf8_if_no_unmappables(string.len() - total_read);
3344                    let rounded = (checked_add(vec.capacity(), needed))
3345                        .unwrap()
3346                        .next_power_of_two();
3347                    let additional = rounded - vec.len();
3348                    vec.reserve_exact(additional);
3349                }
3350            }
3351        }
3352    }
3353
3354    fn new_variant_decoder(&'static self) -> VariantDecoder {
3355        self.variant.new_variant_decoder()
3356    }
3357
3358    /// Instantiates a new decoder for this encoding with BOM sniffing enabled.
3359    ///
3360    /// BOM sniffing may cause the returned decoder to morph into a decoder
3361    /// for UTF-8, UTF-16LE or UTF-16BE instead of this encoding. The BOM
3362    /// does not appear in the output.
3363    ///
3364    /// Available via the C wrapper.
3365    #[inline]
3366    pub fn new_decoder(&'static self) -> Decoder {
3367        Decoder::new(self, self.new_variant_decoder(), BomHandling::Sniff)
3368    }
3369
3370    /// Instantiates a new decoder for this encoding with BOM removal.
3371    ///
3372    /// If the input starts with bytes that are the BOM for this encoding,
3373    /// those bytes are removed. However, the decoder never morphs into a
3374    /// decoder for another encoding: A BOM for another encoding is treated as
3375    /// (potentially malformed) input to the decoding algorithm for this
3376    /// encoding.
3377    ///
3378    /// Available via the C wrapper.
3379    #[inline]
3380    pub fn new_decoder_with_bom_removal(&'static self) -> Decoder {
3381        Decoder::new(self, self.new_variant_decoder(), BomHandling::Remove)
3382    }
3383
3384    /// Instantiates a new decoder for this encoding with BOM handling disabled.
3385    ///
3386    /// If the input starts with bytes that look like a BOM, those bytes are
3387    /// not treated as a BOM. (Hence, the decoder never morphs into a decoder
3388    /// for another encoding.)
3389    ///
3390    /// _Note:_ If the caller has performed BOM sniffing on its own but has not
3391    /// removed the BOM, the caller should use `new_decoder_with_bom_removal()`
3392    /// instead of this method to cause the BOM to be removed.
3393    ///
3394    /// Available via the C wrapper.
3395    #[inline]
3396    pub fn new_decoder_without_bom_handling(&'static self) -> Decoder {
3397        Decoder::new(self, self.new_variant_decoder(), BomHandling::Off)
3398    }
3399
3400    /// Instantiates a new encoder for the [_output encoding_](Encoding::output_encoding)
3401    /// of this encoding.
3402    ///
3403    /// _Note:_ The output encoding of UTF-16BE, UTF-16LE, and replacement is UTF-8. There
3404    /// is no encoder for UTF-16BE, UTF-16LE, and replacement themselves.
3405    ///
3406    /// Available via the C wrapper.
3407    #[inline]
3408    pub fn new_encoder(&'static self) -> Encoder {
3409        let enc = self.output_encoding();
3410        enc.variant.new_encoder(enc)
3411    }
3412
3413    /// Validates UTF-8.
3414    ///
3415    /// Returns the index of the first byte that makes the input malformed as
3416    /// UTF-8 or the length of the slice if the slice is entirely valid.
3417    ///
3418    /// This is currently faster than the corresponding standard library
3419    /// functionality. If this implementation gets upstreamed to the standard
3420    /// library, this method may be removed in the future.
3421    ///
3422    /// Available via the C wrapper.
3423    pub fn utf8_valid_up_to(bytes: &[u8]) -> usize {
3424        utf8_valid_up_to(bytes)
3425    }
3426
3427    /// Validates ASCII.
3428    ///
3429    /// Returns the index of the first byte that makes the input malformed as
3430    /// ASCII or the length of the slice if the slice is entirely valid.
3431    ///
3432    /// Available via the C wrapper.
3433    pub fn ascii_valid_up_to(bytes: &[u8]) -> usize {
3434        ascii_valid_up_to(bytes)
3435    }
3436
3437    /// Validates ISO-2022-JP ASCII-state data.
3438    ///
3439    /// Returns the index of the first byte that makes the input not
3440    /// representable in the ASCII state of ISO-2022-JP or the length of the
3441    /// slice if the slice is entirely representable in the ASCII state of
3442    /// ISO-2022-JP.
3443    ///
3444    /// Available via the C wrapper.
3445    pub fn iso_2022_jp_ascii_valid_up_to(bytes: &[u8]) -> usize {
3446        iso_2022_jp_ascii_valid_up_to(bytes)
3447    }
3448}
3449
3450impl PartialEq for Encoding {
3451    #[inline]
3452    fn eq(&self, other: &Encoding) -> bool {
3453        (self as *const Encoding) == (other as *const Encoding)
3454    }
3455}
3456
3457impl Eq for Encoding {}
3458
3459#[cfg(test)]
3460impl PartialOrd for Encoding {
3461    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
3462        (self as *const Encoding as usize).partial_cmp(&(other as *const Encoding as usize))
3463    }
3464}
3465
3466#[cfg(test)]
3467impl Ord for Encoding {
3468    fn cmp(&self, other: &Self) -> Ordering {
3469        (self as *const Encoding as usize).cmp(&(other as *const Encoding as usize))
3470    }
3471}
3472
3473impl Hash for Encoding {
3474    #[inline]
3475    fn hash<H: Hasher>(&self, state: &mut H) {
3476        (self as *const Encoding).hash(state);
3477    }
3478}
3479
3480impl core::fmt::Debug for Encoding {
3481    #[inline]
3482    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
3483        write!(f, "Encoding {{ {} }}", self.name)
3484    }
3485}
3486
3487#[cfg(feature = "serde")]
3488impl Serialize for Encoding {
3489    #[inline]
3490    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
3491    where
3492        S: Serializer,
3493    {
3494        serializer.serialize_str(self.name)
3495    }
3496}
3497
3498#[cfg(feature = "serde")]
3499struct EncodingVisitor;
3500
3501#[cfg(feature = "serde")]
3502impl<'de> Visitor<'de> for EncodingVisitor {
3503    type Value = &'static Encoding;
3504
3505    fn expecting(&self, formatter: &mut core::fmt::Formatter) -> core::fmt::Result {
3506        formatter.write_str("a valid encoding label")
3507    }
3508
3509    fn visit_str<E>(self, value: &str) -> Result<&'static Encoding, E>
3510    where
3511        E: serde::de::Error,
3512    {
3513        if let Some(enc) = Encoding::for_label(value.as_bytes()) {
3514            Ok(enc)
3515        } else {
3516            Err(E::custom(alloc::format!(
3517                "invalid encoding label: {}",
3518                value
3519            )))
3520        }
3521    }
3522}
3523
3524#[cfg(feature = "serde")]
3525impl<'de> Deserialize<'de> for &'static Encoding {
3526    fn deserialize<D>(deserializer: D) -> Result<&'static Encoding, D::Error>
3527    where
3528        D: Deserializer<'de>,
3529    {
3530        deserializer.deserialize_str(EncodingVisitor)
3531    }
3532}
3533
3534/// Tracks the life cycle of a decoder from BOM sniffing to conversion to end.
3535#[derive(PartialEq, Debug, Copy, Clone)]
3536enum DecoderLifeCycle {
3537    /// The decoder has seen no input yet.
3538    AtStart,
3539    /// The decoder has seen no input yet but expects UTF-8.
3540    AtUtf8Start,
3541    /// The decoder has seen no input yet but expects UTF-16BE.
3542    AtUtf16BeStart,
3543    /// The decoder has seen no input yet but expects UTF-16LE.
3544    AtUtf16LeStart,
3545    /// The decoder has seen EF.
3546    SeenUtf8First,
3547    /// The decoder has seen EF, BB.
3548    SeenUtf8Second,
3549    /// The decoder has seen FE.
3550    SeenUtf16BeFirst,
3551    /// The decoder has seen FF.
3552    SeenUtf16LeFirst,
3553    /// Saw EF, BB but not BF, there was a buffer boundary after BB and the
3554    /// underlying decoder reported EF as an error, so we need to remember to
3555    /// push BB before the next buffer.
3556    ConvertingWithPendingBB,
3557    /// No longer looking for a BOM and EOF not yet seen.
3558    Converting,
3559    /// EOF has been seen.
3560    Finished,
3561}
3562
3563/// Communicate the BOM handling mode.
3564#[derive(Debug, Copy, Clone)]
3565enum BomHandling {
3566    /// Don't handle the BOM
3567    Off,
3568    /// Sniff for UTF-8, UTF-16BE or UTF-16LE BOM
3569    Sniff,
3570    /// Remove the BOM only if it's the BOM for this encoding
3571    Remove,
3572}
3573
3574/// Result of a (potentially partial) decode or encode operation with
3575/// replacement.
3576#[must_use]
3577#[derive(Debug, PartialEq, Eq)]
3578pub enum CoderResult {
3579    /// The input was exhausted.
3580    ///
3581    /// If this result was returned from a call where `last` was `true`, the
3582    /// conversion process has completed. Otherwise, the caller should call a
3583    /// decode or encode method again with more input.
3584    InputEmpty,
3585
3586    /// The converter cannot produce another unit of output, because the output
3587    /// buffer does not have enough space left.
3588    ///
3589    /// The caller must provide more output space upon the next call and re-push
3590    /// the remaining input to the converter.
3591    OutputFull,
3592}
3593
3594/// Result of a (potentially partial) decode operation without replacement.
3595#[must_use]
3596#[derive(Debug, PartialEq, Eq)]
3597pub enum DecoderResult {
3598    /// The input was exhausted.
3599    ///
3600    /// If this result was returned from a call where `last` was `true`, the
3601    /// decoding process has completed. Otherwise, the caller should call a
3602    /// decode method again with more input.
3603    InputEmpty,
3604
3605    /// The decoder cannot produce another unit of output, because the output
3606    /// buffer does not have enough space left.
3607    ///
3608    /// The caller must provide more output space upon the next call and re-push
3609    /// the remaining input to the decoder.
3610    OutputFull,
3611
3612    /// The decoder encountered a malformed byte sequence.
3613    ///
3614    /// The caller must either treat this as a fatal error or must append one
3615    /// REPLACEMENT CHARACTER (U+FFFD) to the output and then re-push the
3616    /// the remaining input to the decoder.
3617    ///
3618    /// The first wrapped integer indicates the length of the malformed byte
3619    /// sequence. The second wrapped integer indicates the number of bytes
3620    /// that were consumed after the malformed sequence. If the second
3621    /// integer is zero, the last byte that was consumed is the last byte of
3622    /// the malformed sequence. Note that the malformed bytes may have been part
3623    /// of an earlier input buffer.
3624    ///
3625    /// The first wrapped integer can have values 1, 2, 3 or 4. The second
3626    /// wrapped integer can have values 0, 1, 2 or 3. The worst-case sum
3627    /// of the two is 6, which happens with ISO-2022-JP.
3628    Malformed(u8, u8), // u8 instead of usize to avoid useless bloat
3629}
3630
3631/// A converter that decodes a byte stream into Unicode according to a
3632/// character encoding in a streaming (incremental) manner.
3633///
3634/// The various `decode_*` methods take an input buffer (`src`) and an output
3635/// buffer `dst` both of which are caller-allocated. There are variants for
3636/// both UTF-8 and UTF-16 output buffers.
3637///
3638/// A `decode_*` method decodes bytes from `src` into Unicode characters stored
3639/// into `dst` until one of the following three things happens:
3640///
3641/// 1. A malformed byte sequence is encountered (`*_without_replacement`
3642///    variants only).
3643///
3644/// 2. The output buffer has been filled so near capacity that the decoder
3645///    cannot be sure that processing an additional byte of input wouldn't
3646///    cause so much output that the output buffer would overflow.
3647///
3648/// 3. All the input bytes have been processed.
3649///
3650/// The `decode_*` method then returns tuple of a status indicating which one
3651/// of the three reasons to return happened, how many input bytes were read,
3652/// how many output code units (`u8` when decoding into UTF-8 and `u16`
3653/// when decoding to UTF-16) were written (except when decoding into `String`,
3654/// whose length change indicates this), and in the case of the
3655/// variants performing replacement, a boolean indicating whether an error was
3656/// replaced with the REPLACEMENT CHARACTER during the call.
3657///
3658/// The number of bytes "written" is what's logically written. Garbage may be
3659/// written in the output buffer beyond the point logically written to.
3660/// Therefore, if you wish to decode into an `&mut str`, you should use the
3661/// methods that take an `&mut str` argument instead of the ones that take an
3662/// `&mut [u8]` argument. The former take care of overwriting the trailing
3663/// garbage to ensure the UTF-8 validity of the `&mut str` as a whole, but the
3664/// latter don't.
3665///
3666/// In the case of the `*_without_replacement` variants, the status is a
3667/// [`DecoderResult`][1] enumeration (possibilities `Malformed`, `OutputFull` and
3668/// `InputEmpty` corresponding to the three cases listed above).
3669///
3670/// In the case of methods whose name does not end with
3671/// `*_without_replacement`, malformed sequences are automatically replaced
3672/// with the REPLACEMENT CHARACTER and errors do not cause the methods to
3673/// return early.
3674///
3675/// When decoding to UTF-8, the output buffer must have at least 4 bytes of
3676/// space. When decoding to UTF-16, the output buffer must have at least two
3677/// UTF-16 code units (`u16`) of space.
3678///
3679/// When decoding to UTF-8 without replacement, the methods are guaranteed
3680/// not to return indicating that more output space is needed if the length
3681/// of the output buffer is at least the length returned by
3682/// [`max_utf8_buffer_length_without_replacement()`][2]. When decoding to UTF-8
3683/// with replacement, the length of the output buffer that guarantees the
3684/// methods not to return indicating that more output space is needed is given
3685/// by [`max_utf8_buffer_length()`][3]. When decoding to UTF-16 with
3686/// or without replacement, the length of the output buffer that guarantees
3687/// the methods not to return indicating that more output space is needed is
3688/// given by [`max_utf16_buffer_length()`][4].
3689///
3690/// The output written into `dst` is guaranteed to be valid UTF-8 or UTF-16,
3691/// and the output after each `decode_*` call is guaranteed to consist of
3692/// complete characters. (I.e. the code unit sequence for the last character is
3693/// guaranteed not to be split across output buffers.)
3694///
3695/// The boolean argument `last` indicates that the end of the stream is reached
3696/// when all the bytes in `src` have been consumed.
3697///
3698/// A `Decoder` object can be used to incrementally decode a byte stream.
3699///
3700/// During the processing of a single stream, the caller must call `decode_*`
3701/// zero or more times with `last` set to `false` and then call `decode_*` at
3702/// least once with `last` set to `true`. If `decode_*` returns `InputEmpty`,
3703/// the processing of the stream has ended. Otherwise, the caller must call
3704/// `decode_*` again with `last` set to `true` (or treat a `Malformed` result as
3705///  a fatal error).
3706///
3707/// Once the stream has ended, the `Decoder` object must not be used anymore.
3708/// That is, you need to create another one to process another stream.
3709///
3710/// When the decoder returns `OutputFull` or the decoder returns `Malformed` and
3711/// the caller does not wish to treat it as a fatal error, the input buffer
3712/// `src` may not have been completely consumed. In that case, the caller must
3713/// pass the unconsumed contents of `src` to `decode_*` again upon the next
3714/// call.
3715///
3716/// [1]: enum.DecoderResult.html
3717/// [2]: #method.max_utf8_buffer_length_without_replacement
3718/// [3]: #method.max_utf8_buffer_length
3719/// [4]: #method.max_utf16_buffer_length
3720///
3721/// # Infinite loops
3722///
3723/// When converting with a fixed-size output buffer whose size is too small to
3724/// accommodate one character or (when applicable) one numeric character
3725/// reference of output, an infinite loop ensues. When converting with a
3726/// fixed-size output buffer, it generally makes sense to make the buffer
3727/// fairly large (e.g. couple of kilobytes).
3728pub struct Decoder {
3729    encoding: &'static Encoding,
3730    variant: VariantDecoder,
3731    life_cycle: DecoderLifeCycle,
3732}
3733
3734impl Decoder {
3735    fn new(enc: &'static Encoding, decoder: VariantDecoder, sniffing: BomHandling) -> Decoder {
3736        Decoder {
3737            encoding: enc,
3738            variant: decoder,
3739            life_cycle: match sniffing {
3740                BomHandling::Off => DecoderLifeCycle::Converting,
3741                BomHandling::Sniff => DecoderLifeCycle::AtStart,
3742                BomHandling::Remove => {
3743                    if enc == UTF_8 {
3744                        DecoderLifeCycle::AtUtf8Start
3745                    } else if enc == UTF_16BE {
3746                        DecoderLifeCycle::AtUtf16BeStart
3747                    } else if enc == UTF_16LE {
3748                        DecoderLifeCycle::AtUtf16LeStart
3749                    } else {
3750                        DecoderLifeCycle::Converting
3751                    }
3752                }
3753            },
3754        }
3755    }
3756
3757    /// The `Encoding` this `Decoder` is for.
3758    ///
3759    /// BOM sniffing can change the return value of this method during the life
3760    /// of the decoder.
3761    ///
3762    /// Available via the C wrapper.
3763    #[inline]
3764    pub fn encoding(&self) -> &'static Encoding {
3765        self.encoding
3766    }
3767
3768    /// Query the worst-case UTF-8 output size _with replacement_.
3769    ///
3770    /// Returns the size of the output buffer in UTF-8 code units (`u8`)
3771    /// that will not overflow given the current state of the decoder and
3772    /// `byte_length` number of additional input bytes when decoding with
3773    /// errors handled by outputting a REPLACEMENT CHARACTER for each malformed
3774    /// sequence or `None` if `usize` would overflow.
3775    ///
3776    /// Available via the C wrapper.
3777    pub fn max_utf8_buffer_length(&self, byte_length: usize) -> Option<usize> {
3778        // Need to consider a) the decoder morphing due to the BOM and b) a partial
3779        // BOM getting pushed to the underlying decoder.
3780        match self.life_cycle {
3781            DecoderLifeCycle::Converting
3782            | DecoderLifeCycle::AtUtf8Start
3783            | DecoderLifeCycle::AtUtf16LeStart
3784            | DecoderLifeCycle::AtUtf16BeStart => {
3785                return self.variant.max_utf8_buffer_length(byte_length);
3786            }
3787            DecoderLifeCycle::AtStart => {
3788                if let Some(utf8_bom) = checked_add(3, byte_length.checked_mul(3)) {
3789                    if let Some(utf16_bom) = checked_add(
3790                        1,
3791                        checked_mul(3, checked_div(byte_length.checked_add(1), 2)),
3792                    ) {
3793                        let utf_bom = core::cmp::max(utf8_bom, utf16_bom);
3794                        let encoding = self.encoding();
3795                        if encoding == UTF_8 || encoding == UTF_16LE || encoding == UTF_16BE {
3796                            // No need to consider the internal state of the underlying decoder,
3797                            // because it is at start, because no data has reached it yet.
3798                            return Some(utf_bom);
3799                        } else if let Some(non_bom) =
3800                            self.variant.max_utf8_buffer_length(byte_length)
3801                        {
3802                            return Some(core::cmp::max(utf_bom, non_bom));
3803                        }
3804                    }
3805                }
3806            }
3807            DecoderLifeCycle::SeenUtf8First | DecoderLifeCycle::SeenUtf8Second => {
3808                // Add two bytes even when only one byte has been seen,
3809                // because the one byte can become a lead byte in multibyte
3810                // decoders, but only after the decoder has been queried
3811                // for max length, so the decoder's own logic for adding
3812                // one for a pending lead cannot work.
3813                if let Some(sum) = byte_length.checked_add(2) {
3814                    if let Some(utf8_bom) = checked_add(3, sum.checked_mul(3)) {
3815                        if self.encoding() == UTF_8 {
3816                            // No need to consider the internal state of the underlying decoder,
3817                            // because it is at start, because no data has reached it yet.
3818                            return Some(utf8_bom);
3819                        } else if let Some(non_bom) = self.variant.max_utf8_buffer_length(sum) {
3820                            return Some(core::cmp::max(utf8_bom, non_bom));
3821                        }
3822                    }
3823                }
3824            }
3825            DecoderLifeCycle::ConvertingWithPendingBB => {
3826                if let Some(sum) = byte_length.checked_add(2) {
3827                    return self.variant.max_utf8_buffer_length(sum);
3828                }
3829            }
3830            DecoderLifeCycle::SeenUtf16LeFirst | DecoderLifeCycle::SeenUtf16BeFirst => {
3831                // Add two bytes even when only one byte has been seen,
3832                // because the one byte can become a lead byte in multibyte
3833                // decoders, but only after the decoder has been queried
3834                // for max length, so the decoder's own logic for adding
3835                // one for a pending lead cannot work.
3836                if let Some(sum) = byte_length.checked_add(2) {
3837                    if let Some(utf16_bom) =
3838                        checked_add(1, checked_mul(3, checked_div(sum.checked_add(1), 2)))
3839                    {
3840                        let encoding = self.encoding();
3841                        if encoding == UTF_16LE || encoding == UTF_16BE {
3842                            // No need to consider the internal state of the underlying decoder,
3843                            // because it is at start, because no data has reached it yet.
3844                            return Some(utf16_bom);
3845                        } else if let Some(non_bom) = self.variant.max_utf8_buffer_length(sum) {
3846                            return Some(core::cmp::max(utf16_bom, non_bom));
3847                        }
3848                    }
3849                }
3850            }
3851            DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
3852        }
3853        None
3854    }
3855
3856    /// Query the worst-case UTF-8 output size _without replacement_.
3857    ///
3858    /// Returns the size of the output buffer in UTF-8 code units (`u8`)
3859    /// that will not overflow given the current state of the decoder and
3860    /// `byte_length` number of additional input bytes when decoding without
3861    /// replacement error handling or `None` if `usize` would overflow.
3862    ///
3863    /// Note that this value may be too small for the `_with_replacement` case.
3864    /// Use `max_utf8_buffer_length()` for that case.
3865    ///
3866    /// Available via the C wrapper.
3867    pub fn max_utf8_buffer_length_without_replacement(&self, byte_length: usize) -> Option<usize> {
3868        // Need to consider a) the decoder morphing due to the BOM and b) a partial
3869        // BOM getting pushed to the underlying decoder.
3870        match self.life_cycle {
3871            DecoderLifeCycle::Converting
3872            | DecoderLifeCycle::AtUtf8Start
3873            | DecoderLifeCycle::AtUtf16LeStart
3874            | DecoderLifeCycle::AtUtf16BeStart => {
3875                return self
3876                    .variant
3877                    .max_utf8_buffer_length_without_replacement(byte_length);
3878            }
3879            DecoderLifeCycle::AtStart => {
3880                if let Some(utf8_bom) = byte_length.checked_add(3) {
3881                    if let Some(utf16_bom) = checked_add(
3882                        1,
3883                        checked_mul(3, checked_div(byte_length.checked_add(1), 2)),
3884                    ) {
3885                        let utf_bom = core::cmp::max(utf8_bom, utf16_bom);
3886                        let encoding = self.encoding();
3887                        if encoding == UTF_8 || encoding == UTF_16LE || encoding == UTF_16BE {
3888                            // No need to consider the internal state of the underlying decoder,
3889                            // because it is at start, because no data has reached it yet.
3890                            return Some(utf_bom);
3891                        } else if let Some(non_bom) = self
3892                            .variant
3893                            .max_utf8_buffer_length_without_replacement(byte_length)
3894                        {
3895                            return Some(core::cmp::max(utf_bom, non_bom));
3896                        }
3897                    }
3898                }
3899            }
3900            DecoderLifeCycle::SeenUtf8First | DecoderLifeCycle::SeenUtf8Second => {
3901                // Add two bytes even when only one byte has been seen,
3902                // because the one byte can become a lead byte in multibyte
3903                // decoders, but only after the decoder has been queried
3904                // for max length, so the decoder's own logic for adding
3905                // one for a pending lead cannot work.
3906                if let Some(sum) = byte_length.checked_add(2) {
3907                    if let Some(utf8_bom) = sum.checked_add(3) {
3908                        if self.encoding() == UTF_8 {
3909                            // No need to consider the internal state of the underlying decoder,
3910                            // because it is at start, because no data has reached it yet.
3911                            return Some(utf8_bom);
3912                        } else if let Some(non_bom) =
3913                            self.variant.max_utf8_buffer_length_without_replacement(sum)
3914                        {
3915                            return Some(core::cmp::max(utf8_bom, non_bom));
3916                        }
3917                    }
3918                }
3919            }
3920            DecoderLifeCycle::ConvertingWithPendingBB => {
3921                if let Some(sum) = byte_length.checked_add(2) {
3922                    return self.variant.max_utf8_buffer_length_without_replacement(sum);
3923                }
3924            }
3925            DecoderLifeCycle::SeenUtf16LeFirst | DecoderLifeCycle::SeenUtf16BeFirst => {
3926                // Add two bytes even when only one byte has been seen,
3927                // because the one byte can become a lead byte in multibyte
3928                // decoders, but only after the decoder has been queried
3929                // for max length, so the decoder's own logic for adding
3930                // one for a pending lead cannot work.
3931                if let Some(sum) = byte_length.checked_add(2) {
3932                    if let Some(utf16_bom) =
3933                        checked_add(1, checked_mul(3, checked_div(sum.checked_add(1), 2)))
3934                    {
3935                        let encoding = self.encoding();
3936                        if encoding == UTF_16LE || encoding == UTF_16BE {
3937                            // No need to consider the internal state of the underlying decoder,
3938                            // because it is at start, because no data has reached it yet.
3939                            return Some(utf16_bom);
3940                        } else if let Some(non_bom) =
3941                            self.variant.max_utf8_buffer_length_without_replacement(sum)
3942                        {
3943                            return Some(core::cmp::max(utf16_bom, non_bom));
3944                        }
3945                    }
3946                }
3947            }
3948            DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
3949        }
3950        None
3951    }
3952
3953    /// Incrementally decode a byte stream into UTF-8 with malformed sequences
3954    /// replaced with the REPLACEMENT CHARACTER.
3955    ///
3956    /// See the documentation of the struct for documentation for `decode_*`
3957    /// methods collectively.
3958    ///
3959    /// Available via the C wrapper.
3960    pub fn decode_to_utf8(
3961        &mut self,
3962        src: &[u8],
3963        dst: &mut [u8],
3964        last: bool,
3965    ) -> (CoderResult, usize, usize, bool) {
3966        let mut had_errors = false;
3967        let mut total_read = 0usize;
3968        let mut total_written = 0usize;
3969        loop {
3970            let (result, read, written) = self.decode_to_utf8_without_replacement(
3971                &src[total_read..],
3972                &mut dst[total_written..],
3973                last,
3974            );
3975            total_read += read;
3976            total_written += written;
3977            match result {
3978                DecoderResult::InputEmpty => {
3979                    return (
3980                        CoderResult::InputEmpty,
3981                        total_read,
3982                        total_written,
3983                        had_errors,
3984                    );
3985                }
3986                DecoderResult::OutputFull => {
3987                    return (
3988                        CoderResult::OutputFull,
3989                        total_read,
3990                        total_written,
3991                        had_errors,
3992                    );
3993                }
3994                DecoderResult::Malformed(_, _) => {
3995                    had_errors = true;
3996                    // There should always be space for the U+FFFD, because
3997                    // otherwise we'd have gotten OutputFull already.
3998                    // XXX: is the above comment actually true for UTF-8 itself?
3999                    // TODO: Consider having fewer bound checks here.
4000                    dst[total_written] = 0xEFu8;
4001                    total_written += 1;
4002                    dst[total_written] = 0xBFu8;
4003                    total_written += 1;
4004                    dst[total_written] = 0xBDu8;
4005                    total_written += 1;
4006                }
4007            }
4008        }
4009    }
4010
4011    /// Incrementally decode a byte stream into UTF-8 with malformed sequences
4012    /// replaced with the REPLACEMENT CHARACTER with type system signaling
4013    /// of UTF-8 validity.
4014    ///
4015    /// This methods calls `decode_to_utf8` and then zeroes
4016    /// out up to three bytes that aren't logically part of the write in order
4017    /// to retain the UTF-8 validity even for the unwritten part of the buffer.
4018    ///
4019    /// See the documentation of the struct for documentation for `decode_*`
4020    /// methods collectively.
4021    ///
4022    /// Available to Rust only.
4023    pub fn decode_to_str(
4024        &mut self,
4025        src: &[u8],
4026        dst: &mut str,
4027        last: bool,
4028    ) -> (CoderResult, usize, usize, bool) {
4029        let bytes: &mut [u8] = unsafe { dst.as_bytes_mut() };
4030        let (result, read, written, replaced) = self.decode_to_utf8(src, bytes, last);
4031        let len = bytes.len();
4032        let mut trail = written;
4033        // Non-UTF-8 ASCII-compatible decoders may write up to `MAX_STRIDE_SIZE`
4034        // bytes of trailing garbage. No need to optimize non-ASCII-compatible
4035        // encodings to avoid overwriting here.
4036        if self.encoding != UTF_8 {
4037            let max = core::cmp::min(len, trail + ascii::MAX_STRIDE_SIZE);
4038            while trail < max {
4039                bytes[trail] = 0;
4040                trail += 1;
4041            }
4042        }
4043        while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
4044            bytes[trail] = 0;
4045            trail += 1;
4046        }
4047        (result, read, written, replaced)
4048    }
4049
4050    /// Incrementally decode a byte stream into UTF-8 with malformed sequences
4051    /// replaced with the REPLACEMENT CHARACTER using a `String` receiver.
4052    ///
4053    /// Like the others, this method follows the logic that the output buffer is
4054    /// caller-allocated. This method treats the capacity of the `String` as
4055    /// the output limit. That is, this method guarantees not to cause a
4056    /// reallocation of the backing buffer of `String`.
4057    ///
4058    /// The return value is a tuple that contains the `DecoderResult`, the
4059    /// number of bytes read and a boolean indicating whether replacements
4060    /// were done. The number of bytes written is signaled via the length of
4061    /// the `String` changing.
4062    ///
4063    /// See the documentation of the struct for documentation for `decode_*`
4064    /// methods collectively.
4065    ///
4066    /// Available to Rust only and only with the `alloc` feature enabled (enabled
4067    /// by default).
4068    #[cfg(feature = "alloc")]
4069    pub fn decode_to_string(
4070        &mut self,
4071        src: &[u8],
4072        dst: &mut String,
4073        last: bool,
4074    ) -> (CoderResult, usize, bool) {
4075        unsafe {
4076            let vec = dst.as_mut_vec();
4077            let old_len = vec.len();
4078            let capacity = vec.capacity();
4079            vec.set_len(capacity);
4080            let (result, read, written, replaced) =
4081                self.decode_to_utf8(src, &mut vec[old_len..], last);
4082            vec.set_len(old_len + written);
4083            (result, read, replaced)
4084        }
4085    }
4086
4087    public_decode_function!(/// Incrementally decode a byte stream into UTF-8
4088                            /// _without replacement_.
4089                            ///
4090                            /// See the documentation of the struct for
4091                            /// documentation for `decode_*` methods
4092                            /// collectively.
4093                            ///
4094                            /// Available via the C wrapper.
4095                            ,
4096                            decode_to_utf8_without_replacement,
4097                            decode_to_utf8_raw,
4098                            decode_to_utf8_checking_end,
4099                            decode_to_utf8_after_one_potential_bom_byte,
4100                            decode_to_utf8_after_two_potential_bom_bytes,
4101                            decode_to_utf8_checking_end_with_offset,
4102                            u8);
4103
4104    /// Incrementally decode a byte stream into UTF-8 with type system signaling
4105    /// of UTF-8 validity.
4106    ///
4107    /// This methods calls `decode_to_utf8` and then zeroes out up to three
4108    /// bytes that aren't logically part of the write in order to retain the
4109    /// UTF-8 validity even for the unwritten part of the buffer.
4110    ///
4111    /// See the documentation of the struct for documentation for `decode_*`
4112    /// methods collectively.
4113    ///
4114    /// Available to Rust only.
4115    pub fn decode_to_str_without_replacement(
4116        &mut self,
4117        src: &[u8],
4118        dst: &mut str,
4119        last: bool,
4120    ) -> (DecoderResult, usize, usize) {
4121        let bytes: &mut [u8] = unsafe { dst.as_bytes_mut() };
4122        let (result, read, written) = self.decode_to_utf8_without_replacement(src, bytes, last);
4123        let len = bytes.len();
4124        let mut trail = written;
4125        // Non-UTF-8 ASCII-compatible decoders may write up to `MAX_STRIDE_SIZE`
4126        // bytes of trailing garbage. No need to optimize non-ASCII-compatible
4127        // encodings to avoid overwriting here.
4128        if self.encoding != UTF_8 {
4129            let max = core::cmp::min(len, trail + ascii::MAX_STRIDE_SIZE);
4130            while trail < max {
4131                bytes[trail] = 0;
4132                trail += 1;
4133            }
4134        }
4135        while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
4136            bytes[trail] = 0;
4137            trail += 1;
4138        }
4139        (result, read, written)
4140    }
4141
4142    /// Incrementally decode a byte stream into UTF-8 using a `String` receiver.
4143    ///
4144    /// Like the others, this method follows the logic that the output buffer is
4145    /// caller-allocated. This method treats the capacity of the `String` as
4146    /// the output limit. That is, this method guarantees not to cause a
4147    /// reallocation of the backing buffer of `String`.
4148    ///
4149    /// The return value is a pair that contains the `DecoderResult` and the
4150    /// number of bytes read. The number of bytes written is signaled via
4151    /// the length of the `String` changing.
4152    ///
4153    /// See the documentation of the struct for documentation for `decode_*`
4154    /// methods collectively.
4155    ///
4156    /// Available to Rust only and only with the `alloc` feature enabled (enabled
4157    /// by default).
4158    #[cfg(feature = "alloc")]
4159    pub fn decode_to_string_without_replacement(
4160        &mut self,
4161        src: &[u8],
4162        dst: &mut String,
4163        last: bool,
4164    ) -> (DecoderResult, usize) {
4165        unsafe {
4166            let vec = dst.as_mut_vec();
4167            let old_len = vec.len();
4168            let capacity = vec.capacity();
4169            vec.set_len(capacity);
4170            let (result, read, written) =
4171                self.decode_to_utf8_without_replacement(src, &mut vec[old_len..], last);
4172            vec.set_len(old_len + written);
4173            (result, read)
4174        }
4175    }
4176
4177    /// Query the worst-case UTF-16 output size (with or without replacement).
4178    ///
4179    /// Returns the size of the output buffer in UTF-16 code units (`u16`)
4180    /// that will not overflow given the current state of the decoder and
4181    /// `byte_length` number of additional input bytes or `None` if `usize`
4182    /// would overflow.
4183    ///
4184    /// Since the REPLACEMENT CHARACTER fits into one UTF-16 code unit, the
4185    /// return value of this method applies also in the
4186    /// `_without_replacement` case.
4187    ///
4188    /// Available via the C wrapper.
4189    pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> {
4190        // Need to consider a) the decoder morphing due to the BOM and b) a partial
4191        // BOM getting pushed to the underlying decoder.
4192        match self.life_cycle {
4193            DecoderLifeCycle::Converting
4194            | DecoderLifeCycle::AtUtf8Start
4195            | DecoderLifeCycle::AtUtf16LeStart
4196            | DecoderLifeCycle::AtUtf16BeStart => {
4197                return self.variant.max_utf16_buffer_length(byte_length);
4198            }
4199            DecoderLifeCycle::AtStart => {
4200                if let Some(utf8_bom) = byte_length.checked_add(1) {
4201                    if let Some(utf16_bom) =
4202                        checked_add(1, checked_div(byte_length.checked_add(1), 2))
4203                    {
4204                        let utf_bom = core::cmp::max(utf8_bom, utf16_bom);
4205                        let encoding = self.encoding();
4206                        if encoding == UTF_8 || encoding == UTF_16LE || encoding == UTF_16BE {
4207                            // No need to consider the internal state of the underlying decoder,
4208                            // because it is at start, because no data has reached it yet.
4209                            return Some(utf_bom);
4210                        } else if let Some(non_bom) =
4211                            self.variant.max_utf16_buffer_length(byte_length)
4212                        {
4213                            return Some(core::cmp::max(utf_bom, non_bom));
4214                        }
4215                    }
4216                }
4217            }
4218            DecoderLifeCycle::SeenUtf8First | DecoderLifeCycle::SeenUtf8Second => {
4219                // Add two bytes even when only one byte has been seen,
4220                // because the one byte can become a lead byte in multibyte
4221                // decoders, but only after the decoder has been queried
4222                // for max length, so the decoder's own logic for adding
4223                // one for a pending lead cannot work.
4224                if let Some(sum) = byte_length.checked_add(2) {
4225                    if let Some(utf8_bom) = sum.checked_add(1) {
4226                        if self.encoding() == UTF_8 {
4227                            // No need to consider the internal state of the underlying decoder,
4228                            // because it is at start, because no data has reached it yet.
4229                            return Some(utf8_bom);
4230                        } else if let Some(non_bom) = self.variant.max_utf16_buffer_length(sum) {
4231                            return Some(core::cmp::max(utf8_bom, non_bom));
4232                        }
4233                    }
4234                }
4235            }
4236            DecoderLifeCycle::ConvertingWithPendingBB => {
4237                if let Some(sum) = byte_length.checked_add(2) {
4238                    return self.variant.max_utf16_buffer_length(sum);
4239                }
4240            }
4241            DecoderLifeCycle::SeenUtf16LeFirst | DecoderLifeCycle::SeenUtf16BeFirst => {
4242                // Add two bytes even when only one byte has been seen,
4243                // because the one byte can become a lead byte in multibyte
4244                // decoders, but only after the decoder has been queried
4245                // for max length, so the decoder's own logic for adding
4246                // one for a pending lead cannot work.
4247                if let Some(sum) = byte_length.checked_add(2) {
4248                    if let Some(utf16_bom) = checked_add(1, checked_div(sum.checked_add(1), 2)) {
4249                        let encoding = self.encoding();
4250                        if encoding == UTF_16LE || encoding == UTF_16BE {
4251                            // No need to consider the internal state of the underlying decoder,
4252                            // because it is at start, because no data has reached it yet.
4253                            return Some(utf16_bom);
4254                        } else if let Some(non_bom) = self.variant.max_utf16_buffer_length(sum) {
4255                            return Some(core::cmp::max(utf16_bom, non_bom));
4256                        }
4257                    }
4258                }
4259            }
4260            DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
4261        }
4262        None
4263    }
4264
4265    /// Incrementally decode a byte stream into UTF-16 with malformed sequences
4266    /// replaced with the REPLACEMENT CHARACTER.
4267    ///
4268    /// See the documentation of the struct for documentation for `decode_*`
4269    /// methods collectively.
4270    ///
4271    /// Available via the C wrapper.
4272    pub fn decode_to_utf16(
4273        &mut self,
4274        src: &[u8],
4275        dst: &mut [u16],
4276        last: bool,
4277    ) -> (CoderResult, usize, usize, bool) {
4278        let mut had_errors = false;
4279        let mut total_read = 0usize;
4280        let mut total_written = 0usize;
4281        loop {
4282            let (result, read, written) = self.decode_to_utf16_without_replacement(
4283                &src[total_read..],
4284                &mut dst[total_written..],
4285                last,
4286            );
4287            total_read += read;
4288            total_written += written;
4289            match result {
4290                DecoderResult::InputEmpty => {
4291                    return (
4292                        CoderResult::InputEmpty,
4293                        total_read,
4294                        total_written,
4295                        had_errors,
4296                    );
4297                }
4298                DecoderResult::OutputFull => {
4299                    return (
4300                        CoderResult::OutputFull,
4301                        total_read,
4302                        total_written,
4303                        had_errors,
4304                    );
4305                }
4306                DecoderResult::Malformed(_, _) => {
4307                    had_errors = true;
4308                    // There should always be space for the U+FFFD, because
4309                    // otherwise we'd have gotten OutputFull already.
4310                    dst[total_written] = 0xFFFD;
4311                    total_written += 1;
4312                }
4313            }
4314        }
4315    }
4316
4317    public_decode_function!(/// Incrementally decode a byte stream into UTF-16
4318                            /// _without replacement_.
4319                            ///
4320                            /// See the documentation of the struct for
4321                            /// documentation for `decode_*` methods
4322                            /// collectively.
4323                            ///
4324                            /// Available via the C wrapper.
4325                            ,
4326                            decode_to_utf16_without_replacement,
4327                            decode_to_utf16_raw,
4328                            decode_to_utf16_checking_end,
4329                            decode_to_utf16_after_one_potential_bom_byte,
4330                            decode_to_utf16_after_two_potential_bom_bytes,
4331                            decode_to_utf16_checking_end_with_offset,
4332                            u16);
4333
4334    /// Checks for compatibility with storing Unicode scalar values as unsigned
4335    /// bytes taking into account the state of the decoder.
4336    ///
4337    /// Returns `None` if the decoder is not in a neutral state, including waiting
4338    /// for the BOM, or if the encoding is never Latin1-byte-compatible.
4339    ///
4340    /// Otherwise returns the index of the first byte whose unsigned value doesn't
4341    /// directly correspond to the decoded Unicode scalar value, or the length
4342    /// of the input if all bytes in the input decode directly to scalar values
4343    /// corresponding to the unsigned byte values.
4344    ///
4345    /// Does not change the state of the decoder.
4346    ///
4347    /// Do not use this unless you are supporting SpiderMonkey/V8-style string
4348    /// storage optimizations.
4349    ///
4350    /// Available via the C wrapper.
4351    pub fn latin1_byte_compatible_up_to(&self, bytes: &[u8]) -> Option<usize> {
4352        match self.life_cycle {
4353            DecoderLifeCycle::Converting => {
4354                return self.variant.latin1_byte_compatible_up_to(bytes);
4355            }
4356            DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
4357            _ => None,
4358        }
4359    }
4360}
4361
4362/// Result of a (potentially partial) encode operation without replacement.
4363#[must_use]
4364#[derive(Debug, PartialEq, Eq)]
4365pub enum EncoderResult {
4366    /// The input was exhausted.
4367    ///
4368    /// If this result was returned from a call where `last` was `true`, the
4369    /// decoding process has completed. Otherwise, the caller should call a
4370    /// decode method again with more input.
4371    InputEmpty,
4372
4373    /// The encoder cannot produce another unit of output, because the output
4374    /// buffer does not have enough space left.
4375    ///
4376    /// The caller must provide more output space upon the next call and re-push
4377    /// the remaining input to the decoder.
4378    OutputFull,
4379
4380    /// The encoder encountered an unmappable character.
4381    ///
4382    /// The caller must either treat this as a fatal error or must append
4383    /// a placeholder to the output and then re-push the remaining input to the
4384    /// encoder.
4385    Unmappable(char),
4386}
4387
4388impl EncoderResult {
4389    fn unmappable_from_bmp(bmp: u16) -> EncoderResult {
4390        EncoderResult::Unmappable(::core::char::from_u32(u32::from(bmp)).unwrap())
4391    }
4392}
4393
4394/// A converter that encodes a Unicode stream into bytes according to a
4395/// character encoding in a streaming (incremental) manner.
4396///
4397/// The various `encode_*` methods take an input buffer (`src`) and an output
4398/// buffer `dst` both of which are caller-allocated. There are variants for
4399/// both UTF-8 and UTF-16 input buffers.
4400///
4401/// An `encode_*` method encode characters from `src` into bytes characters
4402/// stored into `dst` until one of the following three things happens:
4403///
4404/// 1. An unmappable character is encountered (`*_without_replacement` variants
4405///    only).
4406///
4407/// 2. The output buffer has been filled so near capacity that the decoder
4408///    cannot be sure that processing an additional character of input wouldn't
4409///    cause so much output that the output buffer would overflow.
4410///
4411/// 3. All the input characters have been processed.
4412///
4413/// The `encode_*` method then returns tuple of a status indicating which one
4414/// of the three reasons to return happened, how many input code units (`u8`
4415/// when encoding from UTF-8 and `u16` when encoding from UTF-16) were read,
4416/// how many output bytes were written (except when encoding into `Vec<u8>`,
4417/// whose length change indicates this), and in the case of the variants that
4418/// perform replacement, a boolean indicating whether an unmappable
4419/// character was replaced with a numeric character reference during the call.
4420///
4421/// The number of bytes "written" is what's logically written. Garbage may be
4422/// written in the output buffer beyond the point logically written to.
4423///
4424/// In the case of the methods whose name ends with
4425/// `*_without_replacement`, the status is an [`EncoderResult`][1] enumeration
4426/// (possibilities `Unmappable`, `OutputFull` and `InputEmpty` corresponding to
4427/// the three cases listed above).
4428///
4429/// In the case of methods whose name does not end with
4430/// `*_without_replacement`, unmappable characters are automatically replaced
4431/// with the corresponding numeric character references and unmappable
4432/// characters do not cause the methods to return early.
4433///
4434/// When encoding from UTF-8 without replacement, the methods are guaranteed
4435/// not to return indicating that more output space is needed if the length
4436/// of the output buffer is at least the length returned by
4437/// [`max_buffer_length_from_utf8_without_replacement()`][2]. When encoding from
4438/// UTF-8 with replacement, the length of the output buffer that guarantees the
4439/// methods not to return indicating that more output space is needed in the
4440/// absence of unmappable characters is given by
4441/// [`max_buffer_length_from_utf8_if_no_unmappables()`][3]. When encoding from
4442/// UTF-16 without replacement, the methods are guaranteed not to return
4443/// indicating that more output space is needed if the length of the output
4444/// buffer is at least the length returned by
4445/// [`max_buffer_length_from_utf16_without_replacement()`][4]. When encoding
4446/// from UTF-16 with replacement, the the length of the output buffer that
4447/// guarantees the methods not to return indicating that more output space is
4448/// needed in the absence of unmappable characters is given by
4449/// [`max_buffer_length_from_utf16_if_no_unmappables()`][5].
4450/// When encoding with replacement, applications are not expected to size the
4451/// buffer for the worst case ahead of time but to resize the buffer if there
4452/// are unmappable characters. This is why max length queries are only available
4453/// for the case where there are no unmappable characters.
4454///
4455/// When encoding from UTF-8, each `src` buffer _must_ be valid UTF-8. (When
4456/// calling from Rust, the type system takes care of this.) When encoding from
4457/// UTF-16, unpaired surrogates in the input are treated as U+FFFD REPLACEMENT
4458/// CHARACTERS. Therefore, in order for astral characters not to turn into a
4459/// pair of REPLACEMENT CHARACTERS, the caller must ensure that surrogate pairs
4460/// are not split across input buffer boundaries.
4461///
4462/// After an `encode_*` call returns, the output produced so far, taken as a
4463/// whole from the start of the stream, is guaranteed to consist of a valid
4464/// byte sequence in the target encoding. (I.e. the code unit sequence for a
4465/// character is guaranteed not to be split across output buffers. However, due
4466/// to the stateful nature of ISO-2022-JP, the stream needs to be considered
4467/// from the start for it to be valid. For other encodings, the validity holds
4468/// on a per-output buffer basis.)
4469///
4470/// The boolean argument `last` indicates that the end of the stream is reached
4471/// when all the characters in `src` have been consumed. This argument is needed
4472/// for ISO-2022-JP and is ignored for other encodings.
4473///
4474/// An `Encoder` object can be used to incrementally encode a byte stream.
4475///
4476/// During the processing of a single stream, the caller must call `encode_*`
4477/// zero or more times with `last` set to `false` and then call `encode_*` at
4478/// least once with `last` set to `true`. If `encode_*` returns `InputEmpty`,
4479/// the processing of the stream has ended. Otherwise, the caller must call
4480/// `encode_*` again with `last` set to `true` (or treat an `Unmappable` result
4481/// as a fatal error).
4482///
4483/// Once the stream has ended, the `Encoder` object must not be used anymore.
4484/// That is, you need to create another one to process another stream.
4485///
4486/// When the encoder returns `OutputFull` or the encoder returns `Unmappable`
4487/// and the caller does not wish to treat it as a fatal error, the input buffer
4488/// `src` may not have been completely consumed. In that case, the caller must
4489/// pass the unconsumed contents of `src` to `encode_*` again upon the next
4490/// call.
4491///
4492/// [1]: enum.EncoderResult.html
4493/// [2]: #method.max_buffer_length_from_utf8_without_replacement
4494/// [3]: #method.max_buffer_length_from_utf8_if_no_unmappables
4495/// [4]: #method.max_buffer_length_from_utf16_without_replacement
4496/// [5]: #method.max_buffer_length_from_utf16_if_no_unmappables
4497///
4498/// # Infinite loops
4499///
4500/// When converting with a fixed-size output buffer whose size is too small to
4501/// accommodate one character of output, an infinite loop ensues. When
4502/// converting with a fixed-size output buffer, it generally makes sense to
4503/// make the buffer fairly large (e.g. couple of kilobytes).
4504pub struct Encoder {
4505    encoding: &'static Encoding,
4506    variant: VariantEncoder,
4507}
4508
4509impl Encoder {
4510    fn new(enc: &'static Encoding, encoder: VariantEncoder) -> Encoder {
4511        Encoder {
4512            encoding: enc,
4513            variant: encoder,
4514        }
4515    }
4516
4517    /// The `Encoding` this `Encoder` is for.
4518    #[inline]
4519    pub fn encoding(&self) -> &'static Encoding {
4520        self.encoding
4521    }
4522
4523    /// Returns `true` if this is an ISO-2022-JP encoder that's not in the
4524    /// ASCII state and `false` otherwise.
4525    #[inline]
4526    pub fn has_pending_state(&self) -> bool {
4527        self.variant.has_pending_state()
4528    }
4529
4530    /// Query the worst-case output size when encoding from UTF-8 with
4531    /// replacement.
4532    ///
4533    /// Returns the size of the output buffer in bytes that will not overflow
4534    /// given the current state of the encoder and `byte_length` number of
4535    /// additional input code units if there are no unmappable characters in
4536    /// the input or `None` if `usize` would overflow.
4537    ///
4538    /// Available via the C wrapper.
4539    pub fn max_buffer_length_from_utf8_if_no_unmappables(
4540        &self,
4541        byte_length: usize,
4542    ) -> Option<usize> {
4543        checked_add(
4544            if self.encoding().can_encode_everything() {
4545                0
4546            } else {
4547                NCR_EXTRA
4548            },
4549            self.max_buffer_length_from_utf8_without_replacement(byte_length),
4550        )
4551    }
4552
4553    /// Query the worst-case output size when encoding from UTF-8 without
4554    /// replacement.
4555    ///
4556    /// Returns the size of the output buffer in bytes that will not overflow
4557    /// given the current state of the encoder and `byte_length` number of
4558    /// additional input code units or `None` if `usize` would overflow.
4559    ///
4560    /// Available via the C wrapper.
4561    pub fn max_buffer_length_from_utf8_without_replacement(
4562        &self,
4563        byte_length: usize,
4564    ) -> Option<usize> {
4565        self.variant
4566            .max_buffer_length_from_utf8_without_replacement(byte_length)
4567    }
4568
4569    /// Incrementally encode into byte stream from UTF-8 with unmappable
4570    /// characters replaced with HTML (decimal) numeric character references.
4571    ///
4572    /// See the documentation of the struct for documentation for `encode_*`
4573    /// methods collectively.
4574    ///
4575    /// Available via the C wrapper.
4576    pub fn encode_from_utf8(
4577        &mut self,
4578        src: &str,
4579        dst: &mut [u8],
4580        last: bool,
4581    ) -> (CoderResult, usize, usize, bool) {
4582        let dst_len = dst.len();
4583        let effective_dst_len = if self.encoding().can_encode_everything() {
4584            dst_len
4585        } else {
4586            if dst_len < NCR_EXTRA {
4587                if src.is_empty() && !(last && self.has_pending_state()) {
4588                    return (CoderResult::InputEmpty, 0, 0, false);
4589                }
4590                return (CoderResult::OutputFull, 0, 0, false);
4591            }
4592            dst_len - NCR_EXTRA
4593        };
4594        let mut had_unmappables = false;
4595        let mut total_read = 0usize;
4596        let mut total_written = 0usize;
4597        loop {
4598            let (result, read, written) = self.encode_from_utf8_without_replacement(
4599                &src[total_read..],
4600                &mut dst[total_written..effective_dst_len],
4601                last,
4602            );
4603            total_read += read;
4604            total_written += written;
4605            match result {
4606                EncoderResult::InputEmpty => {
4607                    return (
4608                        CoderResult::InputEmpty,
4609                        total_read,
4610                        total_written,
4611                        had_unmappables,
4612                    );
4613                }
4614                EncoderResult::OutputFull => {
4615                    return (
4616                        CoderResult::OutputFull,
4617                        total_read,
4618                        total_written,
4619                        had_unmappables,
4620                    );
4621                }
4622                EncoderResult::Unmappable(unmappable) => {
4623                    had_unmappables = true;
4624                    debug_assert!(dst.len() - total_written >= NCR_EXTRA);
4625                    debug_assert_ne!(self.encoding(), UTF_16BE);
4626                    debug_assert_ne!(self.encoding(), UTF_16LE);
4627                    // Additionally, Iso2022JpEncoder is responsible for
4628                    // transitioning to ASCII when returning with Unmappable.
4629                    total_written += write_ncr(unmappable, &mut dst[total_written..]);
4630                    if total_written >= effective_dst_len {
4631                        if total_read == src.len() && !(last && self.has_pending_state()) {
4632                            return (
4633                                CoderResult::InputEmpty,
4634                                total_read,
4635                                total_written,
4636                                had_unmappables,
4637                            );
4638                        }
4639                        return (
4640                            CoderResult::OutputFull,
4641                            total_read,
4642                            total_written,
4643                            had_unmappables,
4644                        );
4645                    }
4646                }
4647            }
4648        }
4649    }
4650
4651    /// Incrementally encode into byte stream from UTF-8 with unmappable
4652    /// characters replaced with HTML (decimal) numeric character references.
4653    ///
4654    /// See the documentation of the struct for documentation for `encode_*`
4655    /// methods collectively.
4656    ///
4657    /// Available to Rust only and only with the `alloc` feature enabled (enabled
4658    /// by default).
4659    #[cfg(feature = "alloc")]
4660    pub fn encode_from_utf8_to_vec(
4661        &mut self,
4662        src: &str,
4663        dst: &mut Vec<u8>,
4664        last: bool,
4665    ) -> (CoderResult, usize, bool) {
4666        unsafe {
4667            let old_len = dst.len();
4668            let capacity = dst.capacity();
4669            dst.set_len(capacity);
4670            let (result, read, written, replaced) =
4671                self.encode_from_utf8(src, &mut dst[old_len..], last);
4672            dst.set_len(old_len + written);
4673            (result, read, replaced)
4674        }
4675    }
4676
4677    /// Incrementally encode into byte stream from UTF-8 _without replacement_.
4678    ///
4679    /// See the documentation of the struct for documentation for `encode_*`
4680    /// methods collectively.
4681    ///
4682    /// Available via the C wrapper.
4683    pub fn encode_from_utf8_without_replacement(
4684        &mut self,
4685        src: &str,
4686        dst: &mut [u8],
4687        last: bool,
4688    ) -> (EncoderResult, usize, usize) {
4689        self.variant.encode_from_utf8_raw(src, dst, last)
4690    }
4691
4692    /// Incrementally encode into byte stream from UTF-8 _without replacement_.
4693    ///
4694    /// See the documentation of the struct for documentation for `encode_*`
4695    /// methods collectively.
4696    ///
4697    /// Available to Rust only and only with the `alloc` feature enabled (enabled
4698    /// by default).
4699    #[cfg(feature = "alloc")]
4700    pub fn encode_from_utf8_to_vec_without_replacement(
4701        &mut self,
4702        src: &str,
4703        dst: &mut Vec<u8>,
4704        last: bool,
4705    ) -> (EncoderResult, usize) {
4706        unsafe {
4707            let old_len = dst.len();
4708            let capacity = dst.capacity();
4709            dst.set_len(capacity);
4710            let (result, read, written) =
4711                self.encode_from_utf8_without_replacement(src, &mut dst[old_len..], last);
4712            dst.set_len(old_len + written);
4713            (result, read)
4714        }
4715    }
4716
4717    /// Query the worst-case output size when encoding from UTF-16 with
4718    /// replacement.
4719    ///
4720    /// Returns the size of the output buffer in bytes that will not overflow
4721    /// given the current state of the encoder and `u16_length` number of
4722    /// additional input code units if there are no unmappable characters in
4723    /// the input or `None` if `usize` would overflow.
4724    ///
4725    /// Available via the C wrapper.
4726    pub fn max_buffer_length_from_utf16_if_no_unmappables(
4727        &self,
4728        u16_length: usize,
4729    ) -> Option<usize> {
4730        checked_add(
4731            if self.encoding().can_encode_everything() {
4732                0
4733            } else {
4734                NCR_EXTRA
4735            },
4736            self.max_buffer_length_from_utf16_without_replacement(u16_length),
4737        )
4738    }
4739
4740    /// Query the worst-case output size when encoding from UTF-16 without
4741    /// replacement.
4742    ///
4743    /// Returns the size of the output buffer in bytes that will not overflow
4744    /// given the current state of the encoder and `u16_length` number of
4745    /// additional input code units or `None` if `usize` would overflow.
4746    ///
4747    /// Available via the C wrapper.
4748    pub fn max_buffer_length_from_utf16_without_replacement(
4749        &self,
4750        u16_length: usize,
4751    ) -> Option<usize> {
4752        self.variant
4753            .max_buffer_length_from_utf16_without_replacement(u16_length)
4754    }
4755
4756    /// Incrementally encode into byte stream from UTF-16 with unmappable
4757    /// characters replaced with HTML (decimal) numeric character references.
4758    ///
4759    /// See the documentation of the struct for documentation for `encode_*`
4760    /// methods collectively.
4761    ///
4762    /// Available via the C wrapper.
4763    pub fn encode_from_utf16(
4764        &mut self,
4765        src: &[u16],
4766        dst: &mut [u8],
4767        last: bool,
4768    ) -> (CoderResult, usize, usize, bool) {
4769        let dst_len = dst.len();
4770        let effective_dst_len = if self.encoding().can_encode_everything() {
4771            dst_len
4772        } else {
4773            if dst_len < NCR_EXTRA {
4774                if src.is_empty() && !(last && self.has_pending_state()) {
4775                    return (CoderResult::InputEmpty, 0, 0, false);
4776                }
4777                return (CoderResult::OutputFull, 0, 0, false);
4778            }
4779            dst_len - NCR_EXTRA
4780        };
4781        let mut had_unmappables = false;
4782        let mut total_read = 0usize;
4783        let mut total_written = 0usize;
4784        loop {
4785            let (result, read, written) = self.encode_from_utf16_without_replacement(
4786                &src[total_read..],
4787                &mut dst[total_written..effective_dst_len],
4788                last,
4789            );
4790            total_read += read;
4791            total_written += written;
4792            match result {
4793                EncoderResult::InputEmpty => {
4794                    return (
4795                        CoderResult::InputEmpty,
4796                        total_read,
4797                        total_written,
4798                        had_unmappables,
4799                    );
4800                }
4801                EncoderResult::OutputFull => {
4802                    return (
4803                        CoderResult::OutputFull,
4804                        total_read,
4805                        total_written,
4806                        had_unmappables,
4807                    );
4808                }
4809                EncoderResult::Unmappable(unmappable) => {
4810                    had_unmappables = true;
4811                    debug_assert!(dst.len() - total_written >= NCR_EXTRA);
4812                    // There are no UTF-16 encoders and even if there were,
4813                    // they'd never have unmappables.
4814                    debug_assert_ne!(self.encoding(), UTF_16BE);
4815                    debug_assert_ne!(self.encoding(), UTF_16LE);
4816                    // Additionally, Iso2022JpEncoder is responsible for
4817                    // transitioning to ASCII when returning with Unmappable
4818                    // from the jis0208 state. That is, when we encode
4819                    // ISO-2022-JP and come here, the encoder is in either the
4820                    // ASCII or the Roman state. We are allowed to generate any
4821                    // printable ASCII excluding \ and ~.
4822                    total_written += write_ncr(unmappable, &mut dst[total_written..]);
4823                    if total_written >= effective_dst_len {
4824                        if total_read == src.len() && !(last && self.has_pending_state()) {
4825                            return (
4826                                CoderResult::InputEmpty,
4827                                total_read,
4828                                total_written,
4829                                had_unmappables,
4830                            );
4831                        }
4832                        return (
4833                            CoderResult::OutputFull,
4834                            total_read,
4835                            total_written,
4836                            had_unmappables,
4837                        );
4838                    }
4839                }
4840            }
4841        }
4842    }
4843
4844    /// Incrementally encode into byte stream from UTF-16 _without replacement_.
4845    ///
4846    /// See the documentation of the struct for documentation for `encode_*`
4847    /// methods collectively.
4848    ///
4849    /// Available via the C wrapper.
4850    pub fn encode_from_utf16_without_replacement(
4851        &mut self,
4852        src: &[u16],
4853        dst: &mut [u8],
4854        last: bool,
4855    ) -> (EncoderResult, usize, usize) {
4856        self.variant.encode_from_utf16_raw(src, dst, last)
4857    }
4858}
4859
4860/// Format an unmappable as NCR without heap allocation.
4861fn write_ncr(unmappable: char, dst: &mut [u8]) -> usize {
4862    // len is the number of decimal digits needed to represent unmappable plus
4863    // 3 (the length of "&#" and ";").
4864    let mut number = unmappable as u32;
4865    let len = if number >= 1_000_000u32 {
4866        10usize
4867    } else if number >= 100_000u32 {
4868        9usize
4869    } else if number >= 10_000u32 {
4870        8usize
4871    } else if number >= 1_000u32 {
4872        7usize
4873    } else if number >= 100u32 {
4874        6usize
4875    } else {
4876        // Review the outcome of https://github.com/whatwg/encoding/issues/15
4877        // to see if this case is possible
4878        5usize
4879    };
4880    debug_assert!(number >= 10u32);
4881    debug_assert!(len <= dst.len());
4882    let mut pos = len - 1;
4883    dst[pos] = b';';
4884    pos -= 1;
4885    loop {
4886        let rightmost = number % 10;
4887        dst[pos] = rightmost as u8 + b'0';
4888        pos -= 1;
4889        if number < 10 {
4890            break;
4891        }
4892        number /= 10;
4893    }
4894    dst[1] = b'#';
4895    dst[0] = b'&';
4896    len
4897}
4898
4899#[inline(always)]
4900fn in_range16(i: u16, start: u16, end: u16) -> bool {
4901    i.wrapping_sub(start) < (end - start)
4902}
4903
4904#[inline(always)]
4905fn in_range32(i: u32, start: u32, end: u32) -> bool {
4906    i.wrapping_sub(start) < (end - start)
4907}
4908
4909#[inline(always)]
4910fn in_inclusive_range8(i: u8, start: u8, end: u8) -> bool {
4911    i.wrapping_sub(start) <= (end - start)
4912}
4913
4914#[inline(always)]
4915fn in_inclusive_range16(i: u16, start: u16, end: u16) -> bool {
4916    i.wrapping_sub(start) <= (end - start)
4917}
4918
4919#[inline(always)]
4920fn in_inclusive_range32(i: u32, start: u32, end: u32) -> bool {
4921    i.wrapping_sub(start) <= (end - start)
4922}
4923
4924#[inline(always)]
4925fn in_inclusive_range(i: usize, start: usize, end: usize) -> bool {
4926    i.wrapping_sub(start) <= (end - start)
4927}
4928
4929#[inline(always)]
4930fn checked_add(num: usize, opt: Option<usize>) -> Option<usize> {
4931    if let Some(n) = opt {
4932        n.checked_add(num)
4933    } else {
4934        None
4935    }
4936}
4937
4938#[inline(always)]
4939fn checked_add_opt(one: Option<usize>, other: Option<usize>) -> Option<usize> {
4940    if let Some(n) = one {
4941        checked_add(n, other)
4942    } else {
4943        None
4944    }
4945}
4946
4947#[inline(always)]
4948fn checked_mul(num: usize, opt: Option<usize>) -> Option<usize> {
4949    if let Some(n) = opt {
4950        n.checked_mul(num)
4951    } else {
4952        None
4953    }
4954}
4955
4956#[inline(always)]
4957fn checked_div(opt: Option<usize>, num: usize) -> Option<usize> {
4958    if let Some(n) = opt {
4959        n.checked_div(num)
4960    } else {
4961        None
4962    }
4963}
4964
4965#[cfg(feature = "alloc")]
4966#[inline(always)]
4967fn checked_next_power_of_two(opt: Option<usize>) -> Option<usize> {
4968    opt.map(|n| n.next_power_of_two())
4969}
4970
4971#[cfg(feature = "alloc")]
4972#[inline(always)]
4973fn checked_min(one: Option<usize>, other: Option<usize>) -> Option<usize> {
4974    if let Some(a) = one {
4975        if let Some(b) = other {
4976            Some(::core::cmp::min(a, b))
4977        } else {
4978            Some(a)
4979        }
4980    } else {
4981        other
4982    }
4983}
4984
4985// ############## TESTS ###############
4986
4987#[cfg(all(test, feature = "serde"))]
4988#[derive(Serialize, Deserialize, Debug, PartialEq)]
4989struct Demo {
4990    num: u32,
4991    name: String,
4992    enc: &'static Encoding,
4993}
4994
4995#[cfg(test)]
4996mod test_labels_names;
4997
4998#[cfg(all(test, feature = "alloc"))]
4999mod tests {
5000    use super::*;
5001    use alloc::borrow::Cow;
5002
5003    fn sniff_to_utf16(
5004        initial_encoding: &'static Encoding,
5005        expected_encoding: &'static Encoding,
5006        bytes: &[u8],
5007        expect: &[u16],
5008        breaks: &[usize],
5009    ) {
5010        let mut decoder = initial_encoding.new_decoder();
5011
5012        let mut dest: Vec<u16> =
5013            Vec::with_capacity(decoder.max_utf16_buffer_length(bytes.len()).unwrap());
5014        let capacity = dest.capacity();
5015        dest.resize(capacity, 0u16);
5016
5017        let mut total_written = 0usize;
5018        let mut start = 0usize;
5019        for br in breaks {
5020            let (result, read, written, _) =
5021                decoder.decode_to_utf16(&bytes[start..*br], &mut dest[total_written..], false);
5022            total_written += written;
5023            assert_eq!(read, *br - start);
5024            match result {
5025                CoderResult::InputEmpty => {}
5026                CoderResult::OutputFull => {
5027                    unreachable!();
5028                }
5029            }
5030            start = *br;
5031        }
5032        let (result, read, written, _) =
5033            decoder.decode_to_utf16(&bytes[start..], &mut dest[total_written..], true);
5034        total_written += written;
5035        match result {
5036            CoderResult::InputEmpty => {}
5037            CoderResult::OutputFull => {
5038                unreachable!();
5039            }
5040        }
5041        assert_eq!(read, bytes.len() - start);
5042        assert_eq!(total_written, expect.len());
5043        assert_eq!(&dest[..total_written], expect);
5044        assert_eq!(decoder.encoding(), expected_encoding);
5045    }
5046
5047    // Any copyright to the test code below this comment is dedicated to the
5048    // Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
5049
5050    #[test]
5051    fn test_bom_sniffing() {
5052        // ASCII
5053        sniff_to_utf16(
5054            WINDOWS_1252,
5055            WINDOWS_1252,
5056            b"\x61\x62",
5057            &[0x0061u16, 0x0062u16],
5058            &[],
5059        );
5060        // UTF-8
5061        sniff_to_utf16(
5062            WINDOWS_1252,
5063            UTF_8,
5064            b"\xEF\xBB\xBF\x61\x62",
5065            &[0x0061u16, 0x0062u16],
5066            &[],
5067        );
5068        sniff_to_utf16(
5069            WINDOWS_1252,
5070            UTF_8,
5071            b"\xEF\xBB\xBF\x61\x62",
5072            &[0x0061u16, 0x0062u16],
5073            &[1],
5074        );
5075        sniff_to_utf16(
5076            WINDOWS_1252,
5077            UTF_8,
5078            b"\xEF\xBB\xBF\x61\x62",
5079            &[0x0061u16, 0x0062u16],
5080            &[2],
5081        );
5082        sniff_to_utf16(
5083            WINDOWS_1252,
5084            UTF_8,
5085            b"\xEF\xBB\xBF\x61\x62",
5086            &[0x0061u16, 0x0062u16],
5087            &[3],
5088        );
5089        sniff_to_utf16(
5090            WINDOWS_1252,
5091            UTF_8,
5092            b"\xEF\xBB\xBF\x61\x62",
5093            &[0x0061u16, 0x0062u16],
5094            &[4],
5095        );
5096        sniff_to_utf16(
5097            WINDOWS_1252,
5098            UTF_8,
5099            b"\xEF\xBB\xBF\x61\x62",
5100            &[0x0061u16, 0x0062u16],
5101            &[2, 3],
5102        );
5103        sniff_to_utf16(
5104            WINDOWS_1252,
5105            UTF_8,
5106            b"\xEF\xBB\xBF\x61\x62",
5107            &[0x0061u16, 0x0062u16],
5108            &[1, 2],
5109        );
5110        sniff_to_utf16(
5111            WINDOWS_1252,
5112            UTF_8,
5113            b"\xEF\xBB\xBF\x61\x62",
5114            &[0x0061u16, 0x0062u16],
5115            &[1, 3],
5116        );
5117        sniff_to_utf16(
5118            WINDOWS_1252,
5119            UTF_8,
5120            b"\xEF\xBB\xBF\x61\x62",
5121            &[0x0061u16, 0x0062u16],
5122            &[1, 2, 3, 4],
5123        );
5124        sniff_to_utf16(WINDOWS_1252, UTF_8, b"\xEF\xBB\xBF", &[], &[]);
5125        // Not UTF-8
5126        sniff_to_utf16(
5127            WINDOWS_1252,
5128            WINDOWS_1252,
5129            b"\xEF\xBB\x61\x62",
5130            &[0x00EFu16, 0x00BBu16, 0x0061u16, 0x0062u16],
5131            &[],
5132        );
5133        sniff_to_utf16(
5134            WINDOWS_1252,
5135            WINDOWS_1252,
5136            b"\xEF\xBB\x61\x62",
5137            &[0x00EFu16, 0x00BBu16, 0x0061u16, 0x0062u16],
5138            &[1],
5139        );
5140        sniff_to_utf16(
5141            WINDOWS_1252,
5142            WINDOWS_1252,
5143            b"\xEF\x61\x62",
5144            &[0x00EFu16, 0x0061u16, 0x0062u16],
5145            &[],
5146        );
5147        sniff_to_utf16(
5148            WINDOWS_1252,
5149            WINDOWS_1252,
5150            b"\xEF\x61\x62",
5151            &[0x00EFu16, 0x0061u16, 0x0062u16],
5152            &[1],
5153        );
5154        sniff_to_utf16(
5155            WINDOWS_1252,
5156            WINDOWS_1252,
5157            b"\xEF\xBB",
5158            &[0x00EFu16, 0x00BBu16],
5159            &[],
5160        );
5161        sniff_to_utf16(
5162            WINDOWS_1252,
5163            WINDOWS_1252,
5164            b"\xEF\xBB",
5165            &[0x00EFu16, 0x00BBu16],
5166            &[1],
5167        );
5168        sniff_to_utf16(WINDOWS_1252, WINDOWS_1252, b"\xEF", &[0x00EFu16], &[]);
5169        // Not UTF-16
5170        sniff_to_utf16(
5171            WINDOWS_1252,
5172            WINDOWS_1252,
5173            b"\xFE\x61\x62",
5174            &[0x00FEu16, 0x0061u16, 0x0062u16],
5175            &[],
5176        );
5177        sniff_to_utf16(
5178            WINDOWS_1252,
5179            WINDOWS_1252,
5180            b"\xFE\x61\x62",
5181            &[0x00FEu16, 0x0061u16, 0x0062u16],
5182            &[1],
5183        );
5184        sniff_to_utf16(WINDOWS_1252, WINDOWS_1252, b"\xFE", &[0x00FEu16], &[]);
5185        sniff_to_utf16(
5186            WINDOWS_1252,
5187            WINDOWS_1252,
5188            b"\xFF\x61\x62",
5189            &[0x00FFu16, 0x0061u16, 0x0062u16],
5190            &[],
5191        );
5192        sniff_to_utf16(
5193            WINDOWS_1252,
5194            WINDOWS_1252,
5195            b"\xFF\x61\x62",
5196            &[0x00FFu16, 0x0061u16, 0x0062u16],
5197            &[1],
5198        );
5199        sniff_to_utf16(WINDOWS_1252, WINDOWS_1252, b"\xFF", &[0x00FFu16], &[]);
5200        // UTF-16
5201        sniff_to_utf16(WINDOWS_1252, UTF_16BE, b"\xFE\xFF", &[], &[]);
5202        sniff_to_utf16(WINDOWS_1252, UTF_16BE, b"\xFE\xFF", &[], &[1]);
5203        sniff_to_utf16(WINDOWS_1252, UTF_16LE, b"\xFF\xFE", &[], &[]);
5204        sniff_to_utf16(WINDOWS_1252, UTF_16LE, b"\xFF\xFE", &[], &[1]);
5205    }
5206
5207    #[test]
5208    fn test_output_encoding() {
5209        assert_eq!(REPLACEMENT.output_encoding(), UTF_8);
5210        assert_eq!(UTF_16BE.output_encoding(), UTF_8);
5211        assert_eq!(UTF_16LE.output_encoding(), UTF_8);
5212        assert_eq!(UTF_8.output_encoding(), UTF_8);
5213        assert_eq!(WINDOWS_1252.output_encoding(), WINDOWS_1252);
5214        assert_eq!(REPLACEMENT.new_encoder().encoding(), UTF_8);
5215        assert_eq!(UTF_16BE.new_encoder().encoding(), UTF_8);
5216        assert_eq!(UTF_16LE.new_encoder().encoding(), UTF_8);
5217        assert_eq!(UTF_8.new_encoder().encoding(), UTF_8);
5218        assert_eq!(WINDOWS_1252.new_encoder().encoding(), WINDOWS_1252);
5219    }
5220
5221    #[test]
5222    fn test_label_resolution() {
5223        assert_eq!(Encoding::for_label(b"utf-8"), Some(UTF_8));
5224        assert_eq!(Encoding::for_label(b"UTF-8"), Some(UTF_8));
5225        assert_eq!(
5226            Encoding::for_label(b" \t \n \x0C \n utf-8 \r \n \t \x0C "),
5227            Some(UTF_8)
5228        );
5229        assert_eq!(Encoding::for_label(b"utf-8 _"), None);
5230        assert_eq!(Encoding::for_label(b"bogus"), None);
5231        assert_eq!(Encoding::for_label(b"bogusbogusbogusbogus"), None);
5232    }
5233
5234    #[test]
5235    fn test_decode_valid_windows_1257_to_cow() {
5236        let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"abc\x80\xE4");
5237        match cow {
5238            Cow::Borrowed(_) => unreachable!(),
5239            Cow::Owned(s) => {
5240                assert_eq!(s, "abc\u{20AC}\u{00E4}");
5241            }
5242        }
5243        assert_eq!(encoding, WINDOWS_1257);
5244        assert!(!had_errors);
5245    }
5246
5247    #[test]
5248    fn test_decode_invalid_windows_1257_to_cow() {
5249        let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"abc\x80\xA1\xE4");
5250        match cow {
5251            Cow::Borrowed(_) => unreachable!(),
5252            Cow::Owned(s) => {
5253                assert_eq!(s, "abc\u{20AC}\u{FFFD}\u{00E4}");
5254            }
5255        }
5256        assert_eq!(encoding, WINDOWS_1257);
5257        assert!(had_errors);
5258    }
5259
5260    #[test]
5261    fn test_decode_ascii_only_windows_1257_to_cow() {
5262        let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"abc");
5263        match cow {
5264            Cow::Borrowed(s) => {
5265                assert_eq!(s, "abc");
5266            }
5267            Cow::Owned(_) => unreachable!(),
5268        }
5269        assert_eq!(encoding, WINDOWS_1257);
5270        assert!(!had_errors);
5271    }
5272
5273    #[test]
5274    fn test_decode_bomful_valid_utf8_as_windows_1257_to_cow() {
5275        let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5276        match cow {
5277            Cow::Borrowed(s) => {
5278                assert_eq!(s, "\u{20AC}\u{00E4}");
5279            }
5280            Cow::Owned(_) => unreachable!(),
5281        }
5282        assert_eq!(encoding, UTF_8);
5283        assert!(!had_errors);
5284    }
5285
5286    #[test]
5287    fn test_decode_bomful_invalid_utf8_as_windows_1257_to_cow() {
5288        let (cow, encoding, had_errors) =
5289            WINDOWS_1257.decode(b"\xEF\xBB\xBF\xE2\x82\xAC\x80\xC3\xA4");
5290        match cow {
5291            Cow::Borrowed(_) => unreachable!(),
5292            Cow::Owned(s) => {
5293                assert_eq!(s, "\u{20AC}\u{FFFD}\u{00E4}");
5294            }
5295        }
5296        assert_eq!(encoding, UTF_8);
5297        assert!(had_errors);
5298    }
5299
5300    #[test]
5301    fn test_decode_bomful_valid_utf8_as_utf_8_to_cow() {
5302        let (cow, encoding, had_errors) = UTF_8.decode(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5303        match cow {
5304            Cow::Borrowed(s) => {
5305                assert_eq!(s, "\u{20AC}\u{00E4}");
5306            }
5307            Cow::Owned(_) => unreachable!(),
5308        }
5309        assert_eq!(encoding, UTF_8);
5310        assert!(!had_errors);
5311    }
5312
5313    #[test]
5314    fn test_decode_bomful_invalid_utf8_as_utf_8_to_cow() {
5315        let (cow, encoding, had_errors) = UTF_8.decode(b"\xEF\xBB\xBF\xE2\x82\xAC\x80\xC3\xA4");
5316        match cow {
5317            Cow::Borrowed(_) => unreachable!(),
5318            Cow::Owned(s) => {
5319                assert_eq!(s, "\u{20AC}\u{FFFD}\u{00E4}");
5320            }
5321        }
5322        assert_eq!(encoding, UTF_8);
5323        assert!(had_errors);
5324    }
5325
5326    #[test]
5327    fn test_decode_bomful_valid_utf8_as_utf_8_to_cow_with_bom_removal() {
5328        let (cow, had_errors) = UTF_8.decode_with_bom_removal(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5329        match cow {
5330            Cow::Borrowed(s) => {
5331                assert_eq!(s, "\u{20AC}\u{00E4}");
5332            }
5333            Cow::Owned(_) => unreachable!(),
5334        }
5335        assert!(!had_errors);
5336    }
5337
5338    #[test]
5339    fn test_decode_bomful_valid_utf8_as_windows_1257_to_cow_with_bom_removal() {
5340        let (cow, had_errors) =
5341            WINDOWS_1257.decode_with_bom_removal(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5342        match cow {
5343            Cow::Borrowed(_) => unreachable!(),
5344            Cow::Owned(s) => {
5345                assert_eq!(
5346                    s,
5347                    "\u{013C}\u{00BB}\u{00E6}\u{0101}\u{201A}\u{00AC}\u{0106}\u{00A4}"
5348                );
5349            }
5350        }
5351        assert!(!had_errors);
5352    }
5353
5354    #[test]
5355    fn test_decode_valid_windows_1257_to_cow_with_bom_removal() {
5356        let (cow, had_errors) = WINDOWS_1257.decode_with_bom_removal(b"abc\x80\xE4");
5357        match cow {
5358            Cow::Borrowed(_) => unreachable!(),
5359            Cow::Owned(s) => {
5360                assert_eq!(s, "abc\u{20AC}\u{00E4}");
5361            }
5362        }
5363        assert!(!had_errors);
5364    }
5365
5366    #[test]
5367    fn test_decode_invalid_windows_1257_to_cow_with_bom_removal() {
5368        let (cow, had_errors) = WINDOWS_1257.decode_with_bom_removal(b"abc\x80\xA1\xE4");
5369        match cow {
5370            Cow::Borrowed(_) => unreachable!(),
5371            Cow::Owned(s) => {
5372                assert_eq!(s, "abc\u{20AC}\u{FFFD}\u{00E4}");
5373            }
5374        }
5375        assert!(had_errors);
5376    }
5377
5378    #[test]
5379    fn test_decode_ascii_only_windows_1257_to_cow_with_bom_removal() {
5380        let (cow, had_errors) = WINDOWS_1257.decode_with_bom_removal(b"abc");
5381        match cow {
5382            Cow::Borrowed(s) => {
5383                assert_eq!(s, "abc");
5384            }
5385            Cow::Owned(_) => unreachable!(),
5386        }
5387        assert!(!had_errors);
5388    }
5389
5390    #[test]
5391    fn test_decode_bomful_valid_utf8_to_cow_without_bom_handling() {
5392        let (cow, had_errors) =
5393            UTF_8.decode_without_bom_handling(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5394        match cow {
5395            Cow::Borrowed(s) => {
5396                assert_eq!(s, "\u{FEFF}\u{20AC}\u{00E4}");
5397            }
5398            Cow::Owned(_) => unreachable!(),
5399        }
5400        assert!(!had_errors);
5401    }
5402
5403    #[test]
5404    fn test_decode_bomful_invalid_utf8_to_cow_without_bom_handling() {
5405        let (cow, had_errors) =
5406            UTF_8.decode_without_bom_handling(b"\xEF\xBB\xBF\xE2\x82\xAC\x80\xC3\xA4");
5407        match cow {
5408            Cow::Borrowed(_) => unreachable!(),
5409            Cow::Owned(s) => {
5410                assert_eq!(s, "\u{FEFF}\u{20AC}\u{FFFD}\u{00E4}");
5411            }
5412        }
5413        assert!(had_errors);
5414    }
5415
5416    #[test]
5417    fn test_decode_valid_windows_1257_to_cow_without_bom_handling() {
5418        let (cow, had_errors) = WINDOWS_1257.decode_without_bom_handling(b"abc\x80\xE4");
5419        match cow {
5420            Cow::Borrowed(_) => unreachable!(),
5421            Cow::Owned(s) => {
5422                assert_eq!(s, "abc\u{20AC}\u{00E4}");
5423            }
5424        }
5425        assert!(!had_errors);
5426    }
5427
5428    #[test]
5429    fn test_decode_invalid_windows_1257_to_cow_without_bom_handling() {
5430        let (cow, had_errors) = WINDOWS_1257.decode_without_bom_handling(b"abc\x80\xA1\xE4");
5431        match cow {
5432            Cow::Borrowed(_) => unreachable!(),
5433            Cow::Owned(s) => {
5434                assert_eq!(s, "abc\u{20AC}\u{FFFD}\u{00E4}");
5435            }
5436        }
5437        assert!(had_errors);
5438    }
5439
5440    #[test]
5441    fn test_decode_ascii_only_windows_1257_to_cow_without_bom_handling() {
5442        let (cow, had_errors) = WINDOWS_1257.decode_without_bom_handling(b"abc");
5443        match cow {
5444            Cow::Borrowed(s) => {
5445                assert_eq!(s, "abc");
5446            }
5447            Cow::Owned(_) => unreachable!(),
5448        }
5449        assert!(!had_errors);
5450    }
5451
5452    #[test]
5453    fn test_decode_bomful_valid_utf8_to_cow_without_bom_handling_and_without_replacement() {
5454        match UTF_8.decode_without_bom_handling_and_without_replacement(
5455            b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4",
5456        ) {
5457            Some(cow) => match cow {
5458                Cow::Borrowed(s) => {
5459                    assert_eq!(s, "\u{FEFF}\u{20AC}\u{00E4}");
5460                }
5461                Cow::Owned(_) => unreachable!(),
5462            },
5463            None => unreachable!(),
5464        }
5465    }
5466
5467    #[test]
5468    fn test_decode_bomful_invalid_utf8_to_cow_without_bom_handling_and_without_replacement() {
5469        assert!(UTF_8
5470            .decode_without_bom_handling_and_without_replacement(
5471                b"\xEF\xBB\xBF\xE2\x82\xAC\x80\xC3\xA4"
5472            )
5473            .is_none());
5474    }
5475
5476    #[test]
5477    fn test_decode_valid_windows_1257_to_cow_without_bom_handling_and_without_replacement() {
5478        match WINDOWS_1257.decode_without_bom_handling_and_without_replacement(b"abc\x80\xE4") {
5479            Some(cow) => match cow {
5480                Cow::Borrowed(_) => unreachable!(),
5481                Cow::Owned(s) => {
5482                    assert_eq!(s, "abc\u{20AC}\u{00E4}");
5483                }
5484            },
5485            None => unreachable!(),
5486        }
5487    }
5488
5489    #[test]
5490    fn test_decode_invalid_windows_1257_to_cow_without_bom_handling_and_without_replacement() {
5491        assert!(WINDOWS_1257
5492            .decode_without_bom_handling_and_without_replacement(b"abc\x80\xA1\xE4")
5493            .is_none());
5494    }
5495
5496    #[test]
5497    fn test_decode_ascii_only_windows_1257_to_cow_without_bom_handling_and_without_replacement() {
5498        match WINDOWS_1257.decode_without_bom_handling_and_without_replacement(b"abc") {
5499            Some(cow) => match cow {
5500                Cow::Borrowed(s) => {
5501                    assert_eq!(s, "abc");
5502                }
5503                Cow::Owned(_) => unreachable!(),
5504            },
5505            None => unreachable!(),
5506        }
5507    }
5508
5509    #[test]
5510    fn test_encode_ascii_only_windows_1257_to_cow() {
5511        let (cow, encoding, had_errors) = WINDOWS_1257.encode("abc");
5512        match cow {
5513            Cow::Borrowed(s) => {
5514                assert_eq!(s, b"abc");
5515            }
5516            Cow::Owned(_) => unreachable!(),
5517        }
5518        assert_eq!(encoding, WINDOWS_1257);
5519        assert!(!had_errors);
5520    }
5521
5522    #[test]
5523    fn test_encode_valid_windows_1257_to_cow() {
5524        let (cow, encoding, had_errors) = WINDOWS_1257.encode("abc\u{20AC}\u{00E4}");
5525        match cow {
5526            Cow::Borrowed(_) => unreachable!(),
5527            Cow::Owned(s) => {
5528                assert_eq!(s, b"abc\x80\xE4");
5529            }
5530        }
5531        assert_eq!(encoding, WINDOWS_1257);
5532        assert!(!had_errors);
5533    }
5534
5535    #[test]
5536    fn test_utf16_space_with_one_bom_byte() {
5537        let mut decoder = UTF_16LE.new_decoder();
5538        let mut dst = [0u16; 12];
5539        {
5540            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5541            let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], false);
5542            assert_eq!(result, CoderResult::InputEmpty);
5543        }
5544        {
5545            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5546            let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], true);
5547            assert_eq!(result, CoderResult::InputEmpty);
5548        }
5549    }
5550
5551    #[test]
5552    fn test_utf8_space_with_one_bom_byte() {
5553        let mut decoder = UTF_8.new_decoder();
5554        let mut dst = [0u16; 12];
5555        {
5556            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5557            let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], false);
5558            assert_eq!(result, CoderResult::InputEmpty);
5559        }
5560        {
5561            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5562            let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], true);
5563            assert_eq!(result, CoderResult::InputEmpty);
5564        }
5565    }
5566
5567    #[test]
5568    fn test_utf16_space_with_two_bom_bytes() {
5569        let mut decoder = UTF_16LE.new_decoder();
5570        let mut dst = [0u16; 12];
5571        {
5572            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5573            let (result, _, _, _) = decoder.decode_to_utf16(b"\xEF", &mut dst[..needed], false);
5574            assert_eq!(result, CoderResult::InputEmpty);
5575        }
5576        {
5577            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5578            let (result, _, _, _) = decoder.decode_to_utf16(b"\xBB", &mut dst[..needed], false);
5579            assert_eq!(result, CoderResult::InputEmpty);
5580        }
5581        {
5582            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5583            let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], true);
5584            assert_eq!(result, CoderResult::InputEmpty);
5585        }
5586    }
5587
5588    #[test]
5589    fn test_utf8_space_with_two_bom_bytes() {
5590        let mut decoder = UTF_8.new_decoder();
5591        let mut dst = [0u16; 12];
5592        {
5593            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5594            let (result, _, _, _) = decoder.decode_to_utf16(b"\xEF", &mut dst[..needed], false);
5595            assert_eq!(result, CoderResult::InputEmpty);
5596        }
5597        {
5598            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5599            let (result, _, _, _) = decoder.decode_to_utf16(b"\xBB", &mut dst[..needed], false);
5600            assert_eq!(result, CoderResult::InputEmpty);
5601        }
5602        {
5603            let needed = decoder.max_utf16_buffer_length(1).unwrap();
5604            let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], true);
5605            assert_eq!(result, CoderResult::InputEmpty);
5606        }
5607    }
5608
5609    #[test]
5610    fn test_utf16_space_with_one_bom_byte_and_a_second_byte_in_same_call() {
5611        let mut decoder = UTF_16LE.new_decoder();
5612        let mut dst = [0u16; 12];
5613        {
5614            let needed = decoder.max_utf16_buffer_length(2).unwrap();
5615            let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF\xFF", &mut dst[..needed], true);
5616            assert_eq!(result, CoderResult::InputEmpty);
5617        }
5618    }
5619
5620    #[test]
5621    fn test_too_short_buffer_with_iso_2022_jp_ascii_from_utf8() {
5622        let mut dst = [0u8; 8];
5623        let mut encoder = ISO_2022_JP.new_encoder();
5624        {
5625            let (result, _, _, _) = encoder.encode_from_utf8("", &mut dst[..], false);
5626            assert_eq!(result, CoderResult::InputEmpty);
5627        }
5628        {
5629            let (result, _, _, _) = encoder.encode_from_utf8("", &mut dst[..], true);
5630            assert_eq!(result, CoderResult::InputEmpty);
5631        }
5632    }
5633
5634    #[test]
5635    fn test_too_short_buffer_with_iso_2022_jp_roman_from_utf8() {
5636        let mut dst = [0u8; 16];
5637        let mut encoder = ISO_2022_JP.new_encoder();
5638        {
5639            let (result, _, _, _) = encoder.encode_from_utf8("\u{A5}", &mut dst[..], false);
5640            assert_eq!(result, CoderResult::InputEmpty);
5641        }
5642        {
5643            let (result, _, _, _) = encoder.encode_from_utf8("", &mut dst[..8], false);
5644            assert_eq!(result, CoderResult::InputEmpty);
5645        }
5646        {
5647            let (result, _, _, _) = encoder.encode_from_utf8("", &mut dst[..8], true);
5648            assert_eq!(result, CoderResult::OutputFull);
5649        }
5650    }
5651
5652    #[test]
5653    fn test_buffer_end_iso_2022_jp_from_utf8() {
5654        let mut dst = [0u8; 18];
5655        {
5656            let mut encoder = ISO_2022_JP.new_encoder();
5657            let (result, _, _, _) =
5658                encoder.encode_from_utf8("\u{A5}\u{1F4A9}", &mut dst[..], false);
5659            assert_eq!(result, CoderResult::InputEmpty);
5660        }
5661        {
5662            let mut encoder = ISO_2022_JP.new_encoder();
5663            let (result, _, _, _) = encoder.encode_from_utf8("\u{A5}\u{1F4A9}", &mut dst[..], true);
5664            assert_eq!(result, CoderResult::OutputFull);
5665        }
5666        {
5667            let mut encoder = ISO_2022_JP.new_encoder();
5668            let (result, _, _, _) = encoder.encode_from_utf8("\u{1F4A9}", &mut dst[..13], false);
5669            assert_eq!(result, CoderResult::InputEmpty);
5670        }
5671        {
5672            let mut encoder = ISO_2022_JP.new_encoder();
5673            let (result, _, _, _) = encoder.encode_from_utf8("\u{1F4A9}", &mut dst[..13], true);
5674            assert_eq!(result, CoderResult::InputEmpty);
5675        }
5676    }
5677
5678    #[test]
5679    fn test_too_short_buffer_with_iso_2022_jp_ascii_from_utf16() {
5680        let mut dst = [0u8; 8];
5681        let mut encoder = ISO_2022_JP.new_encoder();
5682        {
5683            let (result, _, _, _) = encoder.encode_from_utf16(&[0u16; 0], &mut dst[..], false);
5684            assert_eq!(result, CoderResult::InputEmpty);
5685        }
5686        {
5687            let (result, _, _, _) = encoder.encode_from_utf16(&[0u16; 0], &mut dst[..], true);
5688            assert_eq!(result, CoderResult::InputEmpty);
5689        }
5690    }
5691
5692    #[test]
5693    fn test_too_short_buffer_with_iso_2022_jp_roman_from_utf16() {
5694        let mut dst = [0u8; 16];
5695        let mut encoder = ISO_2022_JP.new_encoder();
5696        {
5697            let (result, _, _, _) = encoder.encode_from_utf16(&[0xA5u16], &mut dst[..], false);
5698            assert_eq!(result, CoderResult::InputEmpty);
5699        }
5700        {
5701            let (result, _, _, _) = encoder.encode_from_utf16(&[0u16; 0], &mut dst[..8], false);
5702            assert_eq!(result, CoderResult::InputEmpty);
5703        }
5704        {
5705            let (result, _, _, _) = encoder.encode_from_utf16(&[0u16; 0], &mut dst[..8], true);
5706            assert_eq!(result, CoderResult::OutputFull);
5707        }
5708    }
5709
5710    #[test]
5711    fn test_buffer_end_iso_2022_jp_from_utf16() {
5712        let mut dst = [0u8; 18];
5713        {
5714            let mut encoder = ISO_2022_JP.new_encoder();
5715            let (result, _, _, _) =
5716                encoder.encode_from_utf16(&[0xA5u16, 0xD83Du16, 0xDCA9u16], &mut dst[..], false);
5717            assert_eq!(result, CoderResult::InputEmpty);
5718        }
5719        {
5720            let mut encoder = ISO_2022_JP.new_encoder();
5721            let (result, _, _, _) =
5722                encoder.encode_from_utf16(&[0xA5u16, 0xD83Du16, 0xDCA9u16], &mut dst[..], true);
5723            assert_eq!(result, CoderResult::OutputFull);
5724        }
5725        {
5726            let mut encoder = ISO_2022_JP.new_encoder();
5727            let (result, _, _, _) =
5728                encoder.encode_from_utf16(&[0xD83Du16, 0xDCA9u16], &mut dst[..13], false);
5729            assert_eq!(result, CoderResult::InputEmpty);
5730        }
5731        {
5732            let mut encoder = ISO_2022_JP.new_encoder();
5733            let (result, _, _, _) =
5734                encoder.encode_from_utf16(&[0xD83Du16, 0xDCA9u16], &mut dst[..13], true);
5735            assert_eq!(result, CoderResult::InputEmpty);
5736        }
5737    }
5738
5739    #[test]
5740    fn test_buffer_end_utf16be() {
5741        let mut decoder = UTF_16BE.new_decoder_without_bom_handling();
5742        let mut dest = [0u8; 4];
5743
5744        assert_eq!(
5745            decoder.decode_to_utf8(&[0xD8, 0x00], &mut dest, false),
5746            (CoderResult::InputEmpty, 2, 0, false)
5747        );
5748
5749        let _ = decoder.decode_to_utf8(&[0xD8, 0x00], &mut dest, true);
5750    }
5751
5752    #[test]
5753    fn test_hash() {
5754        let mut encodings = ::alloc::collections::btree_set::BTreeSet::new();
5755        encodings.insert(UTF_8);
5756        encodings.insert(ISO_2022_JP);
5757        assert!(encodings.contains(UTF_8));
5758        assert!(encodings.contains(ISO_2022_JP));
5759        assert!(!encodings.contains(WINDOWS_1252));
5760        encodings.remove(ISO_2022_JP);
5761        assert!(!encodings.contains(ISO_2022_JP));
5762    }
5763
5764    #[test]
5765    fn test_iso_2022_jp_ncr_extra_from_utf16() {
5766        let mut dst = [0u8; 17];
5767        {
5768            let mut encoder = ISO_2022_JP.new_encoder();
5769            let (result, _, _, _) =
5770                encoder.encode_from_utf16(&[0x3041u16, 0xFFFFu16], &mut dst[..], true);
5771            assert_eq!(result, CoderResult::OutputFull);
5772        }
5773    }
5774
5775    #[test]
5776    fn test_iso_2022_jp_ncr_extra_from_utf8() {
5777        let mut dst = [0u8; 17];
5778        {
5779            let mut encoder = ISO_2022_JP.new_encoder();
5780            let (result, _, _, _) =
5781                encoder.encode_from_utf8("\u{3041}\u{FFFF}", &mut dst[..], true);
5782            assert_eq!(result, CoderResult::OutputFull);
5783        }
5784    }
5785
5786    #[test]
5787    fn test_max_length_with_bom_to_utf8() {
5788        let mut output = [0u8; 20];
5789        let mut decoder = REPLACEMENT.new_decoder();
5790        let input = b"\xEF\xBB\xBFA";
5791        {
5792            let needed = decoder
5793                .max_utf8_buffer_length_without_replacement(input.len())
5794                .unwrap();
5795            let (result, read, written) =
5796                decoder.decode_to_utf8_without_replacement(input, &mut output[..needed], true);
5797            assert_eq!(result, DecoderResult::InputEmpty);
5798            assert_eq!(read, input.len());
5799            assert_eq!(written, 1);
5800            assert_eq!(output[0], 0x41);
5801        }
5802    }
5803
5804    #[cfg(feature = "serde")]
5805    #[test]
5806    fn test_serde() {
5807        let demo = Demo {
5808            num: 42,
5809            name: "foo".into(),
5810            enc: UTF_8,
5811        };
5812
5813        let serialized = serde_json::to_string(&demo).unwrap();
5814
5815        let deserialized: Demo = serde_json::from_str(&serialized).unwrap();
5816        assert_eq!(deserialized, demo);
5817
5818        let bincoded = bincode::serialize(&demo).unwrap();
5819        let debincoded: Demo = bincode::deserialize(&bincoded[..]).unwrap();
5820        assert_eq!(debincoded, demo);
5821    }
5822
5823    #[test]
5824    fn test_is_single_byte() {
5825        assert!(!BIG5.is_single_byte());
5826        assert!(!EUC_JP.is_single_byte());
5827        assert!(!EUC_KR.is_single_byte());
5828        assert!(!GB18030.is_single_byte());
5829        assert!(!GBK.is_single_byte());
5830        assert!(!REPLACEMENT.is_single_byte());
5831        assert!(!SHIFT_JIS.is_single_byte());
5832        assert!(!UTF_8.is_single_byte());
5833        assert!(!UTF_16BE.is_single_byte());
5834        assert!(!UTF_16LE.is_single_byte());
5835        assert!(!ISO_2022_JP.is_single_byte());
5836
5837        assert!(IBM866.is_single_byte());
5838        assert!(ISO_8859_2.is_single_byte());
5839        assert!(ISO_8859_3.is_single_byte());
5840        assert!(ISO_8859_4.is_single_byte());
5841        assert!(ISO_8859_5.is_single_byte());
5842        assert!(ISO_8859_6.is_single_byte());
5843        assert!(ISO_8859_7.is_single_byte());
5844        assert!(ISO_8859_8.is_single_byte());
5845        assert!(ISO_8859_10.is_single_byte());
5846        assert!(ISO_8859_13.is_single_byte());
5847        assert!(ISO_8859_14.is_single_byte());
5848        assert!(ISO_8859_15.is_single_byte());
5849        assert!(ISO_8859_16.is_single_byte());
5850        assert!(ISO_8859_8_I.is_single_byte());
5851        assert!(KOI8_R.is_single_byte());
5852        assert!(KOI8_U.is_single_byte());
5853        assert!(MACINTOSH.is_single_byte());
5854        assert!(WINDOWS_874.is_single_byte());
5855        assert!(WINDOWS_1250.is_single_byte());
5856        assert!(WINDOWS_1251.is_single_byte());
5857        assert!(WINDOWS_1252.is_single_byte());
5858        assert!(WINDOWS_1253.is_single_byte());
5859        assert!(WINDOWS_1254.is_single_byte());
5860        assert!(WINDOWS_1255.is_single_byte());
5861        assert!(WINDOWS_1256.is_single_byte());
5862        assert!(WINDOWS_1257.is_single_byte());
5863        assert!(WINDOWS_1258.is_single_byte());
5864        assert!(X_MAC_CYRILLIC.is_single_byte());
5865        assert!(X_USER_DEFINED.is_single_byte());
5866    }
5867
5868    #[test]
5869    fn test_latin1_byte_compatible_up_to() {
5870        let buffer = b"a\x81\xB6\xF6\xF0\x82\xB4";
5871        assert_eq!(
5872            BIG5.new_decoder_without_bom_handling()
5873                .latin1_byte_compatible_up_to(buffer)
5874                .unwrap(),
5875            1
5876        );
5877        assert_eq!(
5878            EUC_JP
5879                .new_decoder_without_bom_handling()
5880                .latin1_byte_compatible_up_to(buffer)
5881                .unwrap(),
5882            1
5883        );
5884        assert_eq!(
5885            EUC_KR
5886                .new_decoder_without_bom_handling()
5887                .latin1_byte_compatible_up_to(buffer)
5888                .unwrap(),
5889            1
5890        );
5891        assert_eq!(
5892            GB18030
5893                .new_decoder_without_bom_handling()
5894                .latin1_byte_compatible_up_to(buffer)
5895                .unwrap(),
5896            1
5897        );
5898        assert_eq!(
5899            GBK.new_decoder_without_bom_handling()
5900                .latin1_byte_compatible_up_to(buffer)
5901                .unwrap(),
5902            1
5903        );
5904        assert!(REPLACEMENT
5905            .new_decoder_without_bom_handling()
5906            .latin1_byte_compatible_up_to(buffer)
5907            .is_none());
5908        assert_eq!(
5909            SHIFT_JIS
5910                .new_decoder_without_bom_handling()
5911                .latin1_byte_compatible_up_to(buffer)
5912                .unwrap(),
5913            1
5914        );
5915        assert_eq!(
5916            UTF_8
5917                .new_decoder_without_bom_handling()
5918                .latin1_byte_compatible_up_to(buffer)
5919                .unwrap(),
5920            1
5921        );
5922        assert!(UTF_16BE
5923            .new_decoder_without_bom_handling()
5924            .latin1_byte_compatible_up_to(buffer)
5925            .is_none());
5926        assert!(UTF_16LE
5927            .new_decoder_without_bom_handling()
5928            .latin1_byte_compatible_up_to(buffer)
5929            .is_none());
5930        assert_eq!(
5931            ISO_2022_JP
5932                .new_decoder_without_bom_handling()
5933                .latin1_byte_compatible_up_to(buffer)
5934                .unwrap(),
5935            1
5936        );
5937
5938        assert_eq!(
5939            IBM866
5940                .new_decoder_without_bom_handling()
5941                .latin1_byte_compatible_up_to(buffer)
5942                .unwrap(),
5943            1
5944        );
5945        assert_eq!(
5946            ISO_8859_2
5947                .new_decoder_without_bom_handling()
5948                .latin1_byte_compatible_up_to(buffer)
5949                .unwrap(),
5950            2
5951        );
5952        assert_eq!(
5953            ISO_8859_3
5954                .new_decoder_without_bom_handling()
5955                .latin1_byte_compatible_up_to(buffer)
5956                .unwrap(),
5957            2
5958        );
5959        assert_eq!(
5960            ISO_8859_4
5961                .new_decoder_without_bom_handling()
5962                .latin1_byte_compatible_up_to(buffer)
5963                .unwrap(),
5964            2
5965        );
5966        assert_eq!(
5967            ISO_8859_5
5968                .new_decoder_without_bom_handling()
5969                .latin1_byte_compatible_up_to(buffer)
5970                .unwrap(),
5971            2
5972        );
5973        assert_eq!(
5974            ISO_8859_6
5975                .new_decoder_without_bom_handling()
5976                .latin1_byte_compatible_up_to(buffer)
5977                .unwrap(),
5978            2
5979        );
5980        assert_eq!(
5981            ISO_8859_7
5982                .new_decoder_without_bom_handling()
5983                .latin1_byte_compatible_up_to(buffer)
5984                .unwrap(),
5985            2
5986        );
5987        assert_eq!(
5988            ISO_8859_8
5989                .new_decoder_without_bom_handling()
5990                .latin1_byte_compatible_up_to(buffer)
5991                .unwrap(),
5992            3
5993        );
5994        assert_eq!(
5995            ISO_8859_10
5996                .new_decoder_without_bom_handling()
5997                .latin1_byte_compatible_up_to(buffer)
5998                .unwrap(),
5999            2
6000        );
6001        assert_eq!(
6002            ISO_8859_13
6003                .new_decoder_without_bom_handling()
6004                .latin1_byte_compatible_up_to(buffer)
6005                .unwrap(),
6006            4
6007        );
6008        assert_eq!(
6009            ISO_8859_14
6010                .new_decoder_without_bom_handling()
6011                .latin1_byte_compatible_up_to(buffer)
6012                .unwrap(),
6013            4
6014        );
6015        assert_eq!(
6016            ISO_8859_15
6017                .new_decoder_without_bom_handling()
6018                .latin1_byte_compatible_up_to(buffer)
6019                .unwrap(),
6020            6
6021        );
6022        assert_eq!(
6023            ISO_8859_16
6024                .new_decoder_without_bom_handling()
6025                .latin1_byte_compatible_up_to(buffer)
6026                .unwrap(),
6027            4
6028        );
6029        assert_eq!(
6030            ISO_8859_8_I
6031                .new_decoder_without_bom_handling()
6032                .latin1_byte_compatible_up_to(buffer)
6033                .unwrap(),
6034            3
6035        );
6036        assert_eq!(
6037            KOI8_R
6038                .new_decoder_without_bom_handling()
6039                .latin1_byte_compatible_up_to(buffer)
6040                .unwrap(),
6041            1
6042        );
6043        assert_eq!(
6044            KOI8_U
6045                .new_decoder_without_bom_handling()
6046                .latin1_byte_compatible_up_to(buffer)
6047                .unwrap(),
6048            1
6049        );
6050        assert_eq!(
6051            MACINTOSH
6052                .new_decoder_without_bom_handling()
6053                .latin1_byte_compatible_up_to(buffer)
6054                .unwrap(),
6055            1
6056        );
6057        assert_eq!(
6058            WINDOWS_874
6059                .new_decoder_without_bom_handling()
6060                .latin1_byte_compatible_up_to(buffer)
6061                .unwrap(),
6062            2
6063        );
6064        assert_eq!(
6065            WINDOWS_1250
6066                .new_decoder_without_bom_handling()
6067                .latin1_byte_compatible_up_to(buffer)
6068                .unwrap(),
6069            4
6070        );
6071        assert_eq!(
6072            WINDOWS_1251
6073                .new_decoder_without_bom_handling()
6074                .latin1_byte_compatible_up_to(buffer)
6075                .unwrap(),
6076            1
6077        );
6078        assert_eq!(
6079            WINDOWS_1252
6080                .new_decoder_without_bom_handling()
6081                .latin1_byte_compatible_up_to(buffer)
6082                .unwrap(),
6083            5
6084        );
6085        assert_eq!(
6086            WINDOWS_1253
6087                .new_decoder_without_bom_handling()
6088                .latin1_byte_compatible_up_to(buffer)
6089                .unwrap(),
6090            3
6091        );
6092        assert_eq!(
6093            WINDOWS_1254
6094                .new_decoder_without_bom_handling()
6095                .latin1_byte_compatible_up_to(buffer)
6096                .unwrap(),
6097            4
6098        );
6099        assert_eq!(
6100            WINDOWS_1255
6101                .new_decoder_without_bom_handling()
6102                .latin1_byte_compatible_up_to(buffer)
6103                .unwrap(),
6104            3
6105        );
6106        assert_eq!(
6107            WINDOWS_1256
6108                .new_decoder_without_bom_handling()
6109                .latin1_byte_compatible_up_to(buffer)
6110                .unwrap(),
6111            1
6112        );
6113        assert_eq!(
6114            WINDOWS_1257
6115                .new_decoder_without_bom_handling()
6116                .latin1_byte_compatible_up_to(buffer)
6117                .unwrap(),
6118            4
6119        );
6120        assert_eq!(
6121            WINDOWS_1258
6122                .new_decoder_without_bom_handling()
6123                .latin1_byte_compatible_up_to(buffer)
6124                .unwrap(),
6125            4
6126        );
6127        assert_eq!(
6128            X_MAC_CYRILLIC
6129                .new_decoder_without_bom_handling()
6130                .latin1_byte_compatible_up_to(buffer)
6131                .unwrap(),
6132            1
6133        );
6134        assert_eq!(
6135            X_USER_DEFINED
6136                .new_decoder_without_bom_handling()
6137                .latin1_byte_compatible_up_to(buffer)
6138                .unwrap(),
6139            1
6140        );
6141
6142        assert!(UTF_8
6143            .new_decoder()
6144            .latin1_byte_compatible_up_to(buffer)
6145            .is_none());
6146
6147        let mut decoder = UTF_8.new_decoder();
6148        let mut output = [0u16; 4];
6149        let _ = decoder.decode_to_utf16(b"\xEF", &mut output, false);
6150        assert!(decoder.latin1_byte_compatible_up_to(buffer).is_none());
6151        let _ = decoder.decode_to_utf16(b"\xBB\xBF", &mut output, false);
6152        assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), Some(1));
6153        let _ = decoder.decode_to_utf16(b"\xEF", &mut output, false);
6154        assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), None);
6155    }
6156}