brotli/enc/
mod.rs

Help
1#![cfg_attr(not(feature = "std"), allow(unused_imports))]
2#[macro_use]
3pub mod vectorization;
4pub mod backward_references;
5pub mod bit_cost;
6pub mod block_split;
7pub mod brotli_bit_stream;
8pub mod cluster;
9pub mod combined_alloc;
10pub mod command;
11pub mod constants;
12pub mod dictionary_hash;
13pub mod entropy_encode;
14pub mod fast_log;
15pub mod histogram;
16pub mod input_pair;
17pub mod literal_cost;
18pub mod static_dict;
19pub mod static_dict_lut;
20pub mod utf8_util;
21pub mod util;
22pub use self::backward_references::hash_to_binary_tree;
23pub use self::backward_references::hq as backward_references_hq;
24pub mod block_splitter;
25pub mod compress_fragment;
26pub mod compress_fragment_two_pass;
27pub mod context_map_entropy;
28pub mod encode;
29pub mod find_stride;
30pub mod interface;
31pub mod ir_interpret;
32pub mod metablock;
33pub mod pdf;
34pub mod prior_eval;
35pub mod reader;
36pub mod stride_eval;
37pub mod writer;
38pub use self::combined_alloc::{BrotliAlloc, CombiningAllocator};
39mod compat;
40pub mod fixed_queue;
41pub mod multithreading;
42pub mod singlethreading;
43pub mod threading;
44pub mod worker_pool;
45#[cfg(feature = "simd")]
46use core::simd::{f32x8, i16x16, i32x8};
47#[cfg(feature = "simd")]
48pub type s16 = core::simd::i16x16;
49#[cfg(feature = "simd")]
50pub type v8 = core::simd::f32x8;
51#[cfg(feature = "simd")]
52pub type s8 = core::simd::i32x8;
53#[cfg(not(feature = "simd"))]
54pub type s16 = compat::Compat16x16;
55#[cfg(not(feature = "simd"))]
56pub type v8 = compat::CompatF8;
57#[cfg(not(feature = "simd"))]
58pub type s8 = compat::Compat32x8;
59
60mod parameters;
61mod test;
62mod weights;
63pub use self::backward_references::{BrotliEncoderParams, UnionHasher};
64use self::encode::{
65    BrotliEncoderCompressStream, BrotliEncoderCreateInstance, BrotliEncoderDestroyInstance,
66    BrotliEncoderIsFinished, BrotliEncoderOperation, BrotliEncoderSetCustomDictionary,
67};
68pub use self::encode::{
69    BrotliEncoderInitParams, BrotliEncoderMaxCompressedSize, BrotliEncoderMaxCompressedSizeMulti,
70    BrotliEncoderSetParameter,
71};
72pub use self::hash_to_binary_tree::ZopfliNode;
73pub use self::interface::StaticCommand;
74pub use self::pdf::PDF;
75pub use self::util::floatX;
76pub use self::vectorization::{v256, v256i, Mem256f};
77use brotli_decompressor::{CustomRead, CustomWrite};
78pub use interface::{InputPair, InputReference, InputReferenceMut};
79
80pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator};
81#[cfg(feature = "std")]
82pub use alloc_stdlib::StandardAlloc;
83#[cfg(feature = "std")]
84use std::io;
85#[cfg(feature = "std")]
86use std::io::{Error, ErrorKind, Read, Write};
87
88#[cfg(feature = "std")]
89pub use brotli_decompressor::{IntoIoReader, IoReaderWrapper, IoWriterWrapper};
90
91#[cfg(not(feature = "std"))]
92pub use self::singlethreading::{compress_worker_pool, new_work_pool, WorkerPool};
93pub use self::threading::{
94    BatchSpawnableLite, BrotliEncoderThreadError, CompressionThreadResult, Owned, SendAlloc,
95};
96#[cfg(feature = "std")]
97pub use self::worker_pool::{compress_worker_pool, new_work_pool, WorkerPool};
98#[cfg(feature = "std")]
99pub fn compress_multi<
100    Alloc: BrotliAlloc + Send + 'static,
101    SliceW: SliceWrapper<u8> + Send + 'static + Sync,
102>(
103    params: &BrotliEncoderParams,
104    owned_input: &mut Owned<SliceW>,
105    output: &mut [u8],
106    alloc_per_thread: &mut [SendAlloc<
107        CompressionThreadResult<Alloc>,
108        backward_references::UnionHasher<Alloc>,
109        Alloc,
110        <WorkerPool<
111            CompressionThreadResult<Alloc>,
112            backward_references::UnionHasher<Alloc>,
113            Alloc,
114            (SliceW, BrotliEncoderParams),
115        > as BatchSpawnableLite<
116            CompressionThreadResult<Alloc>,
117            backward_references::UnionHasher<Alloc>,
118            Alloc,
119            (SliceW, BrotliEncoderParams),
120        >>::JoinHandle,
121    >],
122) -> Result<usize, BrotliEncoderThreadError>
123where
124    <Alloc as Allocator<u8>>::AllocatedMemory: Send,
125    <Alloc as Allocator<u16>>::AllocatedMemory: Send + Sync,
126    <Alloc as Allocator<u32>>::AllocatedMemory: Send + Sync,
127{
128    let mut work_pool = self::worker_pool::new_work_pool(alloc_per_thread.len() - 1);
129    compress_worker_pool(
130        params,
131        owned_input,
132        output,
133        alloc_per_thread,
134        &mut work_pool,
135    )
136}
137
138#[cfg(feature = "std")]
139pub use self::multithreading::compress_multi as compress_multi_no_threadpool;
140#[cfg(not(feature = "std"))]
141pub use self::singlethreading::compress_multi;
142#[cfg(not(feature = "std"))]
143pub use self::singlethreading::compress_multi as compress_multi_no_threadpool;
144
145#[cfg(feature = "std")]
146pub fn BrotliCompress<InputType, OutputType>(
147    r: &mut InputType,
148    w: &mut OutputType,
149    params: &BrotliEncoderParams,
150) -> Result<usize, io::Error>
151where
152    InputType: Read,
153    OutputType: Write,
154{
155    let mut input_buffer: [u8; 4096] = [0; 4096];
156    let mut output_buffer: [u8; 4096] = [0; 4096];
157    BrotliCompressCustomAlloc(
158        r,
159        w,
160        &mut input_buffer[..],
161        &mut output_buffer[..],
162        params,
163        StandardAlloc::default(),
164    )
165}
166
167#[cfg(feature = "std")]
168pub fn BrotliCompressCustomAlloc<InputType, OutputType, Alloc: BrotliAlloc>(
169    r: &mut InputType,
170    w: &mut OutputType,
171    input_buffer: &mut [u8],
172    output_buffer: &mut [u8],
173    params: &BrotliEncoderParams,
174    alloc: Alloc,
175) -> Result<usize, io::Error>
176where
177    InputType: Read,
178    OutputType: Write,
179{
180    let mut nop_callback = |_data: &mut interface::PredictionModeContextMap<InputReferenceMut>,
181                            _cmds: &mut [interface::StaticCommand],
182                            _mb: interface::InputPair,
183                            _m: &mut Alloc| ();
184    BrotliCompressCustomIo(
185        &mut IoReaderWrapper::<InputType>(r),
186        &mut IoWriterWrapper::<OutputType>(w),
187        input_buffer,
188        output_buffer,
189        params,
190        alloc,
191        &mut nop_callback,
192        Error::new(ErrorKind::UnexpectedEof, "Unexpected EOF"),
193    )
194}
195
196pub fn BrotliCompressCustomIo<
197    ErrType,
198    InputType,
199    OutputType,
200    Alloc: BrotliAlloc,
201    MetablockCallback: FnMut(
202        &mut interface::PredictionModeContextMap<InputReferenceMut>,
203        &mut [interface::StaticCommand],
204        interface::InputPair,
205        &mut Alloc,
206    ),
207>(
208    r: &mut InputType,
209    w: &mut OutputType,
210    input_buffer: &mut [u8],
211    output_buffer: &mut [u8],
212    params: &BrotliEncoderParams,
213    alloc: Alloc,
214    metablock_callback: &mut MetablockCallback,
215    unexpected_eof_error_constant: ErrType,
216) -> Result<usize, ErrType>
217where
218    InputType: CustomRead<ErrType>,
219    OutputType: CustomWrite<ErrType>,
220{
221    BrotliCompressCustomIoCustomDict(
222        r,
223        w,
224        input_buffer,
225        output_buffer,
226        params,
227        alloc,
228        metablock_callback,
229        &[],
230        unexpected_eof_error_constant,
231    )
232}
233pub fn BrotliCompressCustomIoCustomDict<
234    ErrType,
235    InputType,
236    OutputType,
237    Alloc: BrotliAlloc,
238    MetablockCallback: FnMut(
239        &mut interface::PredictionModeContextMap<InputReferenceMut>,
240        &mut [interface::StaticCommand],
241        interface::InputPair,
242        &mut Alloc,
243    ),
244>(
245    r: &mut InputType,
246    w: &mut OutputType,
247    input_buffer: &mut [u8],
248    output_buffer: &mut [u8],
249    params: &BrotliEncoderParams,
250    alloc: Alloc,
251    metablock_callback: &mut MetablockCallback,
252    dict: &[u8],
253    unexpected_eof_error_constant: ErrType,
254) -> Result<usize, ErrType>
255where
256    InputType: CustomRead<ErrType>,
257    OutputType: CustomWrite<ErrType>,
258{
259    assert!(!input_buffer.is_empty());
260    assert!(!output_buffer.is_empty());
261    let mut s_orig = BrotliEncoderCreateInstance(alloc);
262    s_orig.params = params.clone();
263    if !dict.is_empty() {
264        BrotliEncoderSetCustomDictionary(&mut s_orig, dict.len(), dict);
265    }
266    let mut next_in_offset: usize = 0;
267    let mut next_out_offset: usize = 0;
268    let mut total_out = Some(0);
269    let mut read_err: Result<(), ErrType> = Ok(());
270    {
271        let s = &mut s_orig;
272
273        //BrotliEncoderSetParameter(s, BrotliEncoderParameter::BROTLI_PARAM_MODE, 0 as u32); // gen, text, font
274        //BrotliEncoderSetParameter(s,
275        //                          BrotliEncoderParameter::BROTLI_PARAM_SIZE_HINT,
276        //                          input.len() as u32);
277        let mut available_in: usize = 0;
278        let mut available_out: usize = output_buffer.len();
279        let mut eof = false;
280        loop {
281            if available_in == 0 && !eof {
282                next_in_offset = 0;
283                match r.read(input_buffer) {
284                    Err(e) => {
285                        read_err = Err(e);
286                        available_in = 0;
287                        eof = true;
288                    }
289                    Ok(size) => {
290                        if size == 0 {
291                            eof = true;
292                        }
293                        available_in = size;
294                    }
295                }
296            }
297            let op: BrotliEncoderOperation;
298            if available_in == 0 {
299                op = BrotliEncoderOperation::BROTLI_OPERATION_FINISH;
300            } else {
301                op = BrotliEncoderOperation::BROTLI_OPERATION_PROCESS;
302            }
303            let result = BrotliEncoderCompressStream(
304                s,
305                op,
306                &mut available_in,
307                input_buffer,
308                &mut next_in_offset,
309                &mut available_out,
310                output_buffer,
311                &mut next_out_offset,
312                &mut total_out,
313                metablock_callback,
314            );
315            let fin = BrotliEncoderIsFinished(s);
316            if available_out == 0 || fin != 0 {
317                let lim = output_buffer.len() - available_out;
318                assert_eq!(next_out_offset, lim);
319                next_out_offset = 0;
320                while next_out_offset < lim {
321                    match w.write(&mut output_buffer[next_out_offset..lim]) {
322                        Err(e) => {
323                            BrotliEncoderDestroyInstance(s);
324                            read_err?;
325                            return Err(e);
326                        }
327                        Ok(size) => {
328                            next_out_offset += size;
329                        }
330                    }
331                }
332                available_out = output_buffer.len();
333                next_out_offset = 0;
334            }
335            if result <= 0 {
336                if read_err.is_ok() {
337                    read_err = Err(unexpected_eof_error_constant);
338                }
339                break;
340            }
341            if fin != 0 {
342                break;
343            }
344        }
345        BrotliEncoderDestroyInstance(s);
346    }
347    read_err?;
348    Ok(total_out.unwrap())
349}
brotli/enc/mod.rs

brotli/enc/
mod.rs
