moxcms/conversions/avx/
rgb_xyz_opt.rs

1/*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1.  Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2.  Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3.  Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29use crate::conversions::avx::rgb_xyz::AvxAlignedU16;
30use crate::conversions::rgbxyz::TransformMatrixShaperOptimizedV;
31use crate::transform::PointeeSizeExpressible;
32use crate::{CmsError, Layout, TransformExecutor};
33use num_traits::AsPrimitive;
34use std::arch::x86_64::*;
35
36pub(crate) struct TransformShaperRgbOptAvx<
37    T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
38    const SRC_LAYOUT: u8,
39    const DST_LAYOUT: u8,
40> {
41    pub(crate) profile: TransformMatrixShaperOptimizedV<T>,
42    pub(crate) bit_depth: usize,
43    pub(crate) gamma_lut: usize,
44}
45
46impl<
47    T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
48    const SRC_LAYOUT: u8,
49    const DST_LAYOUT: u8,
50> TransformShaperRgbOptAvx<T, SRC_LAYOUT, DST_LAYOUT>
51where
52    u32: AsPrimitive<T>,
53{
54    #[target_feature(enable = "avx2", enable = "fma")]
55    unsafe fn transform_impl<const FMA: bool>(
56        &self,
57        src: &[T],
58        dst: &mut [T],
59    ) -> Result<(), CmsError> {
60        let src_cn = Layout::from(SRC_LAYOUT);
61        let dst_cn = Layout::from(DST_LAYOUT);
62        let src_channels = src_cn.channels();
63        let dst_channels = dst_cn.channels();
64
65        let mut temporary0 = AvxAlignedU16([0; 16]);
66
67        if src.len() / src_channels != dst.len() / dst_channels {
68            return Err(CmsError::LaneSizeMismatch);
69        }
70        if src.len() % src_channels != 0 {
71            return Err(CmsError::LaneMultipleOfChannels);
72        }
73        if dst.len() % dst_channels != 0 {
74            return Err(CmsError::LaneMultipleOfChannels);
75        }
76
77        let t = self.profile.adaptation_matrix.transpose();
78
79        let scale = (self.gamma_lut - 1) as f32;
80        let max_colors: T = ((1 << self.bit_depth) - 1).as_();
81
82        // safety precondition for linearization table
83        if T::FINITE {
84            let cap = (1 << self.bit_depth) - 1;
85            assert!(self.profile.linear.len() >= cap);
86        } else {
87            assert!(self.profile.linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
88        }
89
90        let lut_lin = &self.profile.linear;
91
92        unsafe {
93            let m0 = _mm256_setr_ps(
94                t.v[0][0], t.v[0][1], t.v[0][2], 0., t.v[0][0], t.v[0][1], t.v[0][2], 0.,
95            );
96            let m1 = _mm256_setr_ps(
97                t.v[1][0], t.v[1][1], t.v[1][2], 0., t.v[1][0], t.v[1][1], t.v[1][2], 0.,
98            );
99            let m2 = _mm256_setr_ps(
100                t.v[2][0], t.v[2][1], t.v[2][2], 0., t.v[2][0], t.v[2][1], t.v[2][2], 0.,
101            );
102
103            let zeros = _mm_setzero_ps();
104
105            let v_scale = _mm256_set1_ps(scale);
106
107            let mut src = src;
108            let mut dst = dst;
109
110            let mut src_iter = src.chunks_exact(src_channels * 2);
111            let dst_iter = dst.chunks_exact_mut(dst_channels * 2);
112
113            let (mut r0, mut g0, mut b0, mut a0);
114            let (mut r1, mut g1, mut b1, mut a1);
115
116            if let Some(src) = src_iter.next() {
117                r0 = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.r_i()]._as_usize()));
118                g0 = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.g_i()]._as_usize()));
119                b0 = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.b_i()]._as_usize()));
120                r1 = _mm_broadcast_ss(
121                    lut_lin.get_unchecked(src[src_cn.r_i() + src_channels]._as_usize()),
122                );
123                g1 = _mm_broadcast_ss(
124                    lut_lin.get_unchecked(src[src_cn.g_i() + src_channels]._as_usize()),
125                );
126                b1 = _mm_broadcast_ss(
127                    lut_lin.get_unchecked(src[src_cn.b_i() + src_channels]._as_usize()),
128                );
129                a0 = if src_channels == 4 {
130                    src[src_cn.a_i()]
131                } else {
132                    max_colors
133                };
134                a1 = if src_channels == 4 {
135                    src[src_cn.a_i() + src_channels]
136                } else {
137                    max_colors
138                };
139            } else {
140                r0 = _mm_setzero_ps();
141                g0 = _mm_setzero_ps();
142                b0 = _mm_setzero_ps();
143                a0 = max_colors;
144                r1 = _mm_setzero_ps();
145                g1 = _mm_setzero_ps();
146                b1 = _mm_setzero_ps();
147                a1 = max_colors;
148            }
149
150            for (src, dst) in src_iter.zip(dst_iter) {
151                let r = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(r0), r1);
152                let g = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(g0), g1);
153                let b = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(b0), b1);
154
155                let mut v = if FMA {
156                    let v0 = _mm256_mul_ps(r, m0);
157                    let v1 = _mm256_fmadd_ps(g, m1, v0);
158                    _mm256_fmadd_ps(b, m2, v1)
159                } else {
160                    let v0 = _mm256_mul_ps(r, m0);
161                    let v1 = _mm256_mul_ps(g, m1);
162                    let v2 = _mm256_mul_ps(b, m2);
163
164                    _mm256_add_ps(_mm256_add_ps(v0, v1), v2)
165                };
166
167                v = _mm256_max_ps(v, _mm256_setzero_ps());
168                v = _mm256_mul_ps(v, v_scale);
169                v = _mm256_min_ps(v, v_scale);
170
171                let zx = _mm256_cvtps_epi32(v);
172                _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, zx);
173
174                r0 = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.r_i()]._as_usize()));
175                g0 = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.g_i()]._as_usize()));
176                b0 = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.b_i()]._as_usize()));
177                r1 = _mm_broadcast_ss(
178                    lut_lin.get_unchecked(src[src_cn.r_i() + src_channels]._as_usize()),
179                );
180                g1 = _mm_broadcast_ss(
181                    lut_lin.get_unchecked(src[src_cn.g_i() + src_channels]._as_usize()),
182                );
183                b1 = _mm_broadcast_ss(
184                    lut_lin.get_unchecked(src[src_cn.b_i() + src_channels]._as_usize()),
185                );
186
187                dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
188                dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
189                dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
190                if dst_channels == 4 {
191                    dst[dst_cn.a_i()] = a0;
192                }
193
194                dst[dst_cn.r_i() + dst_channels] = self.profile.gamma[temporary0.0[8] as usize];
195                dst[dst_cn.g_i() + dst_channels] = self.profile.gamma[temporary0.0[10] as usize];
196                dst[dst_cn.b_i() + dst_channels] = self.profile.gamma[temporary0.0[12] as usize];
197                if dst_channels == 4 {
198                    dst[dst_cn.a_i() + dst_channels] = a1;
199                }
200
201                a0 = if src_channels == 4 {
202                    src[src_cn.a_i()]
203                } else {
204                    max_colors
205                };
206                a1 = if src_channels == 4 {
207                    src[src_cn.a_i() + src_channels]
208                } else {
209                    max_colors
210                };
211            }
212
213            if let Some(dst) = dst.chunks_exact_mut(dst_channels * 2).last() {
214                let r = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(r0), r1);
215                let g = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(g0), g1);
216                let b = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(b0), b1);
217
218                let mut v = if FMA {
219                    let v0 = _mm256_mul_ps(r, m0);
220                    let v1 = _mm256_fmadd_ps(g, m1, v0);
221                    _mm256_fmadd_ps(b, m2, v1)
222                } else {
223                    let v0 = _mm256_mul_ps(r, m0);
224                    let v1 = _mm256_mul_ps(g, m1);
225                    let v2 = _mm256_mul_ps(b, m2);
226
227                    _mm256_add_ps(_mm256_add_ps(v0, v1), v2)
228                };
229
230                v = _mm256_max_ps(v, _mm256_setzero_ps());
231                v = _mm256_mul_ps(v, v_scale);
232                v = _mm256_min_ps(v, v_scale);
233
234                let zx = _mm256_cvtps_epi32(v);
235                _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, zx);
236
237                dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
238                dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
239                dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
240                if dst_channels == 4 {
241                    dst[dst_cn.a_i()] = a0;
242                }
243
244                dst[dst_cn.r_i() + dst_channels] = self.profile.gamma[temporary0.0[8] as usize];
245                dst[dst_cn.g_i() + dst_channels] = self.profile.gamma[temporary0.0[10] as usize];
246                dst[dst_cn.b_i() + dst_channels] = self.profile.gamma[temporary0.0[12] as usize];
247                if dst_channels == 4 {
248                    dst[dst_cn.a_i() + dst_channels] = a1;
249                }
250            }
251
252            src = src.chunks_exact(src_channels * 2).remainder();
253            dst = dst.chunks_exact_mut(dst_channels * 2).into_remainder();
254
255            for (src, dst) in src
256                .chunks_exact(src_channels)
257                .zip(dst.chunks_exact_mut(dst_channels))
258            {
259                let r = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.r_i()]._as_usize()));
260                let g = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.g_i()]._as_usize()));
261                let b = _mm_broadcast_ss(lut_lin.get_unchecked(src[src_cn.b_i()]._as_usize()));
262                let a = if src_channels == 4 {
263                    src[src_cn.a_i()]
264                } else {
265                    max_colors
266                };
267
268                let mut v = if FMA {
269                    let v0 = _mm_mul_ps(r, _mm256_castps256_ps128(m0));
270                    let v1 = _mm_fmadd_ps(g, _mm256_castps256_ps128(m1), v0);
271                    _mm_fmadd_ps(b, _mm256_castps256_ps128(m2), v1)
272                } else {
273                    let v0 = _mm_mul_ps(r, _mm256_castps256_ps128(m0));
274                    let v1 = _mm_mul_ps(g, _mm256_castps256_ps128(m1));
275                    let v2 = _mm_mul_ps(b, _mm256_castps256_ps128(m2));
276
277                    _mm_add_ps(_mm_add_ps(v0, v1), v2)
278                };
279
280                v = _mm_max_ps(v, zeros);
281                v = _mm_mul_ps(v, _mm256_castps256_ps128(v_scale));
282                v = _mm_min_ps(v, _mm256_castps256_ps128(v_scale));
283
284                let zx = _mm_cvtps_epi32(v);
285                _mm_store_si128(temporary0.0.as_mut_ptr() as *mut _, zx);
286
287                dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
288                dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
289                dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
290                if dst_channels == 4 {
291                    dst[dst_cn.a_i()] = a;
292                }
293            }
294        }
295
296        Ok(())
297    }
298}
299
300impl<
301    T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
302    const SRC_LAYOUT: u8,
303    const DST_LAYOUT: u8,
304> TransformExecutor<T> for TransformShaperRgbOptAvx<T, SRC_LAYOUT, DST_LAYOUT>
305where
306    u32: AsPrimitive<T>,
307{
308    fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
309        unsafe {
310            assert!(std::arch::is_x86_feature_detected!("fma"));
311            self.transform_impl::<true>(src, dst)
312        }
313    }
314}
moxcms/conversions/avx/rgb_xyz_opt.rs

moxcms/conversions/avx/
rgb_xyz_opt.rs
