1 // SPDX-License-Identifier: Apache-2.0
2 // ----------------------------------------------------------------------------
3 // Copyright 2019-2022 Arm Limited
4 //
5 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
6 // use this file except in compliance with the License. You may obtain a copy
7 // of the License at:
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14 // License for the specific language governing permissions and limitations
15 // under the License.
16 // ----------------------------------------------------------------------------
17
18 /**
19 * @brief 4x32-bit vectors, implemented using plain C++.
20 *
21 * This module implements 4-wide 32-bit float, int, and mask vectors. This
22 * module provides a scalar fallback for VLA code, primarily useful for
23 * debugging VLA algorithms without the complexity of handling SIMD. Only the
24 * baseline level of functionality needed to support VLA is provided.
25 *
26 * Note that the vector conditional operators implemented by this module are
27 * designed to behave like SIMD conditional operators that generate lane masks.
28 * Rather than returning 0/1 booleans like normal C++ code they will return
29 * 0/-1 to give a full lane-width bitmask.
30 *
31 * Note that the documentation for this module still talks about "vectors" to
32 * help developers think about the implied VLA behavior when writing optimized
33 * paths.
34 */
35
36 #ifndef ASTC_VECMATHLIB_NONE_4_H_INCLUDED
37 #define ASTC_VECMATHLIB_NONE_4_H_INCLUDED
38
39 #ifndef ASTCENC_SIMD_INLINE
40 #error "Include astcenc_vecmathlib.h, do not include directly"
41 #endif
42
43 #include <algorithm>
44 #include <cstdio>
45 #include <cstring>
46 #include <cfenv>
47
48 // ============================================================================
49 // vfloat4 data type
50 // ============================================================================
51
52 /**
53 * @brief Data type for 4-wide floats.
54 */
55 struct vfloat4
56 {
57 /**
58 * @brief Construct from zero-initialized value.
59 */
60 ASTCENC_SIMD_INLINE vfloat4() = default;
61
62 /**
63 * @brief Construct from 4 values loaded from an unaligned address.
64 *
65 * Consider using loada() which is better with wider VLA vectors if data is
66 * aligned to vector length.
67 */
vfloat4vfloat468 ASTCENC_SIMD_INLINE explicit vfloat4(const float* p)
69 {
70 m[0] = p[0];
71 m[1] = p[1];
72 m[2] = p[2];
73 m[3] = p[3];
74 }
75
76 /**
77 * @brief Construct from 4 scalar values replicated across all lanes.
78 *
79 * Consider using zero() for constexpr zeros.
80 */
vfloat4vfloat481 ASTCENC_SIMD_INLINE explicit vfloat4(float a)
82 {
83 m[0] = a;
84 m[1] = a;
85 m[2] = a;
86 m[3] = a;
87 }
88
89 /**
90 * @brief Construct from 4 scalar values.
91 *
92 * The value of @c a is stored to lane 0 (LSB) in the SIMD register.
93 */
vfloat4vfloat494 ASTCENC_SIMD_INLINE explicit vfloat4(float a, float b, float c, float d)
95 {
96 m[0] = a;
97 m[1] = b;
98 m[2] = c;
99 m[3] = d;
100 }
101
102 /**
103 * @brief Get the scalar value of a single lane.
104 */
lanevfloat4105 template <int l> ASTCENC_SIMD_INLINE float lane() const
106 {
107 return m[l];
108 }
109
110 /**
111 * @brief Set the scalar value of a single lane.
112 */
set_lanevfloat4113 template <int l> ASTCENC_SIMD_INLINE void set_lane(float a)
114 {
115 m[l] = a;
116 }
117
118 /**
119 * @brief Factory that returns a vector of zeros.
120 */
zerovfloat4121 static ASTCENC_SIMD_INLINE vfloat4 zero()
122 {
123 return vfloat4(0.0f);
124 }
125
126 /**
127 * @brief Factory that returns a replicated scalar loaded from memory.
128 */
load1vfloat4129 static ASTCENC_SIMD_INLINE vfloat4 load1(const float* p)
130 {
131 return vfloat4(*p);
132 }
133
134 /**
135 * @brief Factory that returns a vector loaded from aligned memory.
136 */
loadavfloat4137 static ASTCENC_SIMD_INLINE vfloat4 loada(const float* p)
138 {
139 return vfloat4(p);
140 }
141
142 /**
143 * @brief Factory that returns a vector containing the lane IDs.
144 */
lane_idvfloat4145 static ASTCENC_SIMD_INLINE vfloat4 lane_id()
146 {
147 return vfloat4(0.0f, 1.0f, 2.0f, 3.0f);
148 }
149
150 /**
151 * @brief Return a swizzled float 2.
152 */
swzvfloat4153 template <int l0, int l1> ASTCENC_SIMD_INLINE vfloat4 swz() const
154 {
155 return vfloat4(lane<l0>(), lane<l1>(), 0.0f, 0.0f);
156 }
157
158 /**
159 * @brief Return a swizzled float 3.
160 */
swzvfloat4161 template <int l0, int l1, int l2> ASTCENC_SIMD_INLINE vfloat4 swz() const
162 {
163 return vfloat4(lane<l0>(), lane<l1>(), lane<l2>(), 0.0f);
164 }
165
166 /**
167 * @brief Return a swizzled float 4.
168 */
swzvfloat4169 template <int l0, int l1, int l2, int l3> ASTCENC_SIMD_INLINE vfloat4 swz() const
170 {
171 return vfloat4(lane<l0>(), lane<l1>(), lane<l2>(), lane<l3>());
172 }
173
174 /**
175 * @brief The vector ...
176 */
177 float m[4];
178 };
179
180 // ============================================================================
181 // vint4 data type
182 // ============================================================================
183
184 /**
185 * @brief Data type for 4-wide ints.
186 */
187 struct vint4
188 {
189 /**
190 * @brief Construct from zero-initialized value.
191 */
192 ASTCENC_SIMD_INLINE vint4() = default;
193
194 /**
195 * @brief Construct from 4 values loaded from an unaligned address.
196 *
197 * Consider using vint4::loada() which is better with wider VLA vectors
198 * if data is aligned.
199 */
vint4vint4200 ASTCENC_SIMD_INLINE explicit vint4(const int* p)
201 {
202 m[0] = p[0];
203 m[1] = p[1];
204 m[2] = p[2];
205 m[3] = p[3];
206 }
207
208 /**
209 * @brief Construct from 4 uint8_t loaded from an unaligned address.
210 */
vint4vint4211 ASTCENC_SIMD_INLINE explicit vint4(const uint8_t *p)
212 {
213 m[0] = p[0];
214 m[1] = p[1];
215 m[2] = p[2];
216 m[3] = p[3];
217 }
218
219 /**
220 * @brief Construct from 4 scalar values.
221 *
222 * The value of @c a is stored to lane 0 (LSB) in the SIMD register.
223 */
vint4vint4224 ASTCENC_SIMD_INLINE explicit vint4(int a, int b, int c, int d)
225 {
226 m[0] = a;
227 m[1] = b;
228 m[2] = c;
229 m[3] = d;
230 }
231
232
233 /**
234 * @brief Construct from 4 scalar values replicated across all lanes.
235 *
236 * Consider using vint4::zero() for constexpr zeros.
237 */
vint4vint4238 ASTCENC_SIMD_INLINE explicit vint4(int a)
239 {
240 m[0] = a;
241 m[1] = a;
242 m[2] = a;
243 m[3] = a;
244 }
245
246 /**
247 * @brief Get the scalar value of a single lane.
248 */
lanevint4249 template <int l> ASTCENC_SIMD_INLINE int lane() const
250 {
251 return m[l];
252 }
253
254 /**
255 * @brief Set the scalar value of a single lane.
256 */
set_lanevint4257 template <int l> ASTCENC_SIMD_INLINE void set_lane(int a)
258 {
259 m[l] = a;
260 }
261
262 /**
263 * @brief Factory that returns a vector of zeros.
264 */
zerovint4265 static ASTCENC_SIMD_INLINE vint4 zero()
266 {
267 return vint4(0);
268 }
269
270 /**
271 * @brief Factory that returns a replicated scalar loaded from memory.
272 */
load1vint4273 static ASTCENC_SIMD_INLINE vint4 load1(const int* p)
274 {
275 return vint4(*p);
276 }
277
278 /**
279 * @brief Factory that returns a vector loaded from 16B aligned memory.
280 */
loadavint4281 static ASTCENC_SIMD_INLINE vint4 loada(const int* p)
282 {
283 return vint4(p);
284 }
285
286 /**
287 * @brief Factory that returns a vector containing the lane IDs.
288 */
lane_idvint4289 static ASTCENC_SIMD_INLINE vint4 lane_id()
290 {
291 return vint4(0, 1, 2, 3);
292 }
293
294 /**
295 * @brief The vector ...
296 */
297 int m[4];
298 };
299
300 // ============================================================================
301 // vmask4 data type
302 // ============================================================================
303
304 /**
305 * @brief Data type for 4-wide control plane masks.
306 */
307 struct vmask4
308 {
309 /**
310 * @brief Construct from an existing mask value.
311 */
vmask4vmask4312 ASTCENC_SIMD_INLINE explicit vmask4(int* p)
313 {
314 m[0] = p[0];
315 m[1] = p[1];
316 m[2] = p[2];
317 m[3] = p[3];
318 }
319
320 /**
321 * @brief Construct from 1 scalar value.
322 */
vmask4vmask4323 ASTCENC_SIMD_INLINE explicit vmask4(bool a)
324 {
325 m[0] = a == false ? 0 : -1;
326 m[1] = a == false ? 0 : -1;
327 m[2] = a == false ? 0 : -1;
328 m[3] = a == false ? 0 : -1;
329 }
330
331 /**
332 * @brief Construct from 4 scalar values.
333 *
334 * The value of @c a is stored to lane 0 (LSB) in the SIMD register.
335 */
vmask4vmask4336 ASTCENC_SIMD_INLINE explicit vmask4(bool a, bool b, bool c, bool d)
337 {
338 m[0] = a == false ? 0 : -1;
339 m[1] = b == false ? 0 : -1;
340 m[2] = c == false ? 0 : -1;
341 m[3] = d == false ? 0 : -1;
342 }
343
344
345 /**
346 * @brief The vector ...
347 */
348 int m[4];
349 };
350
351 // ============================================================================
352 // vmask4 operators and functions
353 // ============================================================================
354
355 /**
356 * @brief Overload: mask union (or).
357 */
358 ASTCENC_SIMD_INLINE vmask4 operator|(vmask4 a, vmask4 b)
359 {
360 return vmask4(a.m[0] | b.m[0],
361 a.m[1] | b.m[1],
362 a.m[2] | b.m[2],
363 a.m[3] | b.m[3]);
364 }
365
366 /**
367 * @brief Overload: mask intersect (and).
368 */
369 ASTCENC_SIMD_INLINE vmask4 operator&(vmask4 a, vmask4 b)
370 {
371 return vmask4(a.m[0] & b.m[0],
372 a.m[1] & b.m[1],
373 a.m[2] & b.m[2],
374 a.m[3] & b.m[3]);
375 }
376
377 /**
378 * @brief Overload: mask difference (xor).
379 */
380 ASTCENC_SIMD_INLINE vmask4 operator^(vmask4 a, vmask4 b)
381 {
382 return vmask4(a.m[0] ^ b.m[0],
383 a.m[1] ^ b.m[1],
384 a.m[2] ^ b.m[2],
385 a.m[3] ^ b.m[3]);
386 }
387
388 /**
389 * @brief Overload: mask invert (not).
390 */
391 ASTCENC_SIMD_INLINE vmask4 operator~(vmask4 a)
392 {
393 return vmask4(~a.m[0],
394 ~a.m[1],
395 ~a.m[2],
396 ~a.m[3]);
397 }
398
399 /**
400 * @brief Return a 1-bit mask code indicating mask status.
401 *
402 * bit0 = lane 0
403 */
mask(vmask4 a)404 ASTCENC_SIMD_INLINE unsigned int mask(vmask4 a)
405 {
406 return ((a.m[0] >> 31) & 0x1) |
407 ((a.m[1] >> 30) & 0x2) |
408 ((a.m[2] >> 29) & 0x4) |
409 ((a.m[3] >> 28) & 0x8);
410 }
411
412 // ============================================================================
413 // vint4 operators and functions
414 // ============================================================================
415
416 /**
417 * @brief Overload: vector by vector addition.
418 */
419 ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, vint4 b)
420 {
421 return vint4(a.m[0] + b.m[0],
422 a.m[1] + b.m[1],
423 a.m[2] + b.m[2],
424 a.m[3] + b.m[3]);
425 }
426
427 /**
428 * @brief Overload: vector by vector subtraction.
429 */
430 ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, vint4 b)
431 {
432 return vint4(a.m[0] - b.m[0],
433 a.m[1] - b.m[1],
434 a.m[2] - b.m[2],
435 a.m[3] - b.m[3]);
436 }
437
438 /**
439 * @brief Overload: vector by vector multiplication.
440 */
441 ASTCENC_SIMD_INLINE vint4 operator*(vint4 a, vint4 b)
442 {
443 return vint4(a.m[0] * b.m[0],
444 a.m[1] * b.m[1],
445 a.m[2] * b.m[2],
446 a.m[3] * b.m[3]);
447 }
448
449 /**
450 * @brief Overload: vector bit invert.
451 */
452 ASTCENC_SIMD_INLINE vint4 operator~(vint4 a)
453 {
454 return vint4(~a.m[0],
455 ~a.m[1],
456 ~a.m[2],
457 ~a.m[3]);
458 }
459
460 /**
461 * @brief Overload: vector by vector bitwise or.
462 */
463 ASTCENC_SIMD_INLINE vint4 operator|(vint4 a, vint4 b)
464 {
465 return vint4(a.m[0] | b.m[0],
466 a.m[1] | b.m[1],
467 a.m[2] | b.m[2],
468 a.m[3] | b.m[3]);
469 }
470
471 /**
472 * @brief Overload: vector by vector bitwise and.
473 */
474 ASTCENC_SIMD_INLINE vint4 operator&(vint4 a, vint4 b)
475 {
476 return vint4(a.m[0] & b.m[0],
477 a.m[1] & b.m[1],
478 a.m[2] & b.m[2],
479 a.m[3] & b.m[3]);
480 }
481
482 /**
483 * @brief Overload: vector by vector bitwise xor.
484 */
485 ASTCENC_SIMD_INLINE vint4 operator^(vint4 a, vint4 b)
486 {
487 return vint4(a.m[0] ^ b.m[0],
488 a.m[1] ^ b.m[1],
489 a.m[2] ^ b.m[2],
490 a.m[3] ^ b.m[3]);
491 }
492
493 /**
494 * @brief Overload: vector by vector equality.
495 */
496 ASTCENC_SIMD_INLINE vmask4 operator==(vint4 a, vint4 b)
497 {
498 return vmask4(a.m[0] == b.m[0],
499 a.m[1] == b.m[1],
500 a.m[2] == b.m[2],
501 a.m[3] == b.m[3]);
502 }
503
504 /**
505 * @brief Overload: vector by vector inequality.
506 */
507 ASTCENC_SIMD_INLINE vmask4 operator!=(vint4 a, vint4 b)
508 {
509 return vmask4(a.m[0] != b.m[0],
510 a.m[1] != b.m[1],
511 a.m[2] != b.m[2],
512 a.m[3] != b.m[3]);
513 }
514
515 /**
516 * @brief Overload: vector by vector less than.
517 */
518 ASTCENC_SIMD_INLINE vmask4 operator<(vint4 a, vint4 b)
519 {
520 return vmask4(a.m[0] < b.m[0],
521 a.m[1] < b.m[1],
522 a.m[2] < b.m[2],
523 a.m[3] < b.m[3]);
524 }
525
526 /**
527 * @brief Overload: vector by vector greater than.
528 */
529 ASTCENC_SIMD_INLINE vmask4 operator>(vint4 a, vint4 b)
530 {
531 return vmask4(a.m[0] > b.m[0],
532 a.m[1] > b.m[1],
533 a.m[2] > b.m[2],
534 a.m[3] > b.m[3]);
535 }
536
537 /**
538 * @brief Logical shift left.
539 */
lsl(vint4 a)540 template <int s> ASTCENC_SIMD_INLINE vint4 lsl(vint4 a)
541 {
542 return vint4(a.m[0] << s,
543 a.m[1] << s,
544 a.m[2] << s,
545 a.m[3] << s);
546 }
547
548 /**
549 * @brief Logical shift right.
550 */
lsr(vint4 a)551 template <int s> ASTCENC_SIMD_INLINE vint4 lsr(vint4 a)
552 {
553 unsigned int as0 = static_cast<unsigned int>(a.m[0]) >> s;
554 unsigned int as1 = static_cast<unsigned int>(a.m[1]) >> s;
555 unsigned int as2 = static_cast<unsigned int>(a.m[2]) >> s;
556 unsigned int as3 = static_cast<unsigned int>(a.m[3]) >> s;
557
558 return vint4(static_cast<int>(as0),
559 static_cast<int>(as1),
560 static_cast<int>(as2),
561 static_cast<int>(as3));
562 }
563
564 /**
565 * @brief Arithmetic shift right.
566 */
asr(vint4 a)567 template <int s> ASTCENC_SIMD_INLINE vint4 asr(vint4 a)
568 {
569 return vint4(a.m[0] >> s,
570 a.m[1] >> s,
571 a.m[2] >> s,
572 a.m[3] >> s);
573 }
574
575 /**
576 * @brief Return the min vector of two vectors.
577 */
min(vint4 a,vint4 b)578 ASTCENC_SIMD_INLINE vint4 min(vint4 a, vint4 b)
579 {
580 return vint4(a.m[0] < b.m[0] ? a.m[0] : b.m[0],
581 a.m[1] < b.m[1] ? a.m[1] : b.m[1],
582 a.m[2] < b.m[2] ? a.m[2] : b.m[2],
583 a.m[3] < b.m[3] ? a.m[3] : b.m[3]);
584 }
585
586 /**
587 * @brief Return the min vector of two vectors.
588 */
max(vint4 a,vint4 b)589 ASTCENC_SIMD_INLINE vint4 max(vint4 a, vint4 b)
590 {
591 return vint4(a.m[0] > b.m[0] ? a.m[0] : b.m[0],
592 a.m[1] > b.m[1] ? a.m[1] : b.m[1],
593 a.m[2] > b.m[2] ? a.m[2] : b.m[2],
594 a.m[3] > b.m[3] ? a.m[3] : b.m[3]);
595 }
596
597 /**
598 * @brief Return the horizontal minimum of a single vector.
599 */
hmin(vint4 a)600 ASTCENC_SIMD_INLINE vint4 hmin(vint4 a)
601 {
602 int b = std::min(a.m[0], a.m[1]);
603 int c = std::min(a.m[2], a.m[3]);
604 return vint4(std::min(b, c));
605 }
606
607 /**
608 * @brief Return the horizontal maximum of a single vector.
609 */
hmax(vint4 a)610 ASTCENC_SIMD_INLINE vint4 hmax(vint4 a)
611 {
612 int b = std::max(a.m[0], a.m[1]);
613 int c = std::max(a.m[2], a.m[3]);
614 return vint4(std::max(b, c));
615 }
616
617 /**
618 * @brief Return the horizontal sum of vector lanes as a scalar.
619 */
hadd_s(vint4 a)620 ASTCENC_SIMD_INLINE int hadd_s(vint4 a)
621 {
622 return a.m[0] + a.m[1] + a.m[2] + a.m[3];
623 }
624
625 /**
626 * @brief Store a vector to an aligned memory address.
627 */
storea(vint4 a,int * p)628 ASTCENC_SIMD_INLINE void storea(vint4 a, int* p)
629 {
630 p[0] = a.m[0];
631 p[1] = a.m[1];
632 p[2] = a.m[2];
633 p[3] = a.m[3];
634 }
635
636 /**
637 * @brief Store a vector to an unaligned memory address.
638 */
store(vint4 a,int * p)639 ASTCENC_SIMD_INLINE void store(vint4 a, int* p)
640 {
641 p[0] = a.m[0];
642 p[1] = a.m[1];
643 p[2] = a.m[2];
644 p[3] = a.m[3];
645 }
646
647 /**
648 * @brief Store lowest N (vector width) bytes into an unaligned address.
649 */
store_nbytes(vint4 a,uint8_t * p)650 ASTCENC_SIMD_INLINE void store_nbytes(vint4 a, uint8_t* p)
651 {
652 int* pi = reinterpret_cast<int*>(p);
653 *pi = a.m[0];
654 }
655
656 /**
657 * @brief Gather N (vector width) indices from the array.
658 */
gatheri(const int * base,vint4 indices)659 ASTCENC_SIMD_INLINE vint4 gatheri(const int* base, vint4 indices)
660 {
661 return vint4(base[indices.m[0]],
662 base[indices.m[1]],
663 base[indices.m[2]],
664 base[indices.m[3]]);
665 }
666
667 /**
668 * @brief Pack low 8 bits of N (vector width) lanes into bottom of vector.
669 */
pack_low_bytes(vint4 a)670 ASTCENC_SIMD_INLINE vint4 pack_low_bytes(vint4 a)
671 {
672 int b0 = a.m[0] & 0xFF;
673 int b1 = a.m[1] & 0xFF;
674 int b2 = a.m[2] & 0xFF;
675 int b3 = a.m[3] & 0xFF;
676
677 int b = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
678 return vint4(b, 0, 0, 0);
679 }
680
681 /**
682 * @brief Return lanes from @c b if MSB of @c cond is set, else @c a.
683 */
select(vint4 a,vint4 b,vmask4 cond)684 ASTCENC_SIMD_INLINE vint4 select(vint4 a, vint4 b, vmask4 cond)
685 {
686 return vint4((cond.m[0] & static_cast<int>(0x80000000)) ? b.m[0] : a.m[0],
687 (cond.m[1] & static_cast<int>(0x80000000)) ? b.m[1] : a.m[1],
688 (cond.m[2] & static_cast<int>(0x80000000)) ? b.m[2] : a.m[2],
689 (cond.m[3] & static_cast<int>(0x80000000)) ? b.m[3] : a.m[3]);
690 }
691
692 // ============================================================================
693 // vfloat4 operators and functions
694 // ============================================================================
695
696 /**
697 * @brief Overload: vector by vector addition.
698 */
699 ASTCENC_SIMD_INLINE vfloat4 operator+(vfloat4 a, vfloat4 b)
700 {
701 return vfloat4(a.m[0] + b.m[0],
702 a.m[1] + b.m[1],
703 a.m[2] + b.m[2],
704 a.m[3] + b.m[3]);
705 }
706
707 /**
708 * @brief Overload: vector by vector subtraction.
709 */
710 ASTCENC_SIMD_INLINE vfloat4 operator-(vfloat4 a, vfloat4 b)
711 {
712 return vfloat4(a.m[0] - b.m[0],
713 a.m[1] - b.m[1],
714 a.m[2] - b.m[2],
715 a.m[3] - b.m[3]);
716 }
717
718 /**
719 * @brief Overload: vector by vector multiplication.
720 */
721 ASTCENC_SIMD_INLINE vfloat4 operator*(vfloat4 a, vfloat4 b)
722 {
723 return vfloat4(a.m[0] * b.m[0],
724 a.m[1] * b.m[1],
725 a.m[2] * b.m[2],
726 a.m[3] * b.m[3]);
727 }
728
729 /**
730 * @brief Overload: vector by vector division.
731 */
732 ASTCENC_SIMD_INLINE vfloat4 operator/(vfloat4 a, vfloat4 b)
733 {
734 return vfloat4(a.m[0] / b.m[0],
735 a.m[1] / b.m[1],
736 a.m[2] / b.m[2],
737 a.m[3] / b.m[3]);
738 }
739
740 /**
741 * @brief Overload: vector by vector equality.
742 */
743 ASTCENC_SIMD_INLINE vmask4 operator==(vfloat4 a, vfloat4 b)
744 {
745 return vmask4(a.m[0] == b.m[0],
746 a.m[1] == b.m[1],
747 a.m[2] == b.m[2],
748 a.m[3] == b.m[3]);
749 }
750
751 /**
752 * @brief Overload: vector by vector inequality.
753 */
754 ASTCENC_SIMD_INLINE vmask4 operator!=(vfloat4 a, vfloat4 b)
755 {
756 return vmask4(a.m[0] != b.m[0],
757 a.m[1] != b.m[1],
758 a.m[2] != b.m[2],
759 a.m[3] != b.m[3]);
760 }
761
762 /**
763 * @brief Overload: vector by vector less than.
764 */
765 ASTCENC_SIMD_INLINE vmask4 operator<(vfloat4 a, vfloat4 b)
766 {
767 return vmask4(a.m[0] < b.m[0],
768 a.m[1] < b.m[1],
769 a.m[2] < b.m[2],
770 a.m[3] < b.m[3]);
771 }
772
773 /**
774 * @brief Overload: vector by vector greater than.
775 */
776 ASTCENC_SIMD_INLINE vmask4 operator>(vfloat4 a, vfloat4 b)
777 {
778 return vmask4(a.m[0] > b.m[0],
779 a.m[1] > b.m[1],
780 a.m[2] > b.m[2],
781 a.m[3] > b.m[3]);
782 }
783
784 /**
785 * @brief Overload: vector by vector less than or equal.
786 */
787 ASTCENC_SIMD_INLINE vmask4 operator<=(vfloat4 a, vfloat4 b)
788 {
789 return vmask4(a.m[0] <= b.m[0],
790 a.m[1] <= b.m[1],
791 a.m[2] <= b.m[2],
792 a.m[3] <= b.m[3]);
793 }
794
795 /**
796 * @brief Overload: vector by vector greater than or equal.
797 */
798 ASTCENC_SIMD_INLINE vmask4 operator>=(vfloat4 a, vfloat4 b)
799 {
800 return vmask4(a.m[0] >= b.m[0],
801 a.m[1] >= b.m[1],
802 a.m[2] >= b.m[2],
803 a.m[3] >= b.m[3]);
804 }
805
806 /**
807 * @brief Return the min vector of two vectors.
808 *
809 * If either lane value is NaN, @c b will be returned for that lane.
810 */
min(vfloat4 a,vfloat4 b)811 ASTCENC_SIMD_INLINE vfloat4 min(vfloat4 a, vfloat4 b)
812 {
813 return vfloat4(a.m[0] < b.m[0] ? a.m[0] : b.m[0],
814 a.m[1] < b.m[1] ? a.m[1] : b.m[1],
815 a.m[2] < b.m[2] ? a.m[2] : b.m[2],
816 a.m[3] < b.m[3] ? a.m[3] : b.m[3]);
817 }
818
819 /**
820 * @brief Return the max vector of two vectors.
821 *
822 * If either lane value is NaN, @c b will be returned for that lane.
823 */
max(vfloat4 a,vfloat4 b)824 ASTCENC_SIMD_INLINE vfloat4 max(vfloat4 a, vfloat4 b)
825 {
826 return vfloat4(a.m[0] > b.m[0] ? a.m[0] : b.m[0],
827 a.m[1] > b.m[1] ? a.m[1] : b.m[1],
828 a.m[2] > b.m[2] ? a.m[2] : b.m[2],
829 a.m[3] > b.m[3] ? a.m[3] : b.m[3]);
830 }
831
832 /**
833 * @brief Return the absolute value of the float vector.
834 */
abs(vfloat4 a)835 ASTCENC_SIMD_INLINE vfloat4 abs(vfloat4 a)
836 {
837 return vfloat4(std::abs(a.m[0]),
838 std::abs(a.m[1]),
839 std::abs(a.m[2]),
840 std::abs(a.m[3]));
841 }
842
843 /**
844 * @brief Return a float rounded to the nearest integer value.
845 */
round(vfloat4 a)846 ASTCENC_SIMD_INLINE vfloat4 round(vfloat4 a)
847 {
848 assert(std::fegetround() == FE_TONEAREST);
849 return vfloat4(std::nearbyint(a.m[0]),
850 std::nearbyint(a.m[1]),
851 std::nearbyint(a.m[2]),
852 std::nearbyint(a.m[3]));
853 }
854
855 /**
856 * @brief Return the horizontal minimum of a vector.
857 */
hmin(vfloat4 a)858 ASTCENC_SIMD_INLINE vfloat4 hmin(vfloat4 a)
859 {
860 float tmp1 = std::min(a.m[0], a.m[1]);
861 float tmp2 = std::min(a.m[2], a.m[3]);
862 return vfloat4(std::min(tmp1, tmp2));
863 }
864
865 /**
866 * @brief Return the horizontal maximum of a vector.
867 */
hmax(vfloat4 a)868 ASTCENC_SIMD_INLINE vfloat4 hmax(vfloat4 a)
869 {
870 float tmp1 = std::max(a.m[0], a.m[1]);
871 float tmp2 = std::max(a.m[2], a.m[3]);
872 return vfloat4(std::max(tmp1, tmp2));
873 }
874
875 /**
876 * @brief Return the horizontal sum of a vector.
877 */
hadd_s(vfloat4 a)878 ASTCENC_SIMD_INLINE float hadd_s(vfloat4 a)
879 {
880 // Use halving add, gives invariance with SIMD versions
881 return (a.m[0] + a.m[2]) + (a.m[1] + a.m[3]);
882 }
883
884 /**
885 * @brief Return the sqrt of the lanes in the vector.
886 */
sqrt(vfloat4 a)887 ASTCENC_SIMD_INLINE vfloat4 sqrt(vfloat4 a)
888 {
889 return vfloat4(std::sqrt(a.m[0]),
890 std::sqrt(a.m[1]),
891 std::sqrt(a.m[2]),
892 std::sqrt(a.m[3]));
893 }
894
895 /**
896 * @brief Return lanes from @c b if @c cond is set, else @c a.
897 */
select(vfloat4 a,vfloat4 b,vmask4 cond)898 ASTCENC_SIMD_INLINE vfloat4 select(vfloat4 a, vfloat4 b, vmask4 cond)
899 {
900 return vfloat4((cond.m[0] & static_cast<int>(0x80000000)) ? b.m[0] : a.m[0],
901 (cond.m[1] & static_cast<int>(0x80000000)) ? b.m[1] : a.m[1],
902 (cond.m[2] & static_cast<int>(0x80000000)) ? b.m[2] : a.m[2],
903 (cond.m[3] & static_cast<int>(0x80000000)) ? b.m[3] : a.m[3]);
904 }
905
906 /**
907 * @brief Return lanes from @c b if MSB of @c cond is set, else @c a.
908 */
select_msb(vfloat4 a,vfloat4 b,vmask4 cond)909 ASTCENC_SIMD_INLINE vfloat4 select_msb(vfloat4 a, vfloat4 b, vmask4 cond)
910 {
911 return vfloat4((cond.m[0] & static_cast<int>(0x80000000)) ? b.m[0] : a.m[0],
912 (cond.m[1] & static_cast<int>(0x80000000)) ? b.m[1] : a.m[1],
913 (cond.m[2] & static_cast<int>(0x80000000)) ? b.m[2] : a.m[2],
914 (cond.m[3] & static_cast<int>(0x80000000)) ? b.m[3] : a.m[3]);
915 }
916
917 /**
918 * @brief Load a vector of gathered results from an array;
919 */
gatherf(const float * base,vint4 indices)920 ASTCENC_SIMD_INLINE vfloat4 gatherf(const float* base, vint4 indices)
921 {
922 return vfloat4(base[indices.m[0]],
923 base[indices.m[1]],
924 base[indices.m[2]],
925 base[indices.m[3]]);
926 }
927
928 /**
929 * @brief Store a vector to an unaligned memory address.
930 */
store(vfloat4 a,float * ptr)931 ASTCENC_SIMD_INLINE void store(vfloat4 a, float* ptr)
932 {
933 ptr[0] = a.m[0];
934 ptr[1] = a.m[1];
935 ptr[2] = a.m[2];
936 ptr[3] = a.m[3];
937 }
938
939 /**
940 * @brief Store a vector to an aligned memory address.
941 */
storea(vfloat4 a,float * ptr)942 ASTCENC_SIMD_INLINE void storea(vfloat4 a, float* ptr)
943 {
944 ptr[0] = a.m[0];
945 ptr[1] = a.m[1];
946 ptr[2] = a.m[2];
947 ptr[3] = a.m[3];
948 }
949
950 /**
951 * @brief Return a integer value for a float vector, using truncation.
952 */
float_to_int(vfloat4 a)953 ASTCENC_SIMD_INLINE vint4 float_to_int(vfloat4 a)
954 {
955 return vint4(static_cast<int>(a.m[0]),
956 static_cast<int>(a.m[1]),
957 static_cast<int>(a.m[2]),
958 static_cast<int>(a.m[3]));
959 }
960
961 /**f
962 * @brief Return a integer value for a float vector, using round-to-nearest.
963 */
float_to_int_rtn(vfloat4 a)964 ASTCENC_SIMD_INLINE vint4 float_to_int_rtn(vfloat4 a)
965 {
966 return vint4(static_cast<int>(a.m[0] + 0.5f),
967 static_cast<int>(a.m[1] + 0.5f),
968 static_cast<int>(a.m[2] + 0.5f),
969 static_cast<int>(a.m[3] + 0.5f));
970 }
971
972 /**
973 * @brief Return a float value for a integer vector.
974 */
int_to_float(vint4 a)975 ASTCENC_SIMD_INLINE vfloat4 int_to_float(vint4 a)
976 {
977 return vfloat4(static_cast<float>(a.m[0]),
978 static_cast<float>(a.m[1]),
979 static_cast<float>(a.m[2]),
980 static_cast<float>(a.m[3]));
981 }
982
983 /**
984 * @brief Return a float16 value for a float vector, using round-to-nearest.
985 */
float_to_float16(vfloat4 a)986 ASTCENC_SIMD_INLINE vint4 float_to_float16(vfloat4 a)
987 {
988 return vint4(
989 float_to_sf16(a.lane<0>()),
990 float_to_sf16(a.lane<1>()),
991 float_to_sf16(a.lane<2>()),
992 float_to_sf16(a.lane<3>()));
993 }
994
995 /**
996 * @brief Return a float16 value for a float scalar, using round-to-nearest.
997 */
float_to_float16(float a)998 static inline uint16_t float_to_float16(float a)
999 {
1000 return float_to_sf16(a);
1001 }
1002
1003 /**
1004 * @brief Return a float value for a float16 vector.
1005 */
float16_to_float(vint4 a)1006 ASTCENC_SIMD_INLINE vfloat4 float16_to_float(vint4 a)
1007 {
1008 return vfloat4(
1009 sf16_to_float(static_cast<uint16_t>(a.lane<0>())),
1010 sf16_to_float(static_cast<uint16_t>(a.lane<1>())),
1011 sf16_to_float(static_cast<uint16_t>(a.lane<2>())),
1012 sf16_to_float(static_cast<uint16_t>(a.lane<3>())));
1013 }
1014
1015 /**
1016 * @brief Return a float value for a float16 scalar.
1017 */
float16_to_float(uint16_t a)1018 ASTCENC_SIMD_INLINE float float16_to_float(uint16_t a)
1019 {
1020 return sf16_to_float(a);
1021 }
1022
1023 /**
1024 * @brief Return a float value as an integer bit pattern (i.e. no conversion).
1025 *
1026 * It is a common trick to convert floats into integer bit patterns, perform
1027 * some bit hackery based on knowledge they are IEEE 754 layout, and then
1028 * convert them back again. This is the first half of that flip.
1029 */
float_as_int(vfloat4 a)1030 ASTCENC_SIMD_INLINE vint4 float_as_int(vfloat4 a)
1031 {
1032 vint4 r;
1033 memcpy(r.m, a.m, 4 * 4);
1034 return r;
1035 }
1036
1037 /**
1038 * @brief Return a integer value as a float bit pattern (i.e. no conversion).
1039 *
1040 * It is a common trick to convert floats into integer bit patterns, perform
1041 * some bit hackery based on knowledge they are IEEE 754 layout, and then
1042 * convert them back again. This is the second half of that flip.
1043 */
int_as_float(vint4 a)1044 ASTCENC_SIMD_INLINE vfloat4 int_as_float(vint4 a)
1045 {
1046 vfloat4 r;
1047 memcpy(r.m, a.m, 4 * 4);
1048 return r;
1049 }
1050
1051 /**
1052 * @brief Prepare a vtable lookup table for use with the native SIMD size.
1053 */
vtable_prepare(vint4 t0,vint4 & t0p)1054 ASTCENC_SIMD_INLINE void vtable_prepare(vint4 t0, vint4& t0p)
1055 {
1056 t0p = t0;
1057 }
1058
1059 /**
1060 * @brief Prepare a vtable lookup table for use with the native SIMD size.
1061 */
vtable_prepare(vint4 t0,vint4 t1,vint4 & t0p,vint4 & t1p)1062 ASTCENC_SIMD_INLINE void vtable_prepare(vint4 t0, vint4 t1, vint4& t0p, vint4& t1p)
1063 {
1064 t0p = t0;
1065 t1p = t1;
1066 }
1067
1068 /**
1069 * @brief Prepare a vtable lookup table for use with the native SIMD size.
1070 */
vtable_prepare(vint4 t0,vint4 t1,vint4 t2,vint4 t3,vint4 & t0p,vint4 & t1p,vint4 & t2p,vint4 & t3p)1071 ASTCENC_SIMD_INLINE void vtable_prepare(
1072 vint4 t0, vint4 t1, vint4 t2, vint4 t3,
1073 vint4& t0p, vint4& t1p, vint4& t2p, vint4& t3p)
1074 {
1075 t0p = t0;
1076 t1p = t1;
1077 t2p = t2;
1078 t3p = t3;
1079 }
1080
1081 /**
1082 * @brief Perform an 8-bit 32-entry table lookup, with 32-bit indexes.
1083 */
vtable_8bt_32bi(vint4 t0,vint4 idx)1084 ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 idx)
1085 {
1086 uint8_t table[16];
1087 storea(t0, reinterpret_cast<int*>(table + 0));
1088
1089 return vint4(table[idx.lane<0>()],
1090 table[idx.lane<1>()],
1091 table[idx.lane<2>()],
1092 table[idx.lane<3>()]);
1093 }
1094
1095
1096 /**
1097 * @brief Perform an 8-bit 32-entry table lookup, with 32-bit indexes.
1098 */
vtable_8bt_32bi(vint4 t0,vint4 t1,vint4 idx)1099 ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 t1, vint4 idx)
1100 {
1101 uint8_t table[32];
1102 storea(t0, reinterpret_cast<int*>(table + 0));
1103 storea(t1, reinterpret_cast<int*>(table + 16));
1104
1105 return vint4(table[idx.lane<0>()],
1106 table[idx.lane<1>()],
1107 table[idx.lane<2>()],
1108 table[idx.lane<3>()]);
1109 }
1110
1111 /**
1112 * @brief Perform an 8-bit 64-entry table lookup, with 32-bit indexes.
1113 */
vtable_8bt_32bi(vint4 t0,vint4 t1,vint4 t2,vint4 t3,vint4 idx)1114 ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 t1, vint4 t2, vint4 t3, vint4 idx)
1115 {
1116 uint8_t table[64];
1117 storea(t0, reinterpret_cast<int*>(table + 0));
1118 storea(t1, reinterpret_cast<int*>(table + 16));
1119 storea(t2, reinterpret_cast<int*>(table + 32));
1120 storea(t3, reinterpret_cast<int*>(table + 48));
1121
1122 return vint4(table[idx.lane<0>()],
1123 table[idx.lane<1>()],
1124 table[idx.lane<2>()],
1125 table[idx.lane<3>()]);
1126 }
1127
1128 /**
1129 * @brief Return a vector of interleaved RGBA data.
1130 *
1131 * Input vectors have the value stored in the bottom 8 bits of each lane,
1132 * with high bits set to zero.
1133 *
1134 * Output vector stores a single RGBA texel packed in each lane.
1135 */
interleave_rgba8(vint4 r,vint4 g,vint4 b,vint4 a)1136 ASTCENC_SIMD_INLINE vint4 interleave_rgba8(vint4 r, vint4 g, vint4 b, vint4 a)
1137 {
1138 return r + lsl<8>(g) + lsl<16>(b) + lsl<24>(a);
1139 }
1140
1141 /**
1142 * @brief Store a vector, skipping masked lanes.
1143 *
1144 * All masked lanes must be at the end of vector, after all non-masked lanes.
1145 */
store_lanes_masked(int * base,vint4 data,vmask4 mask)1146 ASTCENC_SIMD_INLINE void store_lanes_masked(int* base, vint4 data, vmask4 mask)
1147 {
1148 if (mask.m[3])
1149 {
1150 store(data, base);
1151 }
1152 else if (mask.m[2])
1153 {
1154 base[0] = data.lane<0>();
1155 base[1] = data.lane<1>();
1156 base[2] = data.lane<2>();
1157 }
1158 else if (mask.m[1])
1159 {
1160 base[0] = data.lane<0>();
1161 base[1] = data.lane<1>();
1162 }
1163 else if (mask.m[0])
1164 {
1165 base[0] = data.lane<0>();
1166 }
1167 }
1168
1169 #endif // #ifndef ASTC_VECMATHLIB_NONE_4_H_INCLUDED
1170