AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

skvx spring cleaning

Browse Source 
- remove some workarounds
  - more SI/SIN/SIT/SINT use
  - rewrap a lot of code to 100 cols
  - etc. misc.

Change-Id: I78b7ff272afcbb8658cf147aad8af85d0e2acf42
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/314676
Auto-Submit: Mike Klein <mtklein@google.com>
Commit-Queue: Herb Derby <herb@google.com>
Reviewed-by: Herb Derby <herb@google.com>
This commit is contained in:
Mike Klein 2020-09-02 09:00:57 -05:00 committed by Skia Commit-Bot
parent 5bc0b65a4b
commit a1711092b2
1 changed files with 189 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

369
include/private/SkVx.h
View File

@ -16,10 +16,8 @@
//
// We've also fixed a few of the caveats that used to make SkNx awkward to work
// with across translation units. skvx::Vec<N,T> always has N*sizeof(T) size
// and alignment[1][2] and is safe to use across translation units freely.
//
// [1] Ideally we'd only align to T, but that tanks ARMv7 NEON codegen.
// [2] Some compilers barf if we try to use N*sizeof(T), so instead we leave them at T.
// and alignment and is safe to use across translation units freely.
// Ideally we'd only align to T, but that tanks ARMv7 NEON codegen.
// Please try to keep this file independent of Skia headers.
#include <algorithm> // std::min, std::max
@ -32,48 +30,30 @@
#include <immintrin.h>
#elif defined(__ARM_NEON)
#include <arm_neon.h>
#endif
#if defined __wasm_simd128__
// WASM SIMD intrinsics definitions: https://github.com/llvm/llvm-project/blob/master/clang/lib/Headers/wasm_simd128.h
#elif defined(__wasm_simd128__)
#include <wasm_simd128.h>
#endif
#if !defined(__clang__) && defined(__GNUC__) && defined(__mips64)
// GCC 7 hits an internal compiler error when targeting MIPS64.
#define SKVX_ALIGNMENT
#elif !defined(__clang__) && defined(_MSC_VER) && defined(_M_IX86)
// Our SkVx unit tests fail when built by MSVC for 32-bit x86.
#define SKVX_ALIGNMENT
#else
#define SKVX_ALIGNMENT alignas(N * sizeof(T))
#endif
#if defined(__GNUC__) && !defined(__clang__) && defined(__SSE__)
// GCC warns about ABI changes when returning >= 32 byte vectors when -mavx is not enabled.
// This only happens for types like VExt whose ABI we don't care about, not for Vec itself.
#pragma GCC diagnostic ignored "-Wpsabi"
#endif
// To avoid ODR violations, all methods must be force-inlined,
// and all standalone functions must be static, perhaps using these helpers.
// To avoid ODR violations, all methods must be force-inlined...
#if defined(_MSC_VER)
#define SKVX_ALWAYS_INLINE __forceinline
#else
#define SKVX_ALWAYS_INLINE __attribute__((always_inline))
#endif
#define SIT template < typename T> static inline
#define SINT template <int N, typename T> static inline
// ... and all standalone functions must be static. Please use these helpers:
#define SI static inline
#define SIT template < typename T> SI
#define SIN template <int N > SI
#define SINT template <int N, typename T> SI
#define SINTU template <int N, typename T, typename U, \
typename=typename std::enable_if<std::is_convertible<U,T>::value>::type> \
static inline
typename=std::enable_if_t<std::is_convertible<U,T>::value>> SI
namespace skvx {
// All Vec have the same simple memory layout, the same as `T vec[N]`.
template <int N, typename T>
struct SKVX_ALIGNMENT Vec {
struct alignas(N*sizeof(T)) Vec {
static_assert((N & (N-1)) == 0, "N must be a power of 2.");
static_assert(sizeof(T) >= alignof(T), "What kind of crazy T is this?");
@ -86,8 +66,7 @@ struct SKVX_ALIGNMENT Vec {
SKVX_ALWAYS_INLINE Vec() = default;
template <typename U,
typename=typename std::enable_if<std::is_convertible<U,T>::value>::type>
template <typename U, typename=std::enable_if_t<std::is_convertible<U,T>::value>>
SKVX_ALWAYS_INLINE
Vec(U x) : lo(x), hi(x) {}
@ -118,8 +97,7 @@ struct Vec<1,T> {
SKVX_ALWAYS_INLINE Vec() = default;
template <typename U,
typename=typename std::enable_if<std::is_convertible<U,T>::value>::type>
template <typename U, typename=std::enable_if_t<std::is_convertible<U,T>::value>>
SKVX_ALWAYS_INLINE
Vec(U x) : val(x) {}
@ -139,14 +117,14 @@ struct Vec<1,T> {
};
template <typename D, typename S>
static inline D unchecked_bit_pun(const S& s) {
SI D unchecked_bit_pun(const S& s) {
D d;
memcpy(&d, &s, sizeof(D));
return d;
}
template <typename D, typename S>
static inline D bit_pun(const S& s) {
SI D bit_pun(const S& s) {
static_assert(sizeof(D) == sizeof(S), "");
return unchecked_bit_pun<D>(s);
}
@ -189,34 +167,60 @@ SINT Vec<2*N,T> join(const Vec<N,T>& lo, const Vec<N,T>& hi) {
// For some reason some (new!) versions of GCC cannot seem to deduce N in the generic
// to_vec<N,T>() below for N=4 and T=float. This workaround seems to help...
static inline Vec<4,float> to_vec(VExt<4,float> v) { return bit_pun<Vec<4,float>>(v); }
SI Vec<4,float> to_vec(VExt<4,float> v) { return bit_pun<Vec<4,float>>(v); }
#endif
SINT VExt<N,T> to_vext(const Vec<N,T>& v) { return bit_pun<VExt<N,T>>(v); }
SINT Vec <N,T> to_vec(const VExt<N,T>& v) { return bit_pun<Vec <N,T>>(v); }
SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) + to_vext(y)); }
SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) - to_vext(y)); }
SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) * to_vext(y)); }
SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) / to_vext(y)); }
SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) {
return to_vec<N,T>(to_vext(x) + to_vext(y));
}
SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) {
return to_vec<N,T>(to_vext(x) - to_vext(y));
}
SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) {
return to_vec<N,T>(to_vext(x) * to_vext(y));
}
SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) {
return to_vec<N,T>(to_vext(x) / to_vext(y));
}
SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) ^ to_vext(y)); }
SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) & to_vext(y)); }
SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) | to_vext(y)); }
SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) {
return to_vec<N,T>(to_vext(x) ^ to_vext(y));
}
SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) {
return to_vec<N,T>(to_vext(x) & to_vext(y));
}
SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) {
return to_vec<N,T>(to_vext(x) | to_vext(y));
}
SINT Vec<N,T> operator!(const Vec<N,T>& x) { return to_vec<N,T>(!to_vext(x)); }
SINT Vec<N,T> operator-(const Vec<N,T>& x) { return to_vec<N,T>(-to_vext(x)); }
SINT Vec<N,T> operator~(const Vec<N,T>& x) { return to_vec<N,T>(~to_vext(x)); }
SINT Vec<N,T> operator<<(const Vec<N,T>& x, int bits) { return to_vec<N,T>(to_vext(x) << bits); }
SINT Vec<N,T> operator>>(const Vec<N,T>& x, int bits) { return to_vec<N,T>(to_vext(x) >> bits); }
SINT Vec<N,T> operator<<(const Vec<N,T>& x, int k) { return to_vec<N,T>(to_vext(x) << k); }
SINT Vec<N,T> operator>>(const Vec<N,T>& x, int k) { return to_vec<N,T>(to_vext(x) >> k); }
SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) == to_vext(y)); }
SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) != to_vext(y)); }
SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) <= to_vext(y)); }
SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) >= to_vext(y)); }
SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) < to_vext(y)); }
SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) > to_vext(y)); }
SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) {
return bit_pun<Vec<N,M<T>>>(to_vext(x) == to_vext(y));
}
SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) {
return bit_pun<Vec<N,M<T>>>(to_vext(x) != to_vext(y));
}
SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) {
return bit_pun<Vec<N,M<T>>>(to_vext(x) <= to_vext(y));
}
SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) {
return bit_pun<Vec<N,M<T>>>(to_vext(x) >= to_vext(y));
}
SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) {
return bit_pun<Vec<N,M<T>>>(to_vext(x) < to_vext(y));
}
SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) {
return bit_pun<Vec<N,M<T>>>(to_vext(x) > to_vext(y));
}
#else
@ -237,39 +241,77 @@ SINT Vec<2*N,T> join(const Vec<N,T>& lo, const Vec<N,T>& hi) {
SIT Vec<1,T> operator-(const Vec<1,T>& x) { return -x.val; }
SIT Vec<1,T> operator~(const Vec<1,T>& x) { return ~x.val; }
SIT Vec<1,T> operator<<(const Vec<1,T>& x, int bits) { return x.val << bits; }
SIT Vec<1,T> operator>>(const Vec<1,T>& x, int bits) { return x.val >> bits; }
SIT Vec<1,T> operator<<(const Vec<1,T>& x, int k) { return x.val << k; }
SIT Vec<1,T> operator>>(const Vec<1,T>& x, int k) { return x.val >> k; }
SIT Vec<1,M<T>> operator==(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val == y.val ? ~0 : 0; }
SIT Vec<1,M<T>> operator!=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val != y.val ? ~0 : 0; }
SIT Vec<1,M<T>> operator<=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val <= y.val ? ~0 : 0; }
SIT Vec<1,M<T>> operator>=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val >= y.val ? ~0 : 0; }
SIT Vec<1,M<T>> operator< (const Vec<1,T>& x, const Vec<1,T>& y) { return x.val < y.val ? ~0 : 0; }
SIT Vec<1,M<T>> operator> (const Vec<1,T>& x, const Vec<1,T>& y) { return x.val > y.val ? ~0 : 0; }
SIT Vec<1,M<T>> operator==(const Vec<1,T>& x, const Vec<1,T>& y) {
return x.val == y.val ? ~0 : 0;
}
SIT Vec<1,M<T>> operator!=(const Vec<1,T>& x, const Vec<1,T>& y) {
return x.val != y.val ? ~0 : 0;
}
SIT Vec<1,M<T>> operator<=(const Vec<1,T>& x, const Vec<1,T>& y) {
return x.val <= y.val ? ~0 : 0;
}
SIT Vec<1,M<T>> operator>=(const Vec<1,T>& x, const Vec<1,T>& y) {
return x.val >= y.val ? ~0 : 0;
}
SIT Vec<1,M<T>> operator< (const Vec<1,T>& x, const Vec<1,T>& y) {
return x.val < y.val ? ~0 : 0;
}
SIT Vec<1,M<T>> operator> (const Vec<1,T>& x, const Vec<1,T>& y) {
return x.val > y.val ? ~0 : 0;
}
// All default N != 1 implementations just recurse on lo and hi halves.
SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo + y.lo, x.hi + y.hi); }
SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo - y.lo, x.hi - y.hi); }
SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo * y.lo, x.hi * y.hi); }
SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo / y.lo, x.hi / y.hi); }
SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo + y.lo, x.hi + y.hi);
}
SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo - y.lo, x.hi - y.hi);
}
SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo * y.lo, x.hi * y.hi);
}
SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo / y.lo, x.hi / y.hi);
}
SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo ^ y.lo, x.hi ^ y.hi); }
SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo & y.lo, x.hi & y.hi); }
SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo | y.lo, x.hi | y.hi); }
SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo ^ y.lo, x.hi ^ y.hi);
}
SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo & y.lo, x.hi & y.hi);
}
SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo | y.lo, x.hi | y.hi);
}
SINT Vec<N,T> operator!(const Vec<N,T>& x) { return join(!x.lo, !x.hi); }
SINT Vec<N,T> operator-(const Vec<N,T>& x) { return join(-x.lo, -x.hi); }
SINT Vec<N,T> operator~(const Vec<N,T>& x) { return join(~x.lo, ~x.hi); }
SINT Vec<N,T> operator<<(const Vec<N,T>& x, int bits) { return join(x.lo << bits, x.hi << bits); }
SINT Vec<N,T> operator>>(const Vec<N,T>& x, int bits) { return join(x.lo >> bits, x.hi >> bits); }
SINT Vec<N,T> operator<<(const Vec<N,T>& x, int k) { return join(x.lo << k, x.hi << k); }
SINT Vec<N,T> operator>>(const Vec<N,T>& x, int k) { return join(x.lo >> k, x.hi >> k); }
SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo == y.lo, x.hi == y.hi); }
SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo != y.lo, x.hi != y.hi); }
SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo <= y.lo, x.hi <= y.hi); }
SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo >= y.lo, x.hi >= y.hi); }
SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo < y.lo, x.hi < y.hi); }
SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo > y.lo, x.hi > y.hi); }
SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo == y.lo, x.hi == y.hi);
}
SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo != y.lo, x.hi != y.hi);
}
SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo <= y.lo, x.hi <= y.hi);
}
SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo >= y.lo, x.hi >= y.hi);
}
SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo < y.lo, x.hi < y.hi);
}
SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) {
return join(x.lo > y.lo, x.hi > y.hi);
}
#endif
// Some operations we want are not expressible with Clang/GCC vector
@ -349,9 +391,15 @@ SINT bool all(const Vec<N,T>& x) { return all(x.lo) && all(x.hi); }
SINT T min(const Vec<N,T>& x) { return std::min(min(x.lo), min(x.hi)); }
SINT T max(const Vec<N,T>& x) { return std::max(max(x.lo), max(x.hi)); }
SINT Vec<N,T> min(const Vec<N,T>& x, const Vec<N,T>& y) { return join(min(x.lo, y.lo), min(x.hi, y.hi)); }
SINT Vec<N,T> max(const Vec<N,T>& x, const Vec<N,T>& y) { return join(max(x.lo, y.lo), max(x.hi, y.hi)); }
SINT Vec<N,T> pow(const Vec<N,T>& x, const Vec<N,T>& y) { return join(pow(x.lo, y.lo), pow(x.hi, y.hi)); }
SINT Vec<N,T> min(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(min(x.lo, y.lo), min(x.hi, y.hi));
}
SINT Vec<N,T> max(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(max(x.lo, y.lo), max(x.hi, y.hi));
}
SINT Vec<N,T> pow(const Vec<N,T>& x, const Vec<N,T>& y) {
return join(pow(x.lo, y.lo), pow(x.hi, y.hi));
}
SINT Vec<N,T> atan(const Vec<N,T>& x) { return join( atan(x.lo), atan(x.hi)); }
SINT Vec<N,T> ceil(const Vec<N,T>& x) { return join( ceil(x.lo), ceil(x.hi)); }
@ -430,10 +478,10 @@ SINT Vec<N,T>& operator>>=(Vec<N,T>& x, int bits) { return (x = x >> bits); }
// cast() Vec<N,S> to Vec<N,D>, as if applying a C-cast to each lane.
template <typename D, typename S>
static inline Vec<1,D> cast(const Vec<1,S>& src) { return (D)src.val; }
SI Vec<1,D> cast(const Vec<1,S>& src) { return (D)src.val; }
template <typename D, int N, typename S>
static inline Vec<N,D> cast(const Vec<N,S>& src) {
SI Vec<N,D> cast(const Vec<N,S>& src) {
#if !defined(SKNX_NO_SIMD) && defined(__clang__)
return to_vec(__builtin_convertvector(to_vext(src), VExt<N,D>));
#else
@ -449,7 +497,7 @@ static inline Vec<N,D> cast(const Vec<N,S>& src) {
// shuffle<3,3,3,3> (rgba) ~> {A,A,A,A}
// The only real restriction is that the output also be a legal N=power-of-two sknx::Vec.
template <int... Ix, int N, typename T>
static inline Vec<sizeof...(Ix),T> shuffle(const Vec<N,T>& x) {
SI Vec<sizeof...(Ix),T> shuffle(const Vec<N,T>& x) {
#if !defined(SKNX_NO_SIMD) && defined(__clang__)
return to_vec<sizeof...(Ix),T>(__builtin_shufflevector(to_vext(x), to_vext(x), Ix...));
#else
@ -457,22 +505,16 @@ static inline Vec<sizeof...(Ix),T> shuffle(const Vec<N,T>& x) {
#endif
}
// fma() delivers a fused mul-add, even if that's really expensive. Call it when you know it's not.
static inline Vec<1,float> fma(const Vec<1,float>& x,
const Vec<1,float>& y,
const Vec<1,float>& z) {
// fma() delivers a fused mul-add, even if that's really expensive.
SI Vec<1,float> fma(const Vec<1,float>& x, const Vec<1,float>& y, const Vec<1,float>& z) {
return std::fma(x.val, y.val, z.val);
}
template <int N>
static inline Vec<N,float> fma(const Vec<N,float>& x,
const Vec<N,float>& y,
const Vec<N,float>& z) {
SIN Vec<N,float> fma(const Vec<N,float>& x, const Vec<N,float>& y, const Vec<N,float>& z) {
return join(fma(x.lo, y.lo, z.lo),
fma(x.hi, y.hi, z.hi));
}
template <int N>
static inline Vec<N,float> fract(const Vec<N,float>& x) {
SIN Vec<N,float> fract(const Vec<N,float>& x) {
return x - floor(x);
}
@ -481,8 +523,7 @@ static inline Vec<N,float> fract(const Vec<N,float>& x) {
// Key constants to watch for:
// - a float is 32-bit, 1-8-23 sign-exponent-mantissa, with 127 exponent bias;
// - a half is 16-bit, 1-5-10 sign-exponent-mantissa, with 15 exponent bias.
template <int N>
static inline Vec<N,uint16_t> to_half_finite_ftz(const Vec<N,float>& x) {
SIN Vec<N,uint16_t> to_half_finite_ftz(const Vec<N,float>& x) {
Vec<N,uint32_t> sem = bit_pun<Vec<N,uint32_t>>(x),
s = sem & 0x8000'0000,
em = sem ^ s,
@ -490,8 +531,7 @@ static inline Vec<N,uint16_t> to_half_finite_ftz(const Vec<N,float>& x) {
return cast<uint16_t>(if_then_else(is_denorm, Vec<N,uint32_t>(0)
, (s>>16) + (em>>13) - ((127-15)<<10)));
}
template <int N>
static inline Vec<N,float> from_half_finite_ftz(const Vec<N,uint16_t>& x) {
SIN Vec<N,float> from_half_finite_ftz(const Vec<N,uint16_t>& x) {
Vec<N,uint32_t> wide = cast<uint32_t>(x),
s = wide & 0x8000,
em = wide ^ s;
@ -501,11 +541,10 @@ static inline Vec<N,float> from_half_finite_ftz(const Vec<N,uint16_t>& x) {
}
// Like if_then_else(), these N=1 base cases won't actually be used unless explicitly called.
static inline Vec<1,uint16_t> to_half(const Vec<1,float>& x) { return to_half_finite_ftz(x); }
static inline Vec<1,float> from_half(const Vec<1,uint16_t>& x) { return from_half_finite_ftz(x); }
SI Vec<1,uint16_t> to_half(const Vec<1,float>& x) { return to_half_finite_ftz(x); }
SI Vec<1,float> from_half(const Vec<1,uint16_t>& x) { return from_half_finite_ftz(x); }
template <int N>
static inline Vec<N,uint16_t> to_half(const Vec<N,float>& x) {
SIN Vec<N,uint16_t> to_half(const Vec<N,float>& x) {
#if defined(__F16C__)
if /*constexpr*/ (N == 8) {
return unchecked_bit_pun<Vec<N,uint16_t>>(_mm256_cvtps_ph(unchecked_bit_pun<__m256>(x),
@ -525,8 +564,7 @@ static inline Vec<N,uint16_t> to_half(const Vec<N,float>& x) {
return to_half_finite_ftz(x);
}
template <int N>
static inline Vec<N,float> from_half(const Vec<N,uint16_t>& x) {
SIN Vec<N,float> from_half(const Vec<N,uint16_t>& x) {
#if defined(__F16C__)
if /*constexpr*/ (N == 8) {
return unchecked_bit_pun<Vec<N,float>>(_mm256_cvtph_ps(unchecked_bit_pun<__m128i>(x)));
@ -546,15 +584,13 @@ static inline Vec<N,float> from_half(const Vec<N,uint16_t>& x) {
// div255(x) = (x + 127) / 255 is a bit-exact rounding divide-by-255, packing down to 8-bit.
template <int N>
static inline Vec<N,uint8_t> div255(const Vec<N,uint16_t>& x) {
SIN Vec<N,uint8_t> div255(const Vec<N,uint16_t>& x) {
return cast<uint8_t>( (x+127)/255 );
}
// approx_scale(x,y) approximates div255(cast<uint16_t>(x)*cast<uint16_t>(y)) within a bit,
// and is always perfect when x or y is 0 or 255.
template <int N>
static inline Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,uint8_t>& y) {
SIN Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,uint8_t>& y) {
// All of (x*y+x)/256, (x*y+y)/256, and (x*y+255)/256 meet the criteria above.
// We happen to have historically picked (x*y+x)/256.
auto X = cast<uint16_t>(x),
@ -564,34 +600,29 @@ static inline Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,u
#if !defined(SKNX_NO_SIMD) && defined(__ARM_NEON)
// With NEON we can do eight u8*u8 -> u16 in one instruction, vmull_u8 (read, mul-long).
static inline Vec<8,uint16_t> mull(const Vec<8,uint8_t>& x,
const Vec<8,uint8_t>& y) {
SI Vec<8,uint16_t> mull(const Vec<8,uint8_t>& x,
const Vec<8,uint8_t>& y) {
return to_vec<8,uint16_t>(vmull_u8(to_vext(x),
to_vext(y)));
}
template <int N>
static inline typename std::enable_if<(N < 8),
Vec<N,uint16_t>>::type mull(const Vec<N,uint8_t>& x,
const Vec<N,uint8_t>& y) {
SIN std::enable_if_t<(N < 8), Vec<N,uint16_t>> mull(const Vec<N,uint8_t>& x,
const Vec<N,uint8_t>& y) {
// N < 8 --> double up data until N == 8, returning the part we need.
return mull(join(x,x),
join(y,y)).lo;
}
template <int N>
static inline typename std::enable_if<(N > 8),
Vec<N,uint16_t>>::type mull(const Vec<N,uint8_t>& x,
const Vec<N,uint8_t>& y) {
SIN std::enable_if_t<(N > 8), Vec<N,uint16_t>> mull(const Vec<N,uint8_t>& x,
const Vec<N,uint8_t>& y) {
// N > 8 --> usual join(lo,hi) strategy to recurse down to N == 8.
return join(mull(x.lo, y.lo),
mull(x.hi, y.hi));
}
#else
// Nothing special when we don't have NEON... just cast up to 16-bit and multiply.
template <int N>
static inline Vec<N,uint16_t> mull(const Vec<N,uint8_t>& x,
const Vec<N,uint8_t>& y) {
SIN Vec<N,uint16_t> mull(const Vec<N,uint8_t>& x,
const Vec<N,uint8_t>& y) {
return cast<uint16_t>(x)
* cast<uint16_t>(y);
}
@ -602,68 +633,62 @@ static inline Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,u
// Platform-specific specializations and overloads can now drop in here.
#if defined(__AVX__)
static inline Vec<8,float> sqrt(const Vec<8,float>& x) {
SI Vec<8,float> sqrt(const Vec<8,float>& x) {
return bit_pun<Vec<8,float>>(_mm256_sqrt_ps(bit_pun<__m256>(x)));
}
static inline Vec<8,float> rsqrt(const Vec<8,float>& x) {
SI Vec<8,float> rsqrt(const Vec<8,float>& x) {
return bit_pun<Vec<8,float>>(_mm256_rsqrt_ps(bit_pun<__m256>(x)));
}
static inline Vec<8,float> rcp(const Vec<8,float>& x) {
SI Vec<8,float> rcp(const Vec<8,float>& x) {
return bit_pun<Vec<8,float>>(_mm256_rcp_ps(bit_pun<__m256>(x)));
}
static inline Vec<8,int> lrint(const Vec<8,float>& x) {
SI Vec<8,int> lrint(const Vec<8,float>& x) {
return bit_pun<Vec<8,int>>(_mm256_cvtps_epi32(bit_pun<__m256>(x)));
}
#endif
#if defined(__SSE__)
static inline Vec<4,float> sqrt(const Vec<4,float>& x) {
SI Vec<4,float> sqrt(const Vec<4,float>& x) {
return bit_pun<Vec<4,float>>(_mm_sqrt_ps(bit_pun<__m128>(x)));
}
static inline Vec<4,float> rsqrt(const Vec<4,float>& x) {
SI Vec<4,float> rsqrt(const Vec<4,float>& x) {
return bit_pun<Vec<4,float>>(_mm_rsqrt_ps(bit_pun<__m128>(x)));
}
static inline Vec<4,float> rcp(const Vec<4,float>& x) {
SI Vec<4,float> rcp(const Vec<4,float>& x) {
return bit_pun<Vec<4,float>>(_mm_rcp_ps(bit_pun<__m128>(x)));
}
static inline Vec<4,int> lrint(const Vec<4,float>& x) {
SI Vec<4,int> lrint(const Vec<4,float>& x) {
return bit_pun<Vec<4,int>>(_mm_cvtps_epi32(bit_pun<__m128>(x)));
}
static inline Vec<2,float> sqrt(const Vec<2,float>& x) {
SI Vec<2,float> sqrt(const Vec<2,float>& x) {
return shuffle<0,1>( sqrt(shuffle<0,1,0,1>(x)));
}
static inline Vec<2,float> rsqrt(const Vec<2,float>& x) {
SI Vec<2,float> rsqrt(const Vec<2,float>& x) {
return shuffle<0,1>(rsqrt(shuffle<0,1,0,1>(x)));
}
static inline Vec<2,float> rcp(const Vec<2,float>& x) {
SI Vec<2,float> rcp(const Vec<2,float>& x) {
return shuffle<0,1>( rcp(shuffle<0,1,0,1>(x)));
}
static inline Vec<2,int> lrint(const Vec<2,float>& x) {
SI Vec<2,int> lrint(const Vec<2,float>& x) {
return shuffle<0,1>(lrint(shuffle<0,1,0,1>(x)));
}
#endif
#if defined(__AVX2__)
static inline Vec<4,float> fma(const Vec<4,float>& x,
const Vec<4,float>& y,
const Vec<4,float>& z) {
SI Vec<4,float> fma(const Vec<4,float>& x, const Vec<4,float>& y, const Vec<4,float>& z) {
return bit_pun<Vec<4,float>>(_mm_fmadd_ps(bit_pun<__m128>(x),
bit_pun<__m128>(y),
bit_pun<__m128>(z)));
}
static inline Vec<8,float> fma(const Vec<8,float>& x,
const Vec<8,float>& y,
const Vec<8,float>& z) {
SI Vec<8,float> fma(const Vec<8,float>& x, const Vec<8,float>& y, const Vec<8,float>& z) {
return bit_pun<Vec<8,float>>(_mm256_fmadd_ps(bit_pun<__m256>(x),
bit_pun<__m256>(y),
bit_pun<__m256>(z)));
}
#elif defined(__aarch64__)
static inline Vec<4,float> fma(const Vec<4,float>& x,
const Vec<4,float>& y,
const Vec<4,float>& z) {
SI Vec<4,float> fma(const Vec<4,float>& x, const Vec<4,float>& y, const Vec<4,float>& z) {
// These instructions tend to work like z += xy, so the order here is z,x,y.
return bit_pun<Vec<4,float>>(vfmaq_f32(bit_pun<float32x4_t>(z),
bit_pun<float32x4_t>(x),
@ -674,72 +699,56 @@ static inline Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,u
// WASM SIMD compatible operations which are not automatically compiled to SIMD commands
// by emscripten:
#if defined __wasm_simd128__
static inline Vec<4,float> min(const Vec<4,float>& x, const Vec<4,float>& y) {
SI Vec<4, float> rcp (const Vec<4, float>& x) { return 1.0f / x; }
SI Vec<2,double> rcp (const Vec<2,double>& x) { return 1.0f / x; }
SI Vec<4, float> rsqrt(const Vec<4, float>& x) { return 1.0f / sqrt(x); }
SI Vec<2,double> rsqrt(const Vec<2,double>& x) { return 1.0f / sqrt(x); }
SI Vec<4,float> min(const Vec<4,float>& x, const Vec<4,float>& y) {
return to_vec<4,float>(wasm_f32x4_min(to_vext(x), to_vext(y)));
}
static inline Vec<4,float> max(const Vec<4,float>& x, const Vec<4,float>& y) {
SI Vec<4,float> max(const Vec<4,float>& x, const Vec<4,float>& y) {
return to_vec<4,float>(wasm_f32x4_max(to_vext(x), to_vext(y)));
}
static inline Vec<4,float> sqrt(const Vec<4,float>& x) {
SI Vec<4,float> sqrt(const Vec<4,float>& x) {
return to_vec<4,float>(wasm_f32x4_sqrt(to_vext(x)));
}
static inline Vec<4,float> abs(const Vec<4,float>& x) {
SI Vec<4,float> abs(const Vec<4,float>& x) {
return to_vec<4,float>(wasm_f32x4_abs(to_vext(x)));
}
static inline Vec<4,float> rcp(const Vec<4,float>& x) {
return 1.0f / x;
}
static inline Vec<4,float> rsqrt(const Vec<4,float>& x) {
return 1.0f / sqrt(x);
}
static inline Vec<2,double> min(const Vec<2,double>& x, const Vec<2,double>& y) {
SI Vec<2,double> min(const Vec<2,double>& x, const Vec<2,double>& y) {
return to_vec<2,double>(wasm_f64x2_min(to_vext(x), to_vext(y)));
}
static inline Vec<2,double> max(const Vec<2,double>& x, const Vec<2,double>& y) {
SI Vec<2,double> max(const Vec<2,double>& x, const Vec<2,double>& y) {
return to_vec<2,double>(wasm_f64x2_max(to_vext(x), to_vext(y)));
}
static inline Vec<2,double> sqrt(const Vec<2,double>& x) {
SI Vec<2,double> sqrt(const Vec<2,double>& x) {
return to_vec<2,double>(wasm_f64x2_sqrt(to_vext(x)));
}
static inline Vec<2,double> abs(const Vec<2,double>& x) {
SI Vec<2,double> abs(const Vec<2,double>& x) {
return to_vec<2,double>(wasm_f64x2_abs(to_vext(x)));
}
static inline Vec<2,double> rcp(const Vec<2,double>& x) {
return 1.0f / x;
}
static inline Vec<2,double> rsqrt(const Vec<2,double>& x) {
return 1.0f / sqrt(x);
}
static inline bool any(const Vec<4,int32_t>& x) {
return wasm_i32x4_any_true(to_vext(x));
}
static inline bool all(const Vec<4,int32_t>& x) {
return wasm_i32x4_all_true(to_vext(x));
}
static inline Vec<4,int32_t> min(const Vec<4,int32_t>& x, const Vec<4,int32_t>& y) {
SI bool any(const Vec<4, int32_t>& x) { return wasm_i32x4_any_true(to_vext(x)); }
SI bool any(const Vec<4,uint32_t>& x) { return wasm_i32x4_any_true(to_vext(x)); }
SI bool all(const Vec<4, int32_t>& x) { return wasm_i32x4_all_true(to_vext(x)); }
SI bool all(const Vec<4,uint32_t>& x) { return wasm_i32x4_all_true(to_vext(x)); }
SI Vec<4,int32_t> min(const Vec<4,int32_t>& x, const Vec<4,int32_t>& y) {
return to_vec<4,int32_t>(wasm_i32x4_min(to_vext(x), to_vext(y)));
}
static inline Vec<4,int32_t> max(const Vec<4,int32_t>& x, const Vec<4,int32_t>& y) {
SI Vec<4,int32_t> max(const Vec<4,int32_t>& x, const Vec<4,int32_t>& y) {
return to_vec<4,int32_t>(wasm_i32x4_max(to_vext(x), to_vext(y)));
}
static inline Vec<4,int32_t> abs(const Vec<4,int32_t>& x) {
SI Vec<4,int32_t> abs(const Vec<4,int32_t>& x) {
return to_vec<4,int32_t>(wasm_i32x4_abs(to_vext(x)));
}
static inline bool any(const Vec<4,uint32_t>& x) {
return wasm_i32x4_any_true(to_vext(x));
}
static inline bool all(const Vec<4,uint32_t>& x) {
return wasm_i32x4_all_true(to_vext(x));
}
static inline Vec<4,uint32_t> min(const Vec<4,uint32_t>& x,
const Vec<4,uint32_t>& y) {
SI Vec<4,uint32_t> min(const Vec<4,uint32_t>& x, const Vec<4,uint32_t>& y) {
return to_vec<4,uint32_t>(wasm_u32x4_min(to_vext(x), to_vext(y)));
}
static inline Vec<4,uint32_t> max(const Vec<4,uint32_t>& x,
const Vec<4,uint32_t>& y) {
SI Vec<4,uint32_t> max(const Vec<4,uint32_t>& x, const Vec<4,uint32_t>& y) {
return to_vec<4,uint32_t>(wasm_u32x4_max(to_vext(x), to_vext(y)));
}
#endif
@ -751,6 +760,6 @@ static inline Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,u
#undef SINTU
#undef SINT
#undef SIT
#undef SKVX_ALIGNMENT
#undef SI
#endif//SKVX_DEFINED