/* * Copyright 2019 Google Inc. * * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. */ #ifndef SKVX_DEFINED #define SKVX_DEFINED // skvx::Vec are SIMD vectors of N T's, a v1.5 successor to SkNx. // // This time we're leaning a bit less on platform-specific intrinsics and a bit // more on Clang/GCC vector extensions, but still keeping the option open to // drop in platform-specific intrinsics, actually more easily than before. // // We've also fixed a few of the caveats that used to make SkNx awkward to work // with across translation units. skvx::Vec always has N*sizeof(T) size // and alignment[1][2] and is safe to use across translation units freely. // // [1] Ideally we'd only align to T, but that tanks ARMv7 NEON codegen. // [2] Some compilers barf if we try to use N*sizeof(T), so instead we leave them at T. // Please try to keep this file independent of Skia headers. #include // std::min, std::max #include // std::ceil, std::floor, std::trunc, std::round, std::sqrt, etc. #include // intXX_t #include // memcpy() #include // std::initializer_list #if defined(__SSE__) #include #elif defined(__ARM_NEON) #include #endif #if !defined(__clang__) && defined(__GNUC__) && defined(__mips64) // GCC 7 hits an internal compiler error when targeting MIPS64. #define SKVX_ALIGNMENT #elif !defined(__clang__) && defined(_MSC_VER) && defined(_M_IX86) // Our SkVx unit tests fail when built by MSVC for 32-bit x86. #define SKVX_ALIGNMENT #else #define SKVX_ALIGNMENT alignas(N * sizeof(T)) #endif #if defined(__GNUC__) && !defined(__clang__) && defined(__SSE__) // GCC warns about ABI changes when returning >= 32 byte vectors when -mavx is not enabled. // This only happens for types like VExt whose ABI we don't care about, not for Vec itself. #pragma GCC diagnostic ignored "-Wpsabi" #endif // To avoid ODR violations, all methods must be force-inlined, // and all standalone functions must be static, perhaps using these helpers. #if defined(_MSC_VER) #define SKVX_ALWAYS_INLINE __forceinline #else #define SKVX_ALWAYS_INLINE __attribute__((always_inline)) #endif #define SIT template < typename T> static inline #define SINT template static inline #define SINTU template ::value>::type> \ static inline namespace skvx { // All Vec have the same simple memory layout, the same as `T vec[N]`. template struct SKVX_ALIGNMENT Vec { static_assert((N & (N-1)) == 0, "N must be a power of 2."); static_assert(sizeof(T) >= alignof(T), "What kind of crazy T is this?"); Vec lo, hi; // Methods belong here in the class declaration of Vec only if: // - they must be here, like constructors or operator[]; // - they'll definitely never want a specialized implementation. // Other operations on Vec should be defined outside the type. SKVX_ALWAYS_INLINE Vec() = default; template ::value>::type> SKVX_ALWAYS_INLINE Vec(U x) : lo(x), hi(x) {} SKVX_ALWAYS_INLINE Vec(std::initializer_list xs) { T vals[N] = {0}; memcpy(vals, xs.begin(), std::min(xs.size(), (size_t)N)*sizeof(T)); lo = Vec::Load(vals + 0); hi = Vec::Load(vals + N/2); } SKVX_ALWAYS_INLINE T operator[](int i) const { return i < N/2 ? lo[i] : hi[i-N/2]; } SKVX_ALWAYS_INLINE T& operator[](int i) { return i < N/2 ? lo[i] : hi[i-N/2]; } SKVX_ALWAYS_INLINE static Vec Load(const void* ptr) { Vec v; memcpy(&v, ptr, sizeof(Vec)); return v; } SKVX_ALWAYS_INLINE void store(void* ptr) const { memcpy(ptr, this, sizeof(Vec)); } }; template struct Vec<1,T> { T val; SKVX_ALWAYS_INLINE Vec() = default; template ::value>::type> SKVX_ALWAYS_INLINE Vec(U x) : val(x) {} SKVX_ALWAYS_INLINE Vec(std::initializer_list xs) : val(xs.size() ? *xs.begin() : 0) {} SKVX_ALWAYS_INLINE T operator[](int) const { return val; } SKVX_ALWAYS_INLINE T& operator[](int) { return val; } SKVX_ALWAYS_INLINE static Vec Load(const void* ptr) { Vec v; memcpy(&v, ptr, sizeof(Vec)); return v; } SKVX_ALWAYS_INLINE void store(void* ptr) const { memcpy(ptr, this, sizeof(Vec)); } }; template static inline D bit_pun(const S& s) { static_assert(sizeof(D) == sizeof(S), ""); D d; memcpy(&d, &s, sizeof(D)); return d; } // Translate from a value type T to its corresponding Mask, the result of a comparison. template struct Mask { using type = T; }; template <> struct Mask { using type = int32_t; }; template <> struct Mask { using type = int64_t; }; template using M = typename Mask::type; // Join two Vec into one Vec<2N,T>. SINT Vec<2*N,T> join(const Vec& lo, const Vec& hi) { Vec<2*N,T> v; v.lo = lo; v.hi = hi; return v; } // We have two default strategies for implementing most operations: // 1) lean on Clang/GCC vector extensions when available; // 2) recurse to scalar portable implementations when not. // At the end we can drop in platform-specific implementations that override either default. #if !defined(SKNX_NO_SIMD) && (defined(__clang__) || defined(__GNUC__)) // VExt types have the same size as Vec and support most operations directly. // N.B. VExt alignment is N*alignof(T), stricter than Vec's alignof(T). #if defined(__clang__) template using VExt = T __attribute__((ext_vector_type(N))); #elif defined(__GNUC__) template struct VExtHelper { typedef T __attribute__((vector_size(N*sizeof(T)))) type; }; template using VExt = typename VExtHelper::type; // For some reason some (new!) versions of GCC cannot seem to deduce N in the generic // to_vec() below for N=4 and T=float. This workaround seems to help... static inline Vec<4,float> to_vec(VExt<4,float> v) { return bit_pun>(v); } #endif SINT VExt to_vext(const Vec& v) { return bit_pun>(v); } SINT Vec to_vec(const VExt& v) { return bit_pun>(v); } SINT Vec operator+(const Vec& x, const Vec& y) { return to_vec(to_vext(x) + to_vext(y)); } SINT Vec operator-(const Vec& x, const Vec& y) { return to_vec(to_vext(x) - to_vext(y)); } SINT Vec operator*(const Vec& x, const Vec& y) { return to_vec(to_vext(x) * to_vext(y)); } SINT Vec operator/(const Vec& x, const Vec& y) { return to_vec(to_vext(x) / to_vext(y)); } SINT Vec operator^(const Vec& x, const Vec& y) { return to_vec(to_vext(x) ^ to_vext(y)); } SINT Vec operator&(const Vec& x, const Vec& y) { return to_vec(to_vext(x) & to_vext(y)); } SINT Vec operator|(const Vec& x, const Vec& y) { return to_vec(to_vext(x) | to_vext(y)); } SINT Vec operator!(const Vec& x) { return to_vec(!to_vext(x)); } SINT Vec operator-(const Vec& x) { return to_vec(-to_vext(x)); } SINT Vec operator~(const Vec& x) { return to_vec(~to_vext(x)); } SINT Vec operator<<(const Vec& x, int bits) { return to_vec(to_vext(x) << bits); } SINT Vec operator>>(const Vec& x, int bits) { return to_vec(to_vext(x) >> bits); } SINT Vec> operator==(const Vec& x, const Vec& y) { return bit_pun>>(to_vext(x) == to_vext(y)); } SINT Vec> operator!=(const Vec& x, const Vec& y) { return bit_pun>>(to_vext(x) != to_vext(y)); } SINT Vec> operator<=(const Vec& x, const Vec& y) { return bit_pun>>(to_vext(x) <= to_vext(y)); } SINT Vec> operator>=(const Vec& x, const Vec& y) { return bit_pun>>(to_vext(x) >= to_vext(y)); } SINT Vec> operator< (const Vec& x, const Vec& y) { return bit_pun>>(to_vext(x) < to_vext(y)); } SINT Vec> operator> (const Vec& x, const Vec& y) { return bit_pun>>(to_vext(x) > to_vext(y)); } #else // Either SKNX_NO_SIMD is defined, or Clang/GCC vector extensions are not available. // We'll implement things portably, in a way that should be easily autovectorizable. // N == 1 scalar implementations. SIT Vec<1,T> operator+(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val + y.val; } SIT Vec<1,T> operator-(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val - y.val; } SIT Vec<1,T> operator*(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val * y.val; } SIT Vec<1,T> operator/(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val / y.val; } SIT Vec<1,T> operator^(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val ^ y.val; } SIT Vec<1,T> operator&(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val & y.val; } SIT Vec<1,T> operator|(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val | y.val; } SIT Vec<1,T> operator!(const Vec<1,T>& x) { return !x.val; } SIT Vec<1,T> operator-(const Vec<1,T>& x) { return -x.val; } SIT Vec<1,T> operator~(const Vec<1,T>& x) { return ~x.val; } SIT Vec<1,T> operator<<(const Vec<1,T>& x, int bits) { return x.val << bits; } SIT Vec<1,T> operator>>(const Vec<1,T>& x, int bits) { return x.val >> bits; } SIT Vec<1,M> operator==(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val == y.val ? ~0 : 0; } SIT Vec<1,M> operator!=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val != y.val ? ~0 : 0; } SIT Vec<1,M> operator<=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val <= y.val ? ~0 : 0; } SIT Vec<1,M> operator>=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val >= y.val ? ~0 : 0; } SIT Vec<1,M> operator< (const Vec<1,T>& x, const Vec<1,T>& y) { return x.val < y.val ? ~0 : 0; } SIT Vec<1,M> operator> (const Vec<1,T>& x, const Vec<1,T>& y) { return x.val > y.val ? ~0 : 0; } // All default N != 1 implementations just recurse on lo and hi halves. SINT Vec operator+(const Vec& x, const Vec& y) { return join(x.lo + y.lo, x.hi + y.hi); } SINT Vec operator-(const Vec& x, const Vec& y) { return join(x.lo - y.lo, x.hi - y.hi); } SINT Vec operator*(const Vec& x, const Vec& y) { return join(x.lo * y.lo, x.hi * y.hi); } SINT Vec operator/(const Vec& x, const Vec& y) { return join(x.lo / y.lo, x.hi / y.hi); } SINT Vec operator^(const Vec& x, const Vec& y) { return join(x.lo ^ y.lo, x.hi ^ y.hi); } SINT Vec operator&(const Vec& x, const Vec& y) { return join(x.lo & y.lo, x.hi & y.hi); } SINT Vec operator|(const Vec& x, const Vec& y) { return join(x.lo | y.lo, x.hi | y.hi); } SINT Vec operator!(const Vec& x) { return join(!x.lo, !x.hi); } SINT Vec operator-(const Vec& x) { return join(-x.lo, -x.hi); } SINT Vec operator~(const Vec& x) { return join(~x.lo, ~x.hi); } SINT Vec operator<<(const Vec& x, int bits) { return join(x.lo << bits, x.hi << bits); } SINT Vec operator>>(const Vec& x, int bits) { return join(x.lo >> bits, x.hi >> bits); } SINT Vec> operator==(const Vec& x, const Vec& y) { return join(x.lo == y.lo, x.hi == y.hi); } SINT Vec> operator!=(const Vec& x, const Vec& y) { return join(x.lo != y.lo, x.hi != y.hi); } SINT Vec> operator<=(const Vec& x, const Vec& y) { return join(x.lo <= y.lo, x.hi <= y.hi); } SINT Vec> operator>=(const Vec& x, const Vec& y) { return join(x.lo >= y.lo, x.hi >= y.hi); } SINT Vec> operator< (const Vec& x, const Vec& y) { return join(x.lo < y.lo, x.hi < y.hi); } SINT Vec> operator> (const Vec& x, const Vec& y) { return join(x.lo > y.lo, x.hi > y.hi); } #endif // Some operations we want are not expressible with Clang/GCC vector // extensions, so we implement them using the recursive approach. // N == 1 scalar implementations. SIT Vec<1,T> if_then_else(const Vec<1,M>& cond, const Vec<1,T>& t, const Vec<1,T>& e) { auto t_bits = bit_pun>(t), e_bits = bit_pun>(e); return bit_pun( (cond.val & t_bits) | (~cond.val & e_bits) ); } SIT bool any(const Vec<1,T>& x) { return x.val != 0; } SIT bool all(const Vec<1,T>& x) { return x.val != 0; } SIT T min(const Vec<1,T>& x) { return x.val; } SIT T max(const Vec<1,T>& x) { return x.val; } SIT Vec<1,T> min(const Vec<1,T>& x, const Vec<1,T>& y) { return std::min(x.val, y.val); } SIT Vec<1,T> max(const Vec<1,T>& x, const Vec<1,T>& y) { return std::max(x.val, y.val); } SIT Vec<1,T> ceil(const Vec<1,T>& x) { return std:: ceil(x.val); } SIT Vec<1,T> floor(const Vec<1,T>& x) { return std::floor(x.val); } SIT Vec<1,T> trunc(const Vec<1,T>& x) { return std::trunc(x.val); } SIT Vec<1,T> round(const Vec<1,T>& x) { return std::round(x.val); } SIT Vec<1,T> sqrt(const Vec<1,T>& x) { return std:: sqrt(x.val); } SIT Vec<1,T> abs(const Vec<1,T>& x) { return std:: abs(x.val); } SIT Vec<1,T> rcp(const Vec<1,T>& x) { return 1 / x.val; } SIT Vec<1,T> rsqrt(const Vec<1,T>& x) { return rcp(sqrt(x)); } SIT Vec<1,T> mad(const Vec<1,T>& f, const Vec<1,T>& m, const Vec<1,T>& a) { return f*m+a; } // All default N != 1 implementations just recurse on lo and hi halves. SINT Vec if_then_else(const Vec>& cond, const Vec& t, const Vec& e) { return join(if_then_else(cond.lo, t.lo, e.lo), if_then_else(cond.hi, t.hi, e.hi)); } SINT bool any(const Vec& x) { return any(x.lo) || any(x.hi); } SINT bool all(const Vec& x) { return all(x.lo) && all(x.hi); } SINT T min(const Vec& x) { return std::min(min(x.lo), min(x.hi)); } SINT T max(const Vec& x) { return std::max(max(x.lo), max(x.hi)); } SINT Vec min(const Vec& x, const Vec& y) { return join(min(x.lo, y.lo), min(x.hi, y.hi)); } SINT Vec max(const Vec& x, const Vec& y) { return join(max(x.lo, y.lo), max(x.hi, y.hi)); } SINT Vec ceil(const Vec& x) { return join( ceil(x.lo), ceil(x.hi)); } SINT Vec floor(const Vec& x) { return join(floor(x.lo), floor(x.hi)); } SINT Vec trunc(const Vec& x) { return join(trunc(x.lo), trunc(x.hi)); } SINT Vec round(const Vec& x) { return join(round(x.lo), round(x.hi)); } SINT Vec sqrt(const Vec& x) { return join( sqrt(x.lo), sqrt(x.hi)); } SINT Vec abs(const Vec& x) { return join( abs(x.lo), abs(x.hi)); } SINT Vec rcp(const Vec& x) { return join( rcp(x.lo), rcp(x.hi)); } SINT Vec rsqrt(const Vec& x) { return join(rsqrt(x.lo), rsqrt(x.hi)); } SINT Vec mad(const Vec& f, const Vec& m, const Vec& a) { return join(mad(f.lo, m.lo, a.lo), mad(f.hi, m.hi, a.hi)); } // Scalar/vector operations just splat the scalar to a vector... SINTU Vec operator+ (U x, const Vec& y) { return Vec(x) + y; } SINTU Vec operator- (U x, const Vec& y) { return Vec(x) - y; } SINTU Vec operator* (U x, const Vec& y) { return Vec(x) * y; } SINTU Vec operator/ (U x, const Vec& y) { return Vec(x) / y; } SINTU Vec operator^ (U x, const Vec& y) { return Vec(x) ^ y; } SINTU Vec operator& (U x, const Vec& y) { return Vec(x) & y; } SINTU Vec operator| (U x, const Vec& y) { return Vec(x) | y; } SINTU Vec> operator==(U x, const Vec& y) { return Vec(x) == y; } SINTU Vec> operator!=(U x, const Vec& y) { return Vec(x) != y; } SINTU Vec> operator<=(U x, const Vec& y) { return Vec(x) <= y; } SINTU Vec> operator>=(U x, const Vec& y) { return Vec(x) >= y; } SINTU Vec> operator< (U x, const Vec& y) { return Vec(x) < y; } SINTU Vec> operator> (U x, const Vec& y) { return Vec(x) > y; } SINTU Vec min(U x, const Vec& y) { return min(Vec(x), y); } SINTU Vec max(U x, const Vec& y) { return max(Vec(x), y); } // ... and same deal for vector/scalar operations. SINTU Vec operator+ (const Vec& x, U y) { return x + Vec(y); } SINTU Vec operator- (const Vec& x, U y) { return x - Vec(y); } SINTU Vec operator* (const Vec& x, U y) { return x * Vec(y); } SINTU Vec operator/ (const Vec& x, U y) { return x / Vec(y); } SINTU Vec operator^ (const Vec& x, U y) { return x ^ Vec(y); } SINTU Vec operator& (const Vec& x, U y) { return x & Vec(y); } SINTU Vec operator| (const Vec& x, U y) { return x | Vec(y); } SINTU Vec> operator==(const Vec& x, U y) { return x == Vec(y); } SINTU Vec> operator!=(const Vec& x, U y) { return x != Vec(y); } SINTU Vec> operator<=(const Vec& x, U y) { return x <= Vec(y); } SINTU Vec> operator>=(const Vec& x, U y) { return x >= Vec(y); } SINTU Vec> operator< (const Vec& x, U y) { return x < Vec(y); } SINTU Vec> operator> (const Vec& x, U y) { return x > Vec(y); } SINTU Vec min(const Vec& x, U y) { return min(x, Vec(y)); } SINTU Vec max(const Vec& x, U y) { return max(x, Vec(y)); } // All vector/scalar combinations for mad() with at least one vector. SINTU Vec mad(U f, const Vec& m, const Vec& a) { return Vec(f)*m + a; } SINTU Vec mad(const Vec& f, U m, const Vec& a) { return f*Vec(m) + a; } SINTU Vec mad(const Vec& f, const Vec& m, U a) { return f*m + Vec(a); } SINTU Vec mad(const Vec& f, U m, U a) { return f*Vec(m) + Vec(a); } SINTU Vec mad(U f, const Vec& m, U a) { return Vec(f)*m + Vec(a); } SINTU Vec mad(U f, U m, const Vec& a) { return Vec(f)*Vec(m) + a; } // The various op= operators, for vectors... SINT Vec& operator+=(Vec& x, const Vec& y) { return (x = x + y); } SINT Vec& operator-=(Vec& x, const Vec& y) { return (x = x - y); } SINT Vec& operator*=(Vec& x, const Vec& y) { return (x = x * y); } SINT Vec& operator/=(Vec& x, const Vec& y) { return (x = x / y); } SINT Vec& operator^=(Vec& x, const Vec& y) { return (x = x ^ y); } SINT Vec& operator&=(Vec& x, const Vec& y) { return (x = x & y); } SINT Vec& operator|=(Vec& x, const Vec& y) { return (x = x | y); } // ... for scalars... SINTU Vec& operator+=(Vec& x, U y) { return (x = x + Vec(y)); } SINTU Vec& operator-=(Vec& x, U y) { return (x = x - Vec(y)); } SINTU Vec& operator*=(Vec& x, U y) { return (x = x * Vec(y)); } SINTU Vec& operator/=(Vec& x, U y) { return (x = x / Vec(y)); } SINTU Vec& operator^=(Vec& x, U y) { return (x = x ^ Vec(y)); } SINTU Vec& operator&=(Vec& x, U y) { return (x = x & Vec(y)); } SINTU Vec& operator|=(Vec& x, U y) { return (x = x | Vec(y)); } // ... and for shifts. SINT Vec& operator<<=(Vec& x, int bits) { return (x = x << bits); } SINT Vec& operator>>=(Vec& x, int bits) { return (x = x >> bits); } // cast() Vec to Vec, as if applying a C-cast to each lane. template static inline Vec<1,D> cast(const Vec<1,S>& src) { return (D)src.val; } template static inline Vec cast(const Vec& src) { #if !defined(SKNX_NO_SIMD) && defined(__clang__) return to_vec(__builtin_convertvector(to_vext(src), VExt)); #else return join(cast(src.lo), cast(src.hi)); #endif } // Shuffle values from a vector pretty arbitrarily: // skvx::Vec<4,float> rgba = {R,G,B,A}; // shuffle<2,1,0,3> (rgba) ~> {B,G,R,A} // shuffle<2,1> (rgba) ~> {B,G} // shuffle<2,1,2,1,2,1,2,1>(rgba) ~> {B,G,B,G,B,G,B,G} // shuffle<3,3,3,3> (rgba) ~> {A,A,A,A} // The only real restriction is that the output also be a legal N=power-of-two sknx::Vec. template static inline Vec shuffle(const Vec& x) { #if !defined(SKNX_NO_SIMD) && defined(__clang__) return to_vec(__builtin_shufflevector(to_vext(x), to_vext(x), Ix...)); #else return { x[Ix]... }; #endif } // div255(x) = (x + 127) / 255 is a bit-exact rounding divide-by-255, packing down to 8-bit. template static inline Vec div255(const Vec& x) { return cast( (x+127)/255 ); } // approx_scale(x,y) approximates div255(cast(x)*cast(y)) within a bit, // and is always perfect when x or y is 0 or 255. template static inline Vec approx_scale(const Vec& x, const Vec& y) { // All of (x*y+x)/256, (x*y+y)/256, and (x*y+255)/256 meet the criteria above. // We happen to have historically picked (x*y+x)/256. auto X = cast(x), Y = cast(y); return cast( (X*Y+X)/256 ); } #if !defined(SKNX_NO_SIMD) && defined(__ARM_NEON) // With NEON we can do eight u8*u8 -> u16 in one instruction, vmull_u8 (read, mul-long). static inline Vec<8,uint16_t> mull(const Vec<8,uint8_t>& x, const Vec<8,uint8_t>& y) { return to_vec<8,uint16_t>(vmull_u8(to_vext(x), to_vext(y))); } template static inline typename std::enable_if<(N < 8), Vec>::type mull(const Vec& x, const Vec& y) { // N < 8 --> double up data until N == 8, returning the part we need. return mull(join(x,x), join(y,y)).lo; } template static inline typename std::enable_if<(N > 8), Vec>::type mull(const Vec& x, const Vec& y) { // N > 8 --> usual join(lo,hi) strategy to recurse down to N == 8. return join(mull(x.lo, y.lo), mull(x.hi, y.hi)); } #else // Nothing special when we don't have NEON... just cast up to 16-bit and multiply. template