2019-02-05 18:42:46 +00:00
|
|
|
/*
|
|
|
|
* Copyright 2019 Google Inc.
|
|
|
|
*
|
|
|
|
* Use of this source code is governed by a BSD-style license that can be
|
|
|
|
* found in the LICENSE file.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef SKVX_DEFINED
|
|
|
|
#define SKVX_DEFINED
|
|
|
|
|
|
|
|
// skvx::Vec<N,T> are SIMD vectors of N T's, a v1.5 successor to SkNx<N,T>.
|
|
|
|
//
|
|
|
|
// This time we're leaning a bit less on platform-specific intrinsics and a bit
|
|
|
|
// more on Clang/GCC vector extensions, but still keeping the option open to
|
|
|
|
// drop in platform-specific intrinsics, actually more easily than before.
|
|
|
|
//
|
|
|
|
// We've also fixed a few of the caveats that used to make SkNx awkward to work
|
|
|
|
// with across translation units. skvx::Vec<N,T> always has N*sizeof(T) size
|
2020-09-02 14:00:57 +00:00
|
|
|
// and alignment and is safe to use across translation units freely.
|
2020-10-12 18:13:28 +00:00
|
|
|
// (Ideally we'd only align to T, but that tanks ARMv7 NEON codegen.)
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2019-06-07 15:57:58 +00:00
|
|
|
// Please try to keep this file independent of Skia headers.
|
2019-02-07 14:49:17 +00:00
|
|
|
#include <algorithm> // std::min, std::max
|
2021-10-11 19:37:05 +00:00
|
|
|
#include <cassert> // assert()
|
2020-10-12 18:13:28 +00:00
|
|
|
#include <cmath> // ceilf, floorf, truncf, roundf, sqrtf, etc.
|
2019-02-05 18:42:46 +00:00
|
|
|
#include <cstdint> // intXX_t
|
|
|
|
#include <cstring> // memcpy()
|
|
|
|
#include <initializer_list> // std::initializer_list
|
2020-10-12 18:13:28 +00:00
|
|
|
#include <utility> // std::index_sequence
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2022-01-18 20:22:45 +00:00
|
|
|
// Users may disable SIMD with SKNX_NO_SIMD, which may be set via compiler flags.
|
|
|
|
// The gn build has no option which sets SKNX_NO_SIMD.
|
|
|
|
// Use SKVX_USE_SIMD internally to avoid confusing double negation.
|
|
|
|
// Do not use 'defined' in a macro expansion.
|
|
|
|
#if !defined(SKNX_NO_SIMD)
|
|
|
|
#define SKVX_USE_SIMD 1
|
|
|
|
#else
|
|
|
|
#define SKVX_USE_SIMD 0
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if SKVX_USE_SIMD
|
|
|
|
#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__)
|
|
|
|
#include <immintrin.h>
|
|
|
|
#elif defined(__ARM_NEON)
|
|
|
|
#include <arm_neon.h>
|
|
|
|
#elif defined(__wasm_simd128__)
|
|
|
|
#include <wasm_simd128.h>
|
|
|
|
#endif
|
2020-06-30 22:08:44 +00:00
|
|
|
#endif
|
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
// To avoid ODR violations, all methods must be force-inlined...
|
2019-12-17 17:40:14 +00:00
|
|
|
#if defined(_MSC_VER)
|
|
|
|
#define SKVX_ALWAYS_INLINE __forceinline
|
|
|
|
#else
|
|
|
|
#define SKVX_ALWAYS_INLINE __attribute__((always_inline))
|
|
|
|
#endif
|
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
// ... and all standalone functions must be static. Please use these helpers:
|
|
|
|
#define SI static inline
|
|
|
|
#define SIT template < typename T> SI
|
|
|
|
#define SIN template <int N > SI
|
|
|
|
#define SINT template <int N, typename T> SI
|
2019-12-17 17:40:14 +00:00
|
|
|
#define SINTU template <int N, typename T, typename U, \
|
2020-09-02 14:00:57 +00:00
|
|
|
typename=std::enable_if_t<std::is_convertible<U,T>::value>> SI
|
2019-02-27 16:24:55 +00:00
|
|
|
|
2019-02-05 18:42:46 +00:00
|
|
|
namespace skvx {
|
|
|
|
|
2021-10-08 21:23:22 +00:00
|
|
|
template <int N, typename T>
|
|
|
|
struct alignas(N*sizeof(T)) Vec;
|
|
|
|
|
2021-10-08 21:11:20 +00:00
|
|
|
template <int... Ix, int N, typename T>
|
|
|
|
SI Vec<sizeof...(Ix),T> shuffle(const Vec<N,T>&);
|
|
|
|
|
|
|
|
template <typename D, typename S>
|
|
|
|
SI D bit_pun(const S&);
|
|
|
|
|
2019-02-05 18:42:46 +00:00
|
|
|
// All Vec have the same simple memory layout, the same as `T vec[N]`.
|
|
|
|
template <int N, typename T>
|
2021-10-08 21:23:22 +00:00
|
|
|
struct alignas(N*sizeof(T)) VecStorage {
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage() = default;
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage(T s) : lo(s), hi(s) {}
|
2021-10-08 20:31:37 +00:00
|
|
|
|
|
|
|
Vec<N/2,T> lo, hi;
|
2021-10-08 21:23:22 +00:00
|
|
|
};
|
|
|
|
|
2021-10-08 21:11:20 +00:00
|
|
|
template <typename T>
|
|
|
|
struct VecStorage<4,T> {
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage() = default;
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage(T s) : lo(s), hi(s) {}
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage(T x, T y, T z, T w) : lo(x,y), hi(z, w) {}
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage(Vec<2,T> xy, T z, T w) : lo(xy), hi(z,w) {}
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage(T x, T y, Vec<2,T> zw) : lo(x,y), hi(zw) {}
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage(Vec<2,T> xy, Vec<2,T> zw) : lo(xy), hi(zw) {}
|
|
|
|
|
|
|
|
SKVX_ALWAYS_INLINE Vec<2,T>& xy() { return lo; }
|
|
|
|
SKVX_ALWAYS_INLINE Vec<2,T>& zw() { return hi; }
|
|
|
|
SKVX_ALWAYS_INLINE T& x() { return lo.lo.val; }
|
|
|
|
SKVX_ALWAYS_INLINE T& y() { return lo.hi.val; }
|
|
|
|
SKVX_ALWAYS_INLINE T& z() { return hi.lo.val; }
|
|
|
|
SKVX_ALWAYS_INLINE T& w() { return hi.hi.val; }
|
|
|
|
|
|
|
|
SKVX_ALWAYS_INLINE Vec<2,T> xy() const { return lo; }
|
|
|
|
SKVX_ALWAYS_INLINE Vec<2,T> zw() const { return hi; }
|
|
|
|
SKVX_ALWAYS_INLINE T x() const { return lo.lo.val; }
|
|
|
|
SKVX_ALWAYS_INLINE T y() const { return lo.hi.val; }
|
|
|
|
SKVX_ALWAYS_INLINE T z() const { return hi.lo.val; }
|
|
|
|
SKVX_ALWAYS_INLINE T w() const { return hi.hi.val; }
|
|
|
|
|
|
|
|
// Exchange-based swizzles. These should take 1 cycle on NEON and 3 (pipelined) cycles on SSE.
|
|
|
|
SKVX_ALWAYS_INLINE Vec<4,T> yxwz() const { return shuffle<1,0,3,2>(bit_pun<Vec<4,T>>(*this)); }
|
|
|
|
SKVX_ALWAYS_INLINE Vec<4,T> zwxy() const { return shuffle<2,3,0,1>(bit_pun<Vec<4,T>>(*this)); }
|
|
|
|
|
|
|
|
Vec<2,T> lo, hi;
|
|
|
|
};
|
|
|
|
|
|
|
|
template <typename T>
|
|
|
|
struct VecStorage<2,T> {
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage() = default;
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage(T s) : lo(s), hi(s) {}
|
|
|
|
SKVX_ALWAYS_INLINE VecStorage(T x, T y) : lo(x), hi(y) {}
|
|
|
|
|
|
|
|
SKVX_ALWAYS_INLINE T& x() { return lo.val; }
|
|
|
|
SKVX_ALWAYS_INLINE T& y() { return hi.val; }
|
|
|
|
|
|
|
|
SKVX_ALWAYS_INLINE T x() const { return lo.val; }
|
|
|
|
SKVX_ALWAYS_INLINE T y() const { return hi.val; }
|
|
|
|
|
|
|
|
// This exchange-based swizzle should take 1 cycle on NEON and 3 (pipelined) cycles on SSE.
|
|
|
|
SKVX_ALWAYS_INLINE Vec<2,T> yx() const { return shuffle<1,0>(bit_pun<Vec<2,T>>(*this)); }
|
|
|
|
|
|
|
|
SKVX_ALWAYS_INLINE Vec<4,T> xyxy() const {
|
|
|
|
return Vec<4,T>(bit_pun<Vec<2,T>>(*this), bit_pun<Vec<2,T>>(*this));
|
|
|
|
}
|
|
|
|
|
|
|
|
Vec<1,T> lo, hi;
|
|
|
|
};
|
|
|
|
|
2021-10-08 21:23:22 +00:00
|
|
|
template <int N, typename T>
|
|
|
|
struct alignas(N*sizeof(T)) Vec : public VecStorage<N,T> {
|
|
|
|
static_assert((N & (N-1)) == 0, "N must be a power of 2.");
|
|
|
|
static_assert(sizeof(T) >= alignof(T), "What kind of unusual T is this?");
|
2019-03-14 18:30:42 +00:00
|
|
|
|
2021-10-08 20:31:37 +00:00
|
|
|
// Methods belong here in the class declaration of Vec only if:
|
|
|
|
// - they must be here, like constructors or operator[];
|
2021-10-07 20:37:06 +00:00
|
|
|
// - they'll definitely never want a specialized implementation.
|
2021-10-08 20:31:37 +00:00
|
|
|
// Other operations on Vec should be defined outside the type.
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2021-10-08 20:31:37 +00:00
|
|
|
SKVX_ALWAYS_INLINE Vec() = default;
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2021-10-08 21:23:22 +00:00
|
|
|
using VecStorage<N,T>::VecStorage;
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2022-05-17 19:24:40 +00:00
|
|
|
// NOTE: Vec{x} produces x000..., whereas Vec(x) produces xxxx.... since this constructor fills
|
|
|
|
// unspecified lanes with 0s, whereas the single T constructor fills all lanes with the value.
|
2021-10-08 20:31:37 +00:00
|
|
|
SKVX_ALWAYS_INLINE Vec(std::initializer_list<T> xs) {
|
|
|
|
T vals[N] = {0};
|
|
|
|
memcpy(vals, xs.begin(), std::min(xs.size(), (size_t)N)*sizeof(T));
|
2019-02-06 16:56:58 +00:00
|
|
|
|
2021-10-08 21:23:22 +00:00
|
|
|
this->lo = Vec<N/2,T>::Load(vals + 0);
|
|
|
|
this->hi = Vec<N/2,T>::Load(vals + N/2);
|
2019-02-06 16:56:58 +00:00
|
|
|
}
|
2021-10-07 20:37:06 +00:00
|
|
|
|
2021-10-08 21:23:22 +00:00
|
|
|
SKVX_ALWAYS_INLINE T operator[](int i) const { return i<N/2 ? this->lo[i] : this->hi[i-N/2]; }
|
|
|
|
SKVX_ALWAYS_INLINE T& operator[](int i) { return i<N/2 ? this->lo[i] : this->hi[i-N/2]; }
|
2021-10-07 20:37:06 +00:00
|
|
|
|
2021-10-08 20:31:37 +00:00
|
|
|
SKVX_ALWAYS_INLINE static Vec Load(const void* ptr) {
|
|
|
|
Vec v;
|
|
|
|
memcpy(&v, ptr, sizeof(Vec));
|
|
|
|
return v;
|
|
|
|
}
|
|
|
|
SKVX_ALWAYS_INLINE void store(void* ptr) const {
|
|
|
|
memcpy(ptr, this, sizeof(Vec));
|
|
|
|
}
|
2019-02-05 18:42:46 +00:00
|
|
|
};
|
|
|
|
|
2019-02-07 14:49:17 +00:00
|
|
|
template <typename T>
|
2021-10-08 20:31:37 +00:00
|
|
|
struct Vec<1,T> {
|
|
|
|
T val;
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2019-12-17 17:40:14 +00:00
|
|
|
SKVX_ALWAYS_INLINE Vec() = default;
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2021-10-08 21:23:22 +00:00
|
|
|
Vec(T s) : val(s) {}
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2021-10-08 20:31:37 +00:00
|
|
|
SKVX_ALWAYS_INLINE Vec(std::initializer_list<T> xs) : val(xs.size() ? *xs.begin() : 0) {}
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2021-10-08 20:31:37 +00:00
|
|
|
SKVX_ALWAYS_INLINE T operator[](int) const { return val; }
|
|
|
|
SKVX_ALWAYS_INLINE T& operator[](int) { return val; }
|
2021-10-07 20:37:06 +00:00
|
|
|
|
2021-10-08 20:31:37 +00:00
|
|
|
SKVX_ALWAYS_INLINE static Vec Load(const void* ptr) {
|
|
|
|
Vec v;
|
|
|
|
memcpy(&v, ptr, sizeof(Vec));
|
|
|
|
return v;
|
|
|
|
}
|
|
|
|
SKVX_ALWAYS_INLINE void store(void* ptr) const {
|
|
|
|
memcpy(ptr, this, sizeof(Vec));
|
|
|
|
}
|
2019-02-07 14:49:17 +00:00
|
|
|
};
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2020-10-12 18:13:28 +00:00
|
|
|
// Ideally we'd only use bit_pun(), but until this file is always built as C++17 with constexpr if,
|
|
|
|
// we'll sometimes find need to use unchecked_bit_pun(). Please do check the call sites yourself!
|
2019-02-07 14:49:17 +00:00
|
|
|
template <typename D, typename S>
|
2020-09-02 14:00:57 +00:00
|
|
|
SI D unchecked_bit_pun(const S& s) {
|
2019-02-07 14:49:17 +00:00
|
|
|
D d;
|
|
|
|
memcpy(&d, &s, sizeof(D));
|
|
|
|
return d;
|
|
|
|
}
|
|
|
|
|
2020-07-10 20:46:46 +00:00
|
|
|
template <typename D, typename S>
|
2020-09-02 14:00:57 +00:00
|
|
|
SI D bit_pun(const S& s) {
|
2020-07-10 20:46:46 +00:00
|
|
|
static_assert(sizeof(D) == sizeof(S), "");
|
|
|
|
return unchecked_bit_pun<D>(s);
|
|
|
|
}
|
|
|
|
|
2019-02-05 18:42:46 +00:00
|
|
|
// Translate from a value type T to its corresponding Mask, the result of a comparison.
|
2019-02-09 18:48:54 +00:00
|
|
|
template <typename T> struct Mask { using type = T; };
|
|
|
|
template <> struct Mask<float > { using type = int32_t; };
|
|
|
|
template <> struct Mask<double> { using type = int64_t; };
|
|
|
|
template <typename T> using M = typename Mask<T>::type;
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2019-02-07 14:49:17 +00:00
|
|
|
// Join two Vec<N,T> into one Vec<2N,T>.
|
2019-04-16 17:07:23 +00:00
|
|
|
SINT Vec<2*N,T> join(const Vec<N,T>& lo, const Vec<N,T>& hi) {
|
2019-02-07 14:49:17 +00:00
|
|
|
Vec<2*N,T> v;
|
|
|
|
v.lo = lo;
|
|
|
|
v.hi = hi;
|
|
|
|
return v;
|
2019-02-05 18:42:46 +00:00
|
|
|
}
|
|
|
|
|
2020-10-12 18:13:28 +00:00
|
|
|
// We have three strategies for implementing Vec operations:
|
2019-02-05 18:42:46 +00:00
|
|
|
// 1) lean on Clang/GCC vector extensions when available;
|
2020-10-12 18:13:28 +00:00
|
|
|
// 2) use map() to apply a scalar function lane-wise;
|
|
|
|
// 3) recurse on lo/hi to scalar portable implementations.
|
|
|
|
// We can slot in platform-specific implementations as overloads for particular Vec<N,T>,
|
|
|
|
// or often integrate them directly into the recursion of style 3), allowing fine control.
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && (defined(__clang__) || defined(__GNUC__))
|
2019-02-05 18:42:46 +00:00
|
|
|
|
|
|
|
// VExt<N,T> types have the same size as Vec<N,T> and support most operations directly.
|
|
|
|
#if defined(__clang__)
|
|
|
|
template <int N, typename T>
|
|
|
|
using VExt = T __attribute__((ext_vector_type(N)));
|
|
|
|
|
|
|
|
#elif defined(__GNUC__)
|
|
|
|
template <int N, typename T>
|
|
|
|
struct VExtHelper {
|
|
|
|
typedef T __attribute__((vector_size(N*sizeof(T)))) type;
|
|
|
|
};
|
|
|
|
|
|
|
|
template <int N, typename T>
|
|
|
|
using VExt = typename VExtHelper<N,T>::type;
|
2019-02-07 14:49:17 +00:00
|
|
|
|
|
|
|
// For some reason some (new!) versions of GCC cannot seem to deduce N in the generic
|
|
|
|
// to_vec<N,T>() below for N=4 and T=float. This workaround seems to help...
|
2020-09-02 14:00:57 +00:00
|
|
|
SI Vec<4,float> to_vec(VExt<4,float> v) { return bit_pun<Vec<4,float>>(v); }
|
2019-02-05 18:42:46 +00:00
|
|
|
#endif
|
|
|
|
|
2019-04-16 17:07:23 +00:00
|
|
|
SINT VExt<N,T> to_vext(const Vec<N,T>& v) { return bit_pun<VExt<N,T>>(v); }
|
|
|
|
SINT Vec <N,T> to_vec(const VExt<N,T>& v) { return bit_pun<Vec <N,T>>(v); }
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return to_vec<N,T>(to_vext(x) + to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return to_vec<N,T>(to_vext(x) - to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return to_vec<N,T>(to_vext(x) * to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return to_vec<N,T>(to_vext(x) / to_vext(y));
|
|
|
|
}
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return to_vec<N,T>(to_vext(x) ^ to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return to_vec<N,T>(to_vext(x) & to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return to_vec<N,T>(to_vext(x) | to_vext(y));
|
|
|
|
}
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2019-04-16 17:07:23 +00:00
|
|
|
SINT Vec<N,T> operator!(const Vec<N,T>& x) { return to_vec<N,T>(!to_vext(x)); }
|
|
|
|
SINT Vec<N,T> operator-(const Vec<N,T>& x) { return to_vec<N,T>(-to_vext(x)); }
|
|
|
|
SINT Vec<N,T> operator~(const Vec<N,T>& x) { return to_vec<N,T>(~to_vext(x)); }
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SINT Vec<N,T> operator<<(const Vec<N,T>& x, int k) { return to_vec<N,T>(to_vext(x) << k); }
|
|
|
|
SINT Vec<N,T> operator>>(const Vec<N,T>& x, int k) { return to_vec<N,T>(to_vext(x) >> k); }
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return bit_pun<Vec<N,M<T>>>(to_vext(x) == to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return bit_pun<Vec<N,M<T>>>(to_vext(x) != to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return bit_pun<Vec<N,M<T>>>(to_vext(x) <= to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return bit_pun<Vec<N,M<T>>>(to_vext(x) >= to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return bit_pun<Vec<N,M<T>>>(to_vext(x) < to_vext(y));
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return bit_pun<Vec<N,M<T>>>(to_vext(x) > to_vext(y));
|
|
|
|
}
|
2019-02-05 18:42:46 +00:00
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
// Either SKNX_NO_SIMD is defined, or Clang/GCC vector extensions are not available.
|
2020-10-12 18:13:28 +00:00
|
|
|
// We'll implement things portably with N==1 scalar implementations and recursion onto them.
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2019-02-07 14:49:17 +00:00
|
|
|
// N == 1 scalar implementations.
|
2019-04-16 17:07:23 +00:00
|
|
|
SIT Vec<1,T> operator+(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val + y.val; }
|
|
|
|
SIT Vec<1,T> operator-(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val - y.val; }
|
|
|
|
SIT Vec<1,T> operator*(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val * y.val; }
|
|
|
|
SIT Vec<1,T> operator/(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val / y.val; }
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2019-04-16 17:07:23 +00:00
|
|
|
SIT Vec<1,T> operator^(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val ^ y.val; }
|
|
|
|
SIT Vec<1,T> operator&(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val & y.val; }
|
|
|
|
SIT Vec<1,T> operator|(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val | y.val; }
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2019-04-16 17:07:23 +00:00
|
|
|
SIT Vec<1,T> operator!(const Vec<1,T>& x) { return !x.val; }
|
|
|
|
SIT Vec<1,T> operator-(const Vec<1,T>& x) { return -x.val; }
|
|
|
|
SIT Vec<1,T> operator~(const Vec<1,T>& x) { return ~x.val; }
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SIT Vec<1,T> operator<<(const Vec<1,T>& x, int k) { return x.val << k; }
|
|
|
|
SIT Vec<1,T> operator>>(const Vec<1,T>& x, int k) { return x.val >> k; }
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SIT Vec<1,M<T>> operator==(const Vec<1,T>& x, const Vec<1,T>& y) {
|
|
|
|
return x.val == y.val ? ~0 : 0;
|
|
|
|
}
|
|
|
|
SIT Vec<1,M<T>> operator!=(const Vec<1,T>& x, const Vec<1,T>& y) {
|
|
|
|
return x.val != y.val ? ~0 : 0;
|
|
|
|
}
|
|
|
|
SIT Vec<1,M<T>> operator<=(const Vec<1,T>& x, const Vec<1,T>& y) {
|
|
|
|
return x.val <= y.val ? ~0 : 0;
|
|
|
|
}
|
|
|
|
SIT Vec<1,M<T>> operator>=(const Vec<1,T>& x, const Vec<1,T>& y) {
|
|
|
|
return x.val >= y.val ? ~0 : 0;
|
|
|
|
}
|
|
|
|
SIT Vec<1,M<T>> operator< (const Vec<1,T>& x, const Vec<1,T>& y) {
|
|
|
|
return x.val < y.val ? ~0 : 0;
|
|
|
|
}
|
|
|
|
SIT Vec<1,M<T>> operator> (const Vec<1,T>& x, const Vec<1,T>& y) {
|
|
|
|
return x.val > y.val ? ~0 : 0;
|
|
|
|
}
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2020-10-12 18:13:28 +00:00
|
|
|
// Recurse on lo/hi down to N==1 scalar implementations.
|
2020-09-02 14:00:57 +00:00
|
|
|
SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo + y.lo, x.hi + y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo - y.lo, x.hi - y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo * y.lo, x.hi * y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo / y.lo, x.hi / y.hi);
|
|
|
|
}
|
2019-04-16 17:07:23 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo ^ y.lo, x.hi ^ y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo & y.lo, x.hi & y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo | y.lo, x.hi | y.hi);
|
|
|
|
}
|
2019-04-16 17:07:23 +00:00
|
|
|
|
|
|
|
SINT Vec<N,T> operator!(const Vec<N,T>& x) { return join(!x.lo, !x.hi); }
|
|
|
|
SINT Vec<N,T> operator-(const Vec<N,T>& x) { return join(-x.lo, -x.hi); }
|
|
|
|
SINT Vec<N,T> operator~(const Vec<N,T>& x) { return join(~x.lo, ~x.hi); }
|
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SINT Vec<N,T> operator<<(const Vec<N,T>& x, int k) { return join(x.lo << k, x.hi << k); }
|
|
|
|
SINT Vec<N,T> operator>>(const Vec<N,T>& x, int k) { return join(x.lo >> k, x.hi >> k); }
|
2019-04-16 17:07:23 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo == y.lo, x.hi == y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo != y.lo, x.hi != y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo <= y.lo, x.hi <= y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo >= y.lo, x.hi >= y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo < y.lo, x.hi < y.hi);
|
|
|
|
}
|
|
|
|
SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) {
|
|
|
|
return join(x.lo > y.lo, x.hi > y.hi);
|
|
|
|
}
|
2019-02-07 14:49:17 +00:00
|
|
|
#endif
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2020-10-12 18:13:28 +00:00
|
|
|
// Scalar/vector operations splat the scalar to a vector.
|
|
|
|
SINTU Vec<N,T> operator+ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) + y; }
|
|
|
|
SINTU Vec<N,T> operator- (U x, const Vec<N,T>& y) { return Vec<N,T>(x) - y; }
|
|
|
|
SINTU Vec<N,T> operator* (U x, const Vec<N,T>& y) { return Vec<N,T>(x) * y; }
|
|
|
|
SINTU Vec<N,T> operator/ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) / y; }
|
|
|
|
SINTU Vec<N,T> operator^ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) ^ y; }
|
|
|
|
SINTU Vec<N,T> operator& (U x, const Vec<N,T>& y) { return Vec<N,T>(x) & y; }
|
|
|
|
SINTU Vec<N,T> operator| (U x, const Vec<N,T>& y) { return Vec<N,T>(x) | y; }
|
|
|
|
SINTU Vec<N,M<T>> operator==(U x, const Vec<N,T>& y) { return Vec<N,T>(x) == y; }
|
|
|
|
SINTU Vec<N,M<T>> operator!=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) != y; }
|
|
|
|
SINTU Vec<N,M<T>> operator<=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) <= y; }
|
|
|
|
SINTU Vec<N,M<T>> operator>=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) >= y; }
|
|
|
|
SINTU Vec<N,M<T>> operator< (U x, const Vec<N,T>& y) { return Vec<N,T>(x) < y; }
|
|
|
|
SINTU Vec<N,M<T>> operator> (U x, const Vec<N,T>& y) { return Vec<N,T>(x) > y; }
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2020-10-12 18:13:28 +00:00
|
|
|
SINTU Vec<N,T> operator+ (const Vec<N,T>& x, U y) { return x + Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,T> operator- (const Vec<N,T>& x, U y) { return x - Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,T> operator* (const Vec<N,T>& x, U y) { return x * Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,T> operator/ (const Vec<N,T>& x, U y) { return x / Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,T> operator^ (const Vec<N,T>& x, U y) { return x ^ Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,T> operator& (const Vec<N,T>& x, U y) { return x & Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,T> operator| (const Vec<N,T>& x, U y) { return x | Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,M<T>> operator==(const Vec<N,T>& x, U y) { return x == Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,M<T>> operator!=(const Vec<N,T>& x, U y) { return x != Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,M<T>> operator<=(const Vec<N,T>& x, U y) { return x <= Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,M<T>> operator>=(const Vec<N,T>& x, U y) { return x >= Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,M<T>> operator< (const Vec<N,T>& x, U y) { return x < Vec<N,T>(y); }
|
|
|
|
SINTU Vec<N,M<T>> operator> (const Vec<N,T>& x, U y) { return x > Vec<N,T>(y); }
|
|
|
|
|
|
|
|
SINT Vec<N,T>& operator+=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x + y); }
|
|
|
|
SINT Vec<N,T>& operator-=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x - y); }
|
|
|
|
SINT Vec<N,T>& operator*=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x * y); }
|
|
|
|
SINT Vec<N,T>& operator/=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x / y); }
|
|
|
|
SINT Vec<N,T>& operator^=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x ^ y); }
|
|
|
|
SINT Vec<N,T>& operator&=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x & y); }
|
|
|
|
SINT Vec<N,T>& operator|=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x | y); }
|
2019-02-05 18:42:46 +00:00
|
|
|
|
2020-10-12 18:13:28 +00:00
|
|
|
SINTU Vec<N,T>& operator+=(Vec<N,T>& x, U y) { return (x = x + Vec<N,T>(y)); }
|
|
|
|
SINTU Vec<N,T>& operator-=(Vec<N,T>& x, U y) { return (x = x - Vec<N,T>(y)); }
|
|
|
|
SINTU Vec<N,T>& operator*=(Vec<N,T>& x, U y) { return (x = x * Vec<N,T>(y)); }
|
|
|
|
SINTU Vec<N,T>& operator/=(Vec<N,T>& x, U y) { return (x = x / Vec<N,T>(y)); }
|
|
|
|
SINTU Vec<N,T>& operator^=(Vec<N,T>& x, U y) { return (x = x ^ Vec<N,T>(y)); }
|
|
|
|
SINTU Vec<N,T>& operator&=(Vec<N,T>& x, U y) { return (x = x & Vec<N,T>(y)); }
|
|
|
|
SINTU Vec<N,T>& operator|=(Vec<N,T>& x, U y) { return (x = x | Vec<N,T>(y)); }
|
|
|
|
|
|
|
|
SINT Vec<N,T>& operator<<=(Vec<N,T>& x, int bits) { return (x = x << bits); }
|
|
|
|
SINT Vec<N,T>& operator>>=(Vec<N,T>& x, int bits) { return (x = x >> bits); }
|
|
|
|
|
|
|
|
// Some operations we want are not expressible with Clang/GCC vector extensions.
|
refactor skvx min/max
Implement min and max using if_then_else(y<x,...) on vectors
rather than recursing to std::min/std::max applied to scalars.
But actually, factor out and use naive_if_then_else(), which Clang can
reason through better than it can our specialized if_then_else(). This
lets every min() or max() I've looked at compile down to ideal codegen,
vmaxps, vpminsw, etc, where if you use if_then_else() you'd see the
literal comparison and blend as written.
I've been looking at q14x2 codegen in the interpreter, and most things
were already good, unexpectedly even uavg_q14x2. The biggest surprise
was how bad the min/max codegen was, and looking back, even the min_f32
and max_f32 codegen is super bad. This CL fixes all that, leaving us
with the ideal codegen using the specific instruction you'd want,
replacing a giant mess of code that recursed down to scalars.
mul_q14x2 is still bad, but an easy follow up.
Change-Id: I77b5d7c9aa20a9a2f5ceb3e40f1e18ace2a1b5c1
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317310
Reviewed-by: Herb Derby <herb@google.com>
Commit-Queue: Mike Klein <mtklein@google.com>
2020-09-16 15:18:47 +00:00
|
|
|
|
|
|
|
// Clang can reason about naive_if_then_else() and optimize through it better
|
|
|
|
// than if_then_else(), so it's sometimes useful to call it directly when we
|
|
|
|
// think an entire expression should optimize away, e.g. min()/max().
|
|
|
|
SINT Vec<N,T> naive_if_then_else(const Vec<N,M<T>>& cond, const Vec<N,T>& t, const Vec<N,T>& e) {
|
|
|
|
return bit_pun<Vec<N,T>>(( cond & bit_pun<Vec<N, M<T>>>(t)) |
|
|
|
|
(~cond & bit_pun<Vec<N, M<T>>>(e)) );
|
|
|
|
}
|
|
|
|
|
2020-10-12 18:13:28 +00:00
|
|
|
SIT Vec<1,T> if_then_else(const Vec<1,M<T>>& cond, const Vec<1,T>& t, const Vec<1,T>& e) {
|
|
|
|
// In practice this scalar implementation is unlikely to be used. See next if_then_else().
|
|
|
|
return bit_pun<Vec<1,T>>(( cond & bit_pun<Vec<1, M<T>>>(t)) |
|
|
|
|
(~cond & bit_pun<Vec<1, M<T>>>(e)) );
|
|
|
|
}
|
2019-04-16 17:07:23 +00:00
|
|
|
SINT Vec<N,T> if_then_else(const Vec<N,M<T>>& cond, const Vec<N,T>& t, const Vec<N,T>& e) {
|
2020-07-10 20:46:46 +00:00
|
|
|
// Specializations inline here so they can generalize what types the apply to.
|
|
|
|
// (This header is used in C++14 contexts, so we have to kind of fake constexpr if.)
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__AVX2__)
|
2020-09-15 20:26:22 +00:00
|
|
|
if /*constexpr*/ (N*sizeof(T) == 32) {
|
|
|
|
return unchecked_bit_pun<Vec<N,T>>(_mm256_blendv_epi8(unchecked_bit_pun<__m256i>(e),
|
|
|
|
unchecked_bit_pun<__m256i>(t),
|
|
|
|
unchecked_bit_pun<__m256i>(cond)));
|
2020-07-10 20:46:46 +00:00
|
|
|
}
|
|
|
|
#endif
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__SSE4_1__)
|
2020-09-15 20:26:22 +00:00
|
|
|
if /*constexpr*/ (N*sizeof(T) == 16) {
|
|
|
|
return unchecked_bit_pun<Vec<N,T>>(_mm_blendv_epi8(unchecked_bit_pun<__m128i>(e),
|
|
|
|
unchecked_bit_pun<__m128i>(t),
|
|
|
|
unchecked_bit_pun<__m128i>(cond)));
|
2020-07-10 20:46:46 +00:00
|
|
|
}
|
|
|
|
#endif
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__ARM_NEON)
|
2020-09-15 20:26:22 +00:00
|
|
|
if /*constexpr*/ (N*sizeof(T) == 16) {
|
|
|
|
return unchecked_bit_pun<Vec<N,T>>(vbslq_u8(unchecked_bit_pun<uint8x16_t>(cond),
|
|
|
|
unchecked_bit_pun<uint8x16_t>(t),
|
|
|
|
unchecked_bit_pun<uint8x16_t>(e)));
|
2020-07-10 20:46:46 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
// Recurse for large vectors to try to hit the specializations above.
|
2020-09-15 20:26:22 +00:00
|
|
|
if /*constexpr*/ (N*sizeof(T) > 16) {
|
2020-07-10 20:46:46 +00:00
|
|
|
return join(if_then_else(cond.lo, t.lo, e.lo),
|
|
|
|
if_then_else(cond.hi, t.hi, e.hi));
|
|
|
|
}
|
|
|
|
// This default can lead to better code than the recursing onto scalars.
|
refactor skvx min/max
Implement min and max using if_then_else(y<x,...) on vectors
rather than recursing to std::min/std::max applied to scalars.
But actually, factor out and use naive_if_then_else(), which Clang can
reason through better than it can our specialized if_then_else(). This
lets every min() or max() I've looked at compile down to ideal codegen,
vmaxps, vpminsw, etc, where if you use if_then_else() you'd see the
literal comparison and blend as written.
I've been looking at q14x2 codegen in the interpreter, and most things
were already good, unexpectedly even uavg_q14x2. The biggest surprise
was how bad the min/max codegen was, and looking back, even the min_f32
and max_f32 codegen is super bad. This CL fixes all that, leaving us
with the ideal codegen using the specific instruction you'd want,
replacing a giant mess of code that recursed down to scalars.
mul_q14x2 is still bad, but an easy follow up.
Change-Id: I77b5d7c9aa20a9a2f5ceb3e40f1e18ace2a1b5c1
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317310
Reviewed-by: Herb Derby <herb@google.com>
Commit-Queue: Mike Klein <mtklein@google.com>
2020-09-16 15:18:47 +00:00
|
|
|
return naive_if_then_else(cond, t, e);
|
2019-02-05 18:42:46 +00:00
|
|
|
}
|
|
|
|
|
2020-09-16 20:50:00 +00:00
|
|
|
SIT bool any(const Vec<1,T>& x) { return x.val != 0; }
|
|
|
|
SINT bool any(const Vec<N,T>& x) {
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__wasm_simd128__)
|
2020-09-16 20:50:00 +00:00
|
|
|
if constexpr (N == 4 && sizeof(T) == 4) {
|
|
|
|
return wasm_i32x4_any_true(unchecked_bit_pun<VExt<4,int>>(x));
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
return any(x.lo)
|
|
|
|
|| any(x.hi);
|
|
|
|
}
|
|
|
|
|
|
|
|
SIT bool all(const Vec<1,T>& x) { return x.val != 0; }
|
|
|
|
SINT bool all(const Vec<N,T>& x) {
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__AVX2__)
|
2020-09-16 20:50:00 +00:00
|
|
|
if /*constexpr*/ (N*sizeof(T) == 32) {
|
|
|
|
return _mm256_testc_si256(unchecked_bit_pun<__m256i>(x),
|
|
|
|
_mm256_set1_epi32(-1));
|
|
|
|
}
|
|
|
|
#endif
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__SSE4_1__)
|
2020-09-16 20:50:00 +00:00
|
|
|
if /*constexpr*/ (N*sizeof(T) == 16) {
|
|
|
|
return _mm_testc_si128(unchecked_bit_pun<__m128i>(x),
|
|
|
|
_mm_set1_epi32(-1));
|
|
|
|
}
|
|
|
|
#endif
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__wasm_simd128__)
|
2020-09-16 20:50:00 +00:00
|
|
|
if /*constexpr*/ (N == 4 && sizeof(T) == 4) {
|
|
|
|
return wasm_i32x4_all_true(unchecked_bit_pun<VExt<4,int>>(x));
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
return all(x.lo)
|
|
|
|
&& all(x.hi);
|
|
|
|
}
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2019-02-06 20:48:12 +00:00
|
|
|
// cast() Vec<N,S> to Vec<N,D>, as if applying a C-cast to each lane.
|
2020-10-12 18:13:28 +00:00
|
|
|
// TODO: implement with map()?
|
2019-02-07 14:49:17 +00:00
|
|
|
template <typename D, typename S>
|
2020-09-02 14:00:57 +00:00
|
|
|
SI Vec<1,D> cast(const Vec<1,S>& src) { return (D)src.val; }
|
2019-02-07 14:49:17 +00:00
|
|
|
|
2019-02-06 16:56:58 +00:00
|
|
|
template <typename D, int N, typename S>
|
2020-09-02 14:00:57 +00:00
|
|
|
SI Vec<N,D> cast(const Vec<N,S>& src) {
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__clang__)
|
2019-04-10 17:40:31 +00:00
|
|
|
return to_vec(__builtin_convertvector(to_vext(src), VExt<N,D>));
|
2019-02-06 16:56:58 +00:00
|
|
|
#else
|
2019-02-07 14:49:17 +00:00
|
|
|
return join(cast<D>(src.lo), cast<D>(src.hi));
|
2019-02-06 16:56:58 +00:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
refactor skvx min/max
Implement min and max using if_then_else(y<x,...) on vectors
rather than recursing to std::min/std::max applied to scalars.
But actually, factor out and use naive_if_then_else(), which Clang can
reason through better than it can our specialized if_then_else(). This
lets every min() or max() I've looked at compile down to ideal codegen,
vmaxps, vpminsw, etc, where if you use if_then_else() you'd see the
literal comparison and blend as written.
I've been looking at q14x2 codegen in the interpreter, and most things
were already good, unexpectedly even uavg_q14x2. The biggest surprise
was how bad the min/max codegen was, and looking back, even the min_f32
and max_f32 codegen is super bad. This CL fixes all that, leaving us
with the ideal codegen using the specific instruction you'd want,
replacing a giant mess of code that recursed down to scalars.
mul_q14x2 is still bad, but an easy follow up.
Change-Id: I77b5d7c9aa20a9a2f5ceb3e40f1e18ace2a1b5c1
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317310
Reviewed-by: Herb Derby <herb@google.com>
Commit-Queue: Mike Klein <mtklein@google.com>
2020-09-16 15:18:47 +00:00
|
|
|
// min/max match logic of std::min/std::max, which is important when NaN is involved.
|
|
|
|
SIT T min(const Vec<1,T>& x) { return x.val; }
|
|
|
|
SIT T max(const Vec<1,T>& x) { return x.val; }
|
|
|
|
SINT T min(const Vec<N,T>& x) { return std::min(min(x.lo), min(x.hi)); }
|
|
|
|
SINT T max(const Vec<N,T>& x) { return std::max(max(x.lo), max(x.hi)); }
|
|
|
|
|
|
|
|
SINT Vec<N,T> min(const Vec<N,T>& x, const Vec<N,T>& y) { return naive_if_then_else(y < x, y, x); }
|
|
|
|
SINT Vec<N,T> max(const Vec<N,T>& x, const Vec<N,T>& y) { return naive_if_then_else(x < y, y, x); }
|
|
|
|
|
|
|
|
SINTU Vec<N,T> min(const Vec<N,T>& x, U y) { return min(x, Vec<N,T>(y)); }
|
|
|
|
SINTU Vec<N,T> max(const Vec<N,T>& x, U y) { return max(x, Vec<N,T>(y)); }
|
|
|
|
SINTU Vec<N,T> min(U x, const Vec<N,T>& y) { return min(Vec<N,T>(x), y); }
|
|
|
|
SINTU Vec<N,T> max(U x, const Vec<N,T>& y) { return max(Vec<N,T>(x), y); }
|
|
|
|
|
2020-10-16 21:12:10 +00:00
|
|
|
// pin matches the logic of SkTPin, which is important when NaN is involved. It always returns
|
|
|
|
// values in the range lo..hi, and if x is NaN, it returns lo.
|
|
|
|
SINT Vec<N,T> pin(const Vec<N,T>& x, const Vec<N,T>& lo, const Vec<N,T>& hi) {
|
|
|
|
return max(lo, min(x, hi));
|
|
|
|
}
|
refactor skvx min/max
Implement min and max using if_then_else(y<x,...) on vectors
rather than recursing to std::min/std::max applied to scalars.
But actually, factor out and use naive_if_then_else(), which Clang can
reason through better than it can our specialized if_then_else(). This
lets every min() or max() I've looked at compile down to ideal codegen,
vmaxps, vpminsw, etc, where if you use if_then_else() you'd see the
literal comparison and blend as written.
I've been looking at q14x2 codegen in the interpreter, and most things
were already good, unexpectedly even uavg_q14x2. The biggest surprise
was how bad the min/max codegen was, and looking back, even the min_f32
and max_f32 codegen is super bad. This CL fixes all that, leaving us
with the ideal codegen using the specific instruction you'd want,
replacing a giant mess of code that recursed down to scalars.
mul_q14x2 is still bad, but an easy follow up.
Change-Id: I77b5d7c9aa20a9a2f5ceb3e40f1e18ace2a1b5c1
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317310
Reviewed-by: Herb Derby <herb@google.com>
Commit-Queue: Mike Klein <mtklein@google.com>
2020-09-16 15:18:47 +00:00
|
|
|
|
2019-02-06 20:48:12 +00:00
|
|
|
// Shuffle values from a vector pretty arbitrarily:
|
|
|
|
// skvx::Vec<4,float> rgba = {R,G,B,A};
|
|
|
|
// shuffle<2,1,0,3> (rgba) ~> {B,G,R,A}
|
|
|
|
// shuffle<2,1> (rgba) ~> {B,G}
|
|
|
|
// shuffle<2,1,2,1,2,1,2,1>(rgba) ~> {B,G,B,G,B,G,B,G}
|
|
|
|
// shuffle<3,3,3,3> (rgba) ~> {A,A,A,A}
|
|
|
|
// The only real restriction is that the output also be a legal N=power-of-two sknx::Vec.
|
|
|
|
template <int... Ix, int N, typename T>
|
2020-09-02 14:00:57 +00:00
|
|
|
SI Vec<sizeof...(Ix),T> shuffle(const Vec<N,T>& x) {
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__clang__)
|
2020-10-12 18:13:28 +00:00
|
|
|
// TODO: can we just always use { x[Ix]... }?
|
2019-04-11 19:14:16 +00:00
|
|
|
return to_vec<sizeof...(Ix),T>(__builtin_shufflevector(to_vext(x), to_vext(x), Ix...));
|
|
|
|
#else
|
2019-02-06 20:48:12 +00:00
|
|
|
return { x[Ix]... };
|
2019-04-11 19:14:16 +00:00
|
|
|
#endif
|
2019-02-06 20:48:12 +00:00
|
|
|
}
|
2019-02-06 16:56:58 +00:00
|
|
|
|
2020-10-12 17:38:10 +00:00
|
|
|
// Call map(fn, x) for a vector with fn() applied to each lane of x, { fn(x[0]), fn(x[1]), ... },
|
|
|
|
// or map(fn, x,y) for a vector of fn(x[i], y[i]), etc.
|
2020-03-05 16:15:35 +00:00
|
|
|
|
2020-10-12 17:38:10 +00:00
|
|
|
template <typename Fn, typename... Args, size_t... I>
|
|
|
|
SI auto map(std::index_sequence<I...>,
|
|
|
|
Fn&& fn, const Args&... args) -> skvx::Vec<sizeof...(I), decltype(fn(args[0]...))> {
|
2020-10-13 18:56:55 +00:00
|
|
|
auto lane = [&](size_t i)
|
|
|
|
#if defined(__clang__)
|
|
|
|
// CFI, specifically -fsanitize=cfi-icall, seems to give a false positive here,
|
|
|
|
// with errors like "control flow integrity check for type 'float (float)
|
|
|
|
// noexcept' failed during indirect function call... note: sqrtf.cfi_jt defined
|
|
|
|
// here". But we can be quite sure fn is the right type: it's all inferred!
|
|
|
|
// So, stifle CFI in this function.
|
|
|
|
__attribute__((no_sanitize("cfi")))
|
|
|
|
#endif
|
|
|
|
{ return fn(args[i]...); };
|
|
|
|
|
2020-10-12 17:38:10 +00:00
|
|
|
return { lane(I)... };
|
|
|
|
}
|
|
|
|
|
|
|
|
template <typename Fn, int N, typename T, typename... Rest>
|
|
|
|
auto map(Fn&& fn, const Vec<N,T>& first, const Rest&... rest) {
|
|
|
|
// Derive an {0...N-1} index_sequence from the size of the first arg: N lanes in, N lanes out.
|
|
|
|
return map(std::make_index_sequence<N>{}, fn, first,rest...);
|
Reland "update skvx scalar-fallback strategy"
This is a reland of 4985db413d3ee27b944936c71c0cd04740cd28da
...with a better implementation of map(). I don't understand
why we had to revert, but it had something with calling the
function pointer in map_(), so maybe this will help.
I've flattened the map_() / map() merge CL into this one,
and marked the resulting map() as no_sanitize("cfi"). I
don't see anything wrong, so I think it's a false positive.
Original change's description:
> update skvx scalar-fallback strategy
>
> Turns out Clang's a lot better at auto-vectorizing "obvious" scalar code
> into obvious vector code when it's written out the long way, e.g.
>
> F32x4 x = ...;
> x = { sqrtf(x[0]), sqrtf(x[1]), sqrtf(x[2]), sqrtf(x[3]) };
>
> vectorizes into sqrtps a lot more reliably than our recurse-onto-scalars
> strategy, and also better than the other naive approach,
>
> F32x4 x = ...;
> for (int i = 0; i < 4; i++) { x[i] = sqrtf(x[i]); }
>
> So here I've added a map(V, fn) -> V' using C++14 tricks to let the
> compiler handle the expansion of x = { fn(x[0]), fn(x[1]), ...
> fn(x[N-1]) } for any N, and implemented most skvx scalar fallback code
> using that.
>
> With these now vectorizing well at any N, we can remove any
> specializations we'd written for particular N, really tidying up.
>
> Over in the SkVM interpreter, this is a big improvement for ceil and
> floor, which were being done 2 floats at a time instead of 8. They're
> now slimmed way down to
>
> shlq $6, %r13
> vroundps $K, (%r12,%r13), %ymm0
> vroundps $K, 32(%r12,%r13), %ymm1
> jmp ...
>
> where K is 9 or 10 depending on the op.
>
> I haven't found a scalar function that Clang will vectorize to vcvtps2pd
> (the rounding one, not truncating vcvttps2pd), so I've kept lrint()
> written the long way, updated to the style I've been using lately with
> specializations inline.
>
> Change-Id: Ia97abe3c876008228bf62b1daacd6f6140408fc4
> Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317375
> Reviewed-by: Herb Derby <herb@google.com>
> Commit-Queue: Mike Klein <mtklein@google.com>
Cq-Include-Trybots: luci.chromium.try:linux_chromium_cfi_rel_ng
Bug: chromium:1129408
Change-Id: Ia9c14074b9a14a67dd221f4925894d35a551f9d7
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317551
Commit-Queue: Mike Klein <mtklein@google.com>
Reviewed-by: Herb Derby <herb@google.com>
2020-09-16 19:33:37 +00:00
|
|
|
}
|
|
|
|
|
2020-10-12 17:38:10 +00:00
|
|
|
SIN Vec<N,float> ceil(const Vec<N,float>& x) { return map( ceilf, x); }
|
|
|
|
SIN Vec<N,float> floor(const Vec<N,float>& x) { return map(floorf, x); }
|
|
|
|
SIN Vec<N,float> trunc(const Vec<N,float>& x) { return map(truncf, x); }
|
|
|
|
SIN Vec<N,float> round(const Vec<N,float>& x) { return map(roundf, x); }
|
|
|
|
SIN Vec<N,float> sqrt(const Vec<N,float>& x) { return map( sqrtf, x); }
|
|
|
|
SIN Vec<N,float> abs(const Vec<N,float>& x) { return map( fabsf, x); }
|
|
|
|
SIN Vec<N,float> fma(const Vec<N,float>& x,
|
|
|
|
const Vec<N,float>& y,
|
|
|
|
const Vec<N,float>& z) {
|
|
|
|
// I don't understand why Clang's codegen is terrible if we write map(fmaf, x,y,z) directly.
|
|
|
|
auto fn = [](float x, float y, float z) { return fmaf(x,y,z); };
|
|
|
|
return map(fn, x,y,z);
|
|
|
|
}
|
Reland "update skvx scalar-fallback strategy"
This is a reland of 4985db413d3ee27b944936c71c0cd04740cd28da
...with a better implementation of map(). I don't understand
why we had to revert, but it had something with calling the
function pointer in map_(), so maybe this will help.
I've flattened the map_() / map() merge CL into this one,
and marked the resulting map() as no_sanitize("cfi"). I
don't see anything wrong, so I think it's a false positive.
Original change's description:
> update skvx scalar-fallback strategy
>
> Turns out Clang's a lot better at auto-vectorizing "obvious" scalar code
> into obvious vector code when it's written out the long way, e.g.
>
> F32x4 x = ...;
> x = { sqrtf(x[0]), sqrtf(x[1]), sqrtf(x[2]), sqrtf(x[3]) };
>
> vectorizes into sqrtps a lot more reliably than our recurse-onto-scalars
> strategy, and also better than the other naive approach,
>
> F32x4 x = ...;
> for (int i = 0; i < 4; i++) { x[i] = sqrtf(x[i]); }
>
> So here I've added a map(V, fn) -> V' using C++14 tricks to let the
> compiler handle the expansion of x = { fn(x[0]), fn(x[1]), ...
> fn(x[N-1]) } for any N, and implemented most skvx scalar fallback code
> using that.
>
> With these now vectorizing well at any N, we can remove any
> specializations we'd written for particular N, really tidying up.
>
> Over in the SkVM interpreter, this is a big improvement for ceil and
> floor, which were being done 2 floats at a time instead of 8. They're
> now slimmed way down to
>
> shlq $6, %r13
> vroundps $K, (%r12,%r13), %ymm0
> vroundps $K, 32(%r12,%r13), %ymm1
> jmp ...
>
> where K is 9 or 10 depending on the op.
>
> I haven't found a scalar function that Clang will vectorize to vcvtps2pd
> (the rounding one, not truncating vcvttps2pd), so I've kept lrint()
> written the long way, updated to the style I've been using lately with
> specializations inline.
>
> Change-Id: Ia97abe3c876008228bf62b1daacd6f6140408fc4
> Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317375
> Reviewed-by: Herb Derby <herb@google.com>
> Commit-Queue: Mike Klein <mtklein@google.com>
Cq-Include-Trybots: luci.chromium.try:linux_chromium_cfi_rel_ng
Bug: chromium:1129408
Change-Id: Ia9c14074b9a14a67dd221f4925894d35a551f9d7
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317551
Commit-Queue: Mike Klein <mtklein@google.com>
Reviewed-by: Herb Derby <herb@google.com>
2020-09-16 19:33:37 +00:00
|
|
|
|
|
|
|
SI Vec<1,int> lrint(const Vec<1,float>& x) {
|
|
|
|
return (int)lrintf(x.val);
|
|
|
|
}
|
|
|
|
SIN Vec<N,int> lrint(const Vec<N,float>& x) {
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__AVX__)
|
Reland "update skvx scalar-fallback strategy"
This is a reland of 4985db413d3ee27b944936c71c0cd04740cd28da
...with a better implementation of map(). I don't understand
why we had to revert, but it had something with calling the
function pointer in map_(), so maybe this will help.
I've flattened the map_() / map() merge CL into this one,
and marked the resulting map() as no_sanitize("cfi"). I
don't see anything wrong, so I think it's a false positive.
Original change's description:
> update skvx scalar-fallback strategy
>
> Turns out Clang's a lot better at auto-vectorizing "obvious" scalar code
> into obvious vector code when it's written out the long way, e.g.
>
> F32x4 x = ...;
> x = { sqrtf(x[0]), sqrtf(x[1]), sqrtf(x[2]), sqrtf(x[3]) };
>
> vectorizes into sqrtps a lot more reliably than our recurse-onto-scalars
> strategy, and also better than the other naive approach,
>
> F32x4 x = ...;
> for (int i = 0; i < 4; i++) { x[i] = sqrtf(x[i]); }
>
> So here I've added a map(V, fn) -> V' using C++14 tricks to let the
> compiler handle the expansion of x = { fn(x[0]), fn(x[1]), ...
> fn(x[N-1]) } for any N, and implemented most skvx scalar fallback code
> using that.
>
> With these now vectorizing well at any N, we can remove any
> specializations we'd written for particular N, really tidying up.
>
> Over in the SkVM interpreter, this is a big improvement for ceil and
> floor, which were being done 2 floats at a time instead of 8. They're
> now slimmed way down to
>
> shlq $6, %r13
> vroundps $K, (%r12,%r13), %ymm0
> vroundps $K, 32(%r12,%r13), %ymm1
> jmp ...
>
> where K is 9 or 10 depending on the op.
>
> I haven't found a scalar function that Clang will vectorize to vcvtps2pd
> (the rounding one, not truncating vcvttps2pd), so I've kept lrint()
> written the long way, updated to the style I've been using lately with
> specializations inline.
>
> Change-Id: Ia97abe3c876008228bf62b1daacd6f6140408fc4
> Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317375
> Reviewed-by: Herb Derby <herb@google.com>
> Commit-Queue: Mike Klein <mtklein@google.com>
Cq-Include-Trybots: luci.chromium.try:linux_chromium_cfi_rel_ng
Bug: chromium:1129408
Change-Id: Ia9c14074b9a14a67dd221f4925894d35a551f9d7
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317551
Commit-Queue: Mike Klein <mtklein@google.com>
Reviewed-by: Herb Derby <herb@google.com>
2020-09-16 19:33:37 +00:00
|
|
|
if /*constexpr*/ (N == 8) {
|
|
|
|
return unchecked_bit_pun<Vec<N,int>>(_mm256_cvtps_epi32(unchecked_bit_pun<__m256>(x)));
|
|
|
|
}
|
|
|
|
#endif
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__SSE__)
|
Reland "update skvx scalar-fallback strategy"
This is a reland of 4985db413d3ee27b944936c71c0cd04740cd28da
...with a better implementation of map(). I don't understand
why we had to revert, but it had something with calling the
function pointer in map_(), so maybe this will help.
I've flattened the map_() / map() merge CL into this one,
and marked the resulting map() as no_sanitize("cfi"). I
don't see anything wrong, so I think it's a false positive.
Original change's description:
> update skvx scalar-fallback strategy
>
> Turns out Clang's a lot better at auto-vectorizing "obvious" scalar code
> into obvious vector code when it's written out the long way, e.g.
>
> F32x4 x = ...;
> x = { sqrtf(x[0]), sqrtf(x[1]), sqrtf(x[2]), sqrtf(x[3]) };
>
> vectorizes into sqrtps a lot more reliably than our recurse-onto-scalars
> strategy, and also better than the other naive approach,
>
> F32x4 x = ...;
> for (int i = 0; i < 4; i++) { x[i] = sqrtf(x[i]); }
>
> So here I've added a map(V, fn) -> V' using C++14 tricks to let the
> compiler handle the expansion of x = { fn(x[0]), fn(x[1]), ...
> fn(x[N-1]) } for any N, and implemented most skvx scalar fallback code
> using that.
>
> With these now vectorizing well at any N, we can remove any
> specializations we'd written for particular N, really tidying up.
>
> Over in the SkVM interpreter, this is a big improvement for ceil and
> floor, which were being done 2 floats at a time instead of 8. They're
> now slimmed way down to
>
> shlq $6, %r13
> vroundps $K, (%r12,%r13), %ymm0
> vroundps $K, 32(%r12,%r13), %ymm1
> jmp ...
>
> where K is 9 or 10 depending on the op.
>
> I haven't found a scalar function that Clang will vectorize to vcvtps2pd
> (the rounding one, not truncating vcvttps2pd), so I've kept lrint()
> written the long way, updated to the style I've been using lately with
> specializations inline.
>
> Change-Id: Ia97abe3c876008228bf62b1daacd6f6140408fc4
> Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317375
> Reviewed-by: Herb Derby <herb@google.com>
> Commit-Queue: Mike Klein <mtklein@google.com>
Cq-Include-Trybots: luci.chromium.try:linux_chromium_cfi_rel_ng
Bug: chromium:1129408
Change-Id: Ia9c14074b9a14a67dd221f4925894d35a551f9d7
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317551
Commit-Queue: Mike Klein <mtklein@google.com>
Reviewed-by: Herb Derby <herb@google.com>
2020-09-16 19:33:37 +00:00
|
|
|
if /*constexpr*/ (N == 4) {
|
|
|
|
return unchecked_bit_pun<Vec<N,int>>(_mm_cvtps_epi32(unchecked_bit_pun<__m128>(x)));
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
return join(lrint(x.lo),
|
|
|
|
lrint(x.hi));
|
2020-04-30 16:06:23 +00:00
|
|
|
}
|
|
|
|
|
Reland "update skvx scalar-fallback strategy"
This is a reland of 4985db413d3ee27b944936c71c0cd04740cd28da
...with a better implementation of map(). I don't understand
why we had to revert, but it had something with calling the
function pointer in map_(), so maybe this will help.
I've flattened the map_() / map() merge CL into this one,
and marked the resulting map() as no_sanitize("cfi"). I
don't see anything wrong, so I think it's a false positive.
Original change's description:
> update skvx scalar-fallback strategy
>
> Turns out Clang's a lot better at auto-vectorizing "obvious" scalar code
> into obvious vector code when it's written out the long way, e.g.
>
> F32x4 x = ...;
> x = { sqrtf(x[0]), sqrtf(x[1]), sqrtf(x[2]), sqrtf(x[3]) };
>
> vectorizes into sqrtps a lot more reliably than our recurse-onto-scalars
> strategy, and also better than the other naive approach,
>
> F32x4 x = ...;
> for (int i = 0; i < 4; i++) { x[i] = sqrtf(x[i]); }
>
> So here I've added a map(V, fn) -> V' using C++14 tricks to let the
> compiler handle the expansion of x = { fn(x[0]), fn(x[1]), ...
> fn(x[N-1]) } for any N, and implemented most skvx scalar fallback code
> using that.
>
> With these now vectorizing well at any N, we can remove any
> specializations we'd written for particular N, really tidying up.
>
> Over in the SkVM interpreter, this is a big improvement for ceil and
> floor, which were being done 2 floats at a time instead of 8. They're
> now slimmed way down to
>
> shlq $6, %r13
> vroundps $K, (%r12,%r13), %ymm0
> vroundps $K, 32(%r12,%r13), %ymm1
> jmp ...
>
> where K is 9 or 10 depending on the op.
>
> I haven't found a scalar function that Clang will vectorize to vcvtps2pd
> (the rounding one, not truncating vcvttps2pd), so I've kept lrint()
> written the long way, updated to the style I've been using lately with
> specializations inline.
>
> Change-Id: Ia97abe3c876008228bf62b1daacd6f6140408fc4
> Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317375
> Reviewed-by: Herb Derby <herb@google.com>
> Commit-Queue: Mike Klein <mtklein@google.com>
Cq-Include-Trybots: luci.chromium.try:linux_chromium_cfi_rel_ng
Bug: chromium:1129408
Change-Id: Ia9c14074b9a14a67dd221f4925894d35a551f9d7
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/317551
Commit-Queue: Mike Klein <mtklein@google.com>
Reviewed-by: Herb Derby <herb@google.com>
2020-09-16 19:33:37 +00:00
|
|
|
SIN Vec<N,float> fract(const Vec<N,float>& x) { return x - floor(x); }
|
|
|
|
|
2020-10-12 18:13:28 +00:00
|
|
|
// The default logic for to_half/from_half is borrowed from skcms,
|
|
|
|
// and assumes inputs are finite and treat/flush denorm half floats as/to zero.
|
2020-07-15 14:58:51 +00:00
|
|
|
// Key constants to watch for:
|
|
|
|
// - a float is 32-bit, 1-8-23 sign-exponent-mantissa, with 127 exponent bias;
|
|
|
|
// - a half is 16-bit, 1-5-10 sign-exponent-mantissa, with 15 exponent bias.
|
2020-09-02 14:00:57 +00:00
|
|
|
SIN Vec<N,uint16_t> to_half_finite_ftz(const Vec<N,float>& x) {
|
2020-07-15 14:58:51 +00:00
|
|
|
Vec<N,uint32_t> sem = bit_pun<Vec<N,uint32_t>>(x),
|
|
|
|
s = sem & 0x8000'0000,
|
|
|
|
em = sem ^ s,
|
|
|
|
is_denorm = em < 0x3880'0000;
|
|
|
|
return cast<uint16_t>(if_then_else(is_denorm, Vec<N,uint32_t>(0)
|
|
|
|
, (s>>16) + (em>>13) - ((127-15)<<10)));
|
|
|
|
}
|
2020-09-02 14:00:57 +00:00
|
|
|
SIN Vec<N,float> from_half_finite_ftz(const Vec<N,uint16_t>& x) {
|
2020-07-15 14:58:51 +00:00
|
|
|
Vec<N,uint32_t> wide = cast<uint32_t>(x),
|
|
|
|
s = wide & 0x8000,
|
|
|
|
em = wide ^ s;
|
|
|
|
auto is_denorm = bit_pun<Vec<N,int32_t>>(em < 0x0400);
|
|
|
|
return if_then_else(is_denorm, Vec<N,float>(0)
|
|
|
|
, bit_pun<Vec<N,float>>( (s<<16) + (em<<13) + ((127-15)<<23) ));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Like if_then_else(), these N=1 base cases won't actually be used unless explicitly called.
|
2020-09-02 14:00:57 +00:00
|
|
|
SI Vec<1,uint16_t> to_half(const Vec<1,float>& x) { return to_half_finite_ftz(x); }
|
|
|
|
SI Vec<1,float> from_half(const Vec<1,uint16_t>& x) { return from_half_finite_ftz(x); }
|
2020-07-15 14:58:51 +00:00
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SIN Vec<N,uint16_t> to_half(const Vec<N,float>& x) {
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__F16C__)
|
2020-07-15 14:58:51 +00:00
|
|
|
if /*constexpr*/ (N == 8) {
|
|
|
|
return unchecked_bit_pun<Vec<N,uint16_t>>(_mm256_cvtps_ph(unchecked_bit_pun<__m256>(x),
|
|
|
|
_MM_FROUND_CUR_DIRECTION));
|
|
|
|
}
|
|
|
|
#endif
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__aarch64__)
|
2020-07-15 14:58:51 +00:00
|
|
|
if /*constexpr*/ (N == 4) {
|
|
|
|
return unchecked_bit_pun<Vec<N,uint16_t>>(vcvt_f16_f32(unchecked_bit_pun<float32x4_t>(x)));
|
|
|
|
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
if /*constexpr*/ (N > 4) {
|
|
|
|
return join(to_half(x.lo),
|
|
|
|
to_half(x.hi));
|
|
|
|
}
|
|
|
|
return to_half_finite_ftz(x);
|
|
|
|
}
|
|
|
|
|
2020-09-02 14:00:57 +00:00
|
|
|
SIN Vec<N,float> from_half(const Vec<N,uint16_t>& x) {
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__F16C__)
|
2020-07-15 14:58:51 +00:00
|
|
|
if /*constexpr*/ (N == 8) {
|
|
|
|
return unchecked_bit_pun<Vec<N,float>>(_mm256_cvtph_ps(unchecked_bit_pun<__m128i>(x)));
|
|
|
|
}
|
|
|
|
#endif
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__aarch64__)
|
2020-07-15 14:58:51 +00:00
|
|
|
if /*constexpr*/ (N == 4) {
|
2020-07-31 18:04:25 +00:00
|
|
|
return unchecked_bit_pun<Vec<N,float>>(vcvt_f32_f16(unchecked_bit_pun<float16x4_t>(x)));
|
2020-07-15 14:58:51 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
if /*constexpr*/ (N > 4) {
|
|
|
|
return join(from_half(x.lo),
|
|
|
|
from_half(x.hi));
|
|
|
|
}
|
|
|
|
return from_half_finite_ftz(x);
|
|
|
|
}
|
|
|
|
|
2019-04-11 16:52:51 +00:00
|
|
|
// div255(x) = (x + 127) / 255 is a bit-exact rounding divide-by-255, packing down to 8-bit.
|
2020-09-02 14:00:57 +00:00
|
|
|
SIN Vec<N,uint8_t> div255(const Vec<N,uint16_t>& x) {
|
2019-04-11 16:52:51 +00:00
|
|
|
return cast<uint8_t>( (x+127)/255 );
|
|
|
|
}
|
|
|
|
|
|
|
|
// approx_scale(x,y) approximates div255(cast<uint16_t>(x)*cast<uint16_t>(y)) within a bit,
|
|
|
|
// and is always perfect when x or y is 0 or 255.
|
2020-09-02 14:00:57 +00:00
|
|
|
SIN Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,uint8_t>& y) {
|
2019-04-11 16:52:51 +00:00
|
|
|
// All of (x*y+x)/256, (x*y+y)/256, and (x*y+255)/256 meet the criteria above.
|
|
|
|
// We happen to have historically picked (x*y+x)/256.
|
|
|
|
auto X = cast<uint16_t>(x),
|
|
|
|
Y = cast<uint16_t>(y);
|
|
|
|
return cast<uint8_t>( (X*Y+X)/256 );
|
|
|
|
}
|
|
|
|
|
2021-10-11 19:37:05 +00:00
|
|
|
// The ScaledDividerU32 takes a divisor > 1, and creates a function divide(numerator) that
|
|
|
|
// calculates a numerator / denominator. For this to be rounded properly, numerator should have
|
|
|
|
// half added in:
|
|
|
|
// divide(numerator + half) == floor(numerator/denominator + 1/2).
|
|
|
|
//
|
|
|
|
// This gives an answer within +/- 1 from the true value.
|
|
|
|
//
|
|
|
|
// Derivation of half:
|
|
|
|
// numerator/denominator + 1/2 = (numerator + half) / d
|
|
|
|
// numerator + denominator / 2 = numerator + half
|
|
|
|
// half = denominator / 2.
|
|
|
|
//
|
|
|
|
// Because half is divided by 2, that division must also be rounded.
|
|
|
|
// half == denominator / 2 = (denominator + 1) / 2.
|
|
|
|
//
|
|
|
|
// The divisorFactor is just a scaled value:
|
|
|
|
// divisorFactor = (1 / divisor) * 2 ^ 32.
|
|
|
|
// The maximum that can be divided and rounded is UINT_MAX - half.
|
|
|
|
class ScaledDividerU32 {
|
|
|
|
public:
|
|
|
|
explicit ScaledDividerU32(uint32_t divisor)
|
|
|
|
: fDivisorFactor{(uint32_t)(std::round((1.0 / divisor) * (1ull << 32)))}
|
|
|
|
, fHalf{(divisor + 1) >> 1} {
|
|
|
|
assert(divisor > 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
Vec<4, uint32_t> divide(const Vec<4, uint32_t>& numerator) const {
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__ARM_NEON)
|
2021-10-11 19:37:05 +00:00
|
|
|
uint64x2_t hi = vmull_n_u32(vget_high_u32(to_vext(numerator)), fDivisorFactor);
|
|
|
|
uint64x2_t lo = vmull_n_u32(vget_low_u32(to_vext(numerator)), fDivisorFactor);
|
|
|
|
|
|
|
|
return to_vec<4, uint32_t>(vcombine_u32(vshrn_n_u64(lo,32), vshrn_n_u64(hi,32)));
|
2022-01-18 20:22:45 +00:00
|
|
|
#else
|
2021-10-11 19:37:05 +00:00
|
|
|
return cast<uint32_t>((cast<uint64_t>(numerator) * fDivisorFactor) >> 32);
|
2022-01-18 20:22:45 +00:00
|
|
|
#endif
|
2021-10-11 19:37:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
uint32_t half() const { return fHalf; }
|
|
|
|
|
|
|
|
private:
|
|
|
|
const uint32_t fDivisorFactor;
|
|
|
|
const uint32_t fHalf;
|
|
|
|
};
|
|
|
|
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__ARM_NEON)
|
|
|
|
// With NEON we can do eight u8*u8 -> u16 in one instruction, vmull_u8 (read, mul-long).
|
|
|
|
SI Vec<8,uint16_t> mull(const Vec<8,uint8_t>& x,
|
|
|
|
const Vec<8,uint8_t>& y) {
|
|
|
|
return to_vec<8,uint16_t>(vmull_u8(to_vext(x),
|
|
|
|
to_vext(y)));
|
|
|
|
}
|
2019-04-16 17:07:23 +00:00
|
|
|
|
2022-01-18 20:22:45 +00:00
|
|
|
SIN std::enable_if_t<(N < 8), Vec<N,uint16_t>> mull(const Vec<N,uint8_t>& x,
|
|
|
|
const Vec<N,uint8_t>& y) {
|
|
|
|
// N < 8 --> double up data until N == 8, returning the part we need.
|
|
|
|
return mull(join(x,x),
|
|
|
|
join(y,y)).lo;
|
|
|
|
}
|
|
|
|
|
|
|
|
SIN std::enable_if_t<(N > 8), Vec<N,uint16_t>> mull(const Vec<N,uint8_t>& x,
|
|
|
|
const Vec<N,uint8_t>& y) {
|
|
|
|
// N > 8 --> usual join(lo,hi) strategy to recurse down to N == 8.
|
|
|
|
return join(mull(x.lo, y.lo),
|
|
|
|
mull(x.hi, y.hi));
|
|
|
|
}
|
2019-04-16 17:07:23 +00:00
|
|
|
|
|
|
|
#else
|
2022-01-18 20:22:45 +00:00
|
|
|
|
|
|
|
// Nothing special when we don't have NEON... just cast up to 16-bit and multiply.
|
|
|
|
SIN Vec<N,uint16_t> mull(const Vec<N,uint8_t>& x,
|
|
|
|
const Vec<N,uint8_t>& y) {
|
|
|
|
return cast<uint16_t>(x)
|
|
|
|
* cast<uint16_t>(y);
|
|
|
|
}
|
2019-04-16 17:07:23 +00:00
|
|
|
#endif
|
|
|
|
|
2022-05-16 18:58:37 +00:00
|
|
|
SINT T dot(const Vec<N, T>& a, const Vec<N, T>& b) {
|
|
|
|
auto ab = a*b;
|
|
|
|
if constexpr (N == 2) {
|
|
|
|
return ab[0] + ab[1];
|
|
|
|
} else if constexpr (N == 4) {
|
|
|
|
return ab[0] + ab[1] + ab[2] + ab[3];
|
|
|
|
} else {
|
|
|
|
T sum = ab[0];
|
|
|
|
for (int i = 1; i < N; ++i) {
|
|
|
|
sum += ab[i];
|
|
|
|
}
|
|
|
|
return sum;
|
|
|
|
}
|
2021-10-08 16:20:06 +00:00
|
|
|
}
|
|
|
|
|
2022-05-16 18:58:37 +00:00
|
|
|
SI float cross(const Vec<2, float>& a, const Vec<2, float>& b) {
|
|
|
|
auto x = a * shuffle<1,0>(b);
|
|
|
|
return x[0] - x[1];
|
|
|
|
}
|
2021-10-11 15:42:49 +00:00
|
|
|
|
2021-10-08 16:20:06 +00:00
|
|
|
// De-interleaving load of 4 vectors.
|
|
|
|
//
|
|
|
|
// WARNING: These are really only supported well on NEON. Consider restructuring your data before
|
|
|
|
// resorting to these methods.
|
|
|
|
SIT void strided_load4(const T* v,
|
2022-05-16 18:58:37 +00:00
|
|
|
Vec<1,T>& a,
|
|
|
|
Vec<1,T>& b,
|
|
|
|
Vec<1,T>& c,
|
|
|
|
Vec<1,T>& d) {
|
2021-10-08 16:20:06 +00:00
|
|
|
a.val = v[0];
|
|
|
|
b.val = v[1];
|
|
|
|
c.val = v[2];
|
|
|
|
d.val = v[3];
|
|
|
|
}
|
|
|
|
SINT void strided_load4(const T* v,
|
2022-05-16 18:58:37 +00:00
|
|
|
Vec<N,T>& a,
|
|
|
|
Vec<N,T>& b,
|
|
|
|
Vec<N,T>& c,
|
|
|
|
Vec<N,T>& d) {
|
2021-10-08 16:20:06 +00:00
|
|
|
strided_load4(v, a.lo, b.lo, c.lo, d.lo);
|
|
|
|
strided_load4(v + 4*(N/2), a.hi, b.hi, c.hi, d.hi);
|
|
|
|
}
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__ARM_NEON)
|
2021-10-08 16:20:06 +00:00
|
|
|
#define IMPL_LOAD4_TRANSPOSED(N, T, VLD) \
|
|
|
|
SI void strided_load4(const T* v, \
|
2022-05-16 18:58:37 +00:00
|
|
|
Vec<N,T>& a, \
|
|
|
|
Vec<N,T>& b, \
|
|
|
|
Vec<N,T>& c, \
|
|
|
|
Vec<N,T>& d) { \
|
2021-10-08 16:20:06 +00:00
|
|
|
auto mat = VLD(v); \
|
2022-05-16 18:58:37 +00:00
|
|
|
a = bit_pun<Vec<N,T>>(mat.val[0]); \
|
|
|
|
b = bit_pun<Vec<N,T>>(mat.val[1]); \
|
|
|
|
c = bit_pun<Vec<N,T>>(mat.val[2]); \
|
|
|
|
d = bit_pun<Vec<N,T>>(mat.val[3]); \
|
2021-10-08 16:20:06 +00:00
|
|
|
}
|
2022-02-11 21:15:06 +00:00
|
|
|
IMPL_LOAD4_TRANSPOSED(2, uint32_t, vld4_u32)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(4, uint16_t, vld4_u16)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(8, uint8_t, vld4_u8)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(2, int32_t, vld4_s32)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(4, int16_t, vld4_s16)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(8, int8_t, vld4_s8)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(2, float, vld4_f32)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(4, uint32_t, vld4q_u32)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(8, uint16_t, vld4q_u16)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(16, uint8_t, vld4q_u8)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(4, int32_t, vld4q_s32)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(8, int16_t, vld4q_s16)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(16, int8_t, vld4q_s8)
|
|
|
|
IMPL_LOAD4_TRANSPOSED(4, float, vld4q_f32)
|
2021-10-08 16:20:06 +00:00
|
|
|
#undef IMPL_LOAD4_TRANSPOSED
|
2022-01-18 20:22:45 +00:00
|
|
|
|
|
|
|
#elif SKVX_USE_SIMD && defined(__SSE__)
|
|
|
|
|
2021-10-08 16:20:06 +00:00
|
|
|
SI void strided_load4(const float* v,
|
|
|
|
Vec<4,float>& a,
|
|
|
|
Vec<4,float>& b,
|
|
|
|
Vec<4,float>& c,
|
|
|
|
Vec<4,float>& d) {
|
|
|
|
__m128 a_ = _mm_loadu_ps(v);
|
|
|
|
__m128 b_ = _mm_loadu_ps(v+4);
|
|
|
|
__m128 c_ = _mm_loadu_ps(v+8);
|
|
|
|
__m128 d_ = _mm_loadu_ps(v+12);
|
|
|
|
_MM_TRANSPOSE4_PS(a_, b_, c_, d_);
|
|
|
|
a = bit_pun<Vec<4,float>>(a_);
|
|
|
|
b = bit_pun<Vec<4,float>>(b_);
|
|
|
|
c = bit_pun<Vec<4,float>>(c_);
|
|
|
|
d = bit_pun<Vec<4,float>>(d_);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
// De-interleaving load of 2 vectors.
|
|
|
|
//
|
|
|
|
// WARNING: These are really only supported well on NEON. Consider restructuring your data before
|
|
|
|
// resorting to these methods.
|
2022-05-16 18:58:37 +00:00
|
|
|
SIT void strided_load2(const T* v, Vec<1,T>& a, Vec<1,T>& b) {
|
2021-10-08 16:20:06 +00:00
|
|
|
a.val = v[0];
|
|
|
|
b.val = v[1];
|
|
|
|
}
|
2022-05-16 18:58:37 +00:00
|
|
|
SINT void strided_load2(const T* v, Vec<N,T>& a, Vec<N,T>& b) {
|
2021-10-08 16:20:06 +00:00
|
|
|
strided_load2(v, a.lo, b.lo);
|
|
|
|
strided_load2(v + 2*(N/2), a.hi, b.hi);
|
|
|
|
}
|
2022-01-18 20:22:45 +00:00
|
|
|
#if SKVX_USE_SIMD && defined(__ARM_NEON)
|
2021-10-08 16:20:06 +00:00
|
|
|
#define IMPL_LOAD2_TRANSPOSED(N, T, VLD) \
|
2022-05-16 18:58:37 +00:00
|
|
|
SI void strided_load2(const T* v, Vec<N,T>& a, Vec<N,T>& b) { \
|
2021-10-08 16:20:06 +00:00
|
|
|
auto mat = VLD(v); \
|
2022-05-16 18:58:37 +00:00
|
|
|
a = bit_pun<Vec<N,T>>(mat.val[0]); \
|
|
|
|
b = bit_pun<Vec<N,T>>(mat.val[1]); \
|
2021-10-08 16:20:06 +00:00
|
|
|
}
|
2022-02-11 21:15:06 +00:00
|
|
|
IMPL_LOAD2_TRANSPOSED(2, uint32_t, vld2_u32)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(4, uint16_t, vld2_u16)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(8, uint8_t, vld2_u8)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(2, int32_t, vld2_s32)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(4, int16_t, vld2_s16)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(8, int8_t, vld2_s8)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(2, float, vld2_f32)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(4, uint32_t, vld2q_u32)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(8, uint16_t, vld2q_u16)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(16, uint8_t, vld2q_u8)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(4, int32_t, vld2q_s32)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(8, int16_t, vld2q_s16)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(16, int8_t, vld2q_s8)
|
|
|
|
IMPL_LOAD2_TRANSPOSED(4, float, vld2q_f32)
|
2021-10-08 16:20:06 +00:00
|
|
|
#undef IMPL_LOAD2_TRANSPOSED
|
|
|
|
#endif
|
|
|
|
|
2022-05-16 18:58:37 +00:00
|
|
|
// Define commonly used aliases
|
|
|
|
using float2 = Vec< 2, float>;
|
|
|
|
using float4 = Vec< 4, float>;
|
|
|
|
using float8 = Vec< 8, float>;
|
|
|
|
|
|
|
|
using double2 = Vec< 2, double>;
|
|
|
|
using double4 = Vec< 4, double>;
|
|
|
|
using double8 = Vec< 8, double>;
|
|
|
|
|
|
|
|
using byte2 = Vec< 2, uint8_t>;
|
|
|
|
using byte4 = Vec< 4, uint8_t>;
|
|
|
|
using byte8 = Vec< 8, uint8_t>;
|
|
|
|
using byte16 = Vec<16, uint8_t>;
|
|
|
|
|
|
|
|
using int2 = Vec< 2, int32_t>;
|
|
|
|
using int4 = Vec< 4, int32_t>;
|
|
|
|
using int8 = Vec< 8, int32_t>;
|
|
|
|
|
|
|
|
using uint2 = Vec< 2, uint32_t>;
|
|
|
|
using uint4 = Vec< 4, uint32_t>;
|
|
|
|
using uint8 = Vec< 8, uint32_t>;
|
|
|
|
|
|
|
|
using long2 = Vec< 2, int64_t>;
|
|
|
|
using long4 = Vec< 4, int64_t>;
|
|
|
|
using long8 = Vec< 8, int64_t>;
|
|
|
|
|
|
|
|
// Use with from_half and to_half to convert between floatX, and use these for storage.
|
|
|
|
using half2 = Vec< 2, uint16_t>;
|
|
|
|
using half4 = Vec< 4, uint16_t>;
|
|
|
|
using half8 = Vec< 8, uint16_t>;
|
|
|
|
|
2019-02-07 14:49:17 +00:00
|
|
|
} // namespace skvx
|
|
|
|
|
2019-03-14 18:30:42 +00:00
|
|
|
#undef SINTU
|
2019-02-07 14:49:17 +00:00
|
|
|
#undef SINT
|
2020-11-23 21:25:03 +00:00
|
|
|
#undef SIN
|
2019-02-07 14:49:17 +00:00
|
|
|
#undef SIT
|
2020-09-02 14:00:57 +00:00
|
|
|
#undef SI
|
2020-11-23 21:25:03 +00:00
|
|
|
#undef SKVX_ALWAYS_INLINE
|
2022-01-18 20:22:45 +00:00
|
|
|
#undef SKVX_USE_SIMD
|
2019-02-05 18:42:46 +00:00
|
|
|
|
|
|
|
#endif//SKVX_DEFINED
|