Enable AArch64 versions of ARGB32 NEON drawhelpers

Fix the ARGB32 NEON-optimized drawhelpers so they also build on AArch64.

The RGB16 NEON-drawhelpers are not converted as they use more assembly.

Change-Id: I8b75fadf5bad74360a2ab6aec9a6bf50df80c1b9
Reviewed-by: Erik Verbruggen <erik.verbruggen@theqtcompany.com>

src/gui/painting/painting.pri

@@ -101,11 +101,11 @@ SSE4_1_SOURCES += painting/qdrawhelper_sse4.cpp \
                   painting/qimagescale_sse4.cpp
 AVX2_SOURCES += painting/qdrawhelper_avx2.cpp
 
-!ios:!contains(QT_ARCH, "arm64")  {
+!ios {
     CONFIG += no_clang_integrated_as
     NEON_SOURCES += painting/qdrawhelper_neon.cpp
     NEON_HEADERS += painting/qdrawhelper_neon_p.h
-    NEON_ASM += ../3rdparty/pixman/pixman-arm-neon-asm.S painting/qdrawhelper_neon_asm.S
+    !contains(QT_ARCH, "arm64"): NEON_ASM += ../3rdparty/pixman/pixman-arm-neon-asm.S painting/qdrawhelper_neon_asm.S
 }
 
 MIPS_DSP_SOURCES += painting/qdrawhelper_mips_dsp.cpp

src/gui/painting/qdrawhelper.cpp

@@ -6309,7 +6309,7 @@ void qt_memfill16(quint16 *dest, quint16 color, int count)
     qt_memfill_template<quint16>(dest, color, count);
 }
 #endif
-#if !defined(__SSE2__) && (!defined(__ARM_NEON__) || defined(Q_PROCESSOR_ARM_64))
+#if !defined(__SSE2__) && !defined(__ARM_NEON__)
 #  ifdef QT_COMPILER_SUPPORTS_MIPS_DSP
 extern "C" void qt_memfill32_asm_mips_dsp(quint32 *, quint32, int);
 #  endif
@@ -6425,14 +6425,11 @@ void qInitDrawhelperAsm()
 
 #endif // SSE2
 
-#if defined(__ARM_NEON__) && !defined(Q_OS_IOS) && !defined(Q_PROCESSOR_ARM_64)
+#if defined(__ARM_NEON__) && !defined(Q_OS_IOS)
     qBlendFunctions[QImage::Format_RGB32][QImage::Format_RGB32] = qt_blend_rgb32_on_rgb32_neon;
     qBlendFunctions[QImage::Format_ARGB32_Premultiplied][QImage::Format_RGB32] = qt_blend_rgb32_on_rgb32_neon;
     qBlendFunctions[QImage::Format_RGB32][QImage::Format_ARGB32_Premultiplied] = qt_blend_argb32_on_argb32_neon;
     qBlendFunctions[QImage::Format_ARGB32_Premultiplied][QImage::Format_ARGB32_Premultiplied] = qt_blend_argb32_on_argb32_neon;
-    qBlendFunctions[QImage::Format_RGB16][QImage::Format_ARGB32_Premultiplied] = qt_blend_argb32_on_rgb16_neon;
-    qBlendFunctions[QImage::Format_ARGB32_Premultiplied][QImage::Format_RGB16] = qt_blend_rgb16_on_argb32_neon;
-    qBlendFunctions[QImage::Format_RGB16][QImage::Format_RGB16] = qt_blend_rgb16_on_rgb16_neon;
 #if Q_BYTE_ORDER == Q_LITTLE_ENDIAN
     qBlendFunctions[QImage::Format_RGBX8888][QImage::Format_RGBX8888] = qt_blend_rgb32_on_rgb32_neon;
     qBlendFunctions[QImage::Format_RGBA8888_Premultiplied][QImage::Format_RGBX8888] = qt_blend_rgb32_on_rgb32_neon;
@@ -6440,6 +6437,21 @@ void qInitDrawhelperAsm()
     qBlendFunctions[QImage::Format_RGBA8888_Premultiplied][QImage::Format_RGBA8888_Premultiplied] = qt_blend_argb32_on_argb32_neon;
 #endif
 
+    qt_functionForMode_C[QPainter::CompositionMode_SourceOver] = qt_blend_argb32_on_argb32_scanline_neon;
+    qt_functionForModeSolid_C[QPainter::CompositionMode_SourceOver] = comp_func_solid_SourceOver_neon;
+    qt_functionForMode_C[QPainter::CompositionMode_Plus] = comp_func_Plus_neon;
+
+    extern const uint * QT_FASTCALL qt_fetch_radial_gradient_neon(uint *buffer, const Operator *op, const QSpanData *data,
+                                                                  int y, int x, int length);
+
+    qt_fetch_radial_gradient = qt_fetch_radial_gradient_neon;
+
+#if !defined(Q_PROCESSOR_ARM_64)
+    // The RGB16 helpers are using Arm32 assemblythat has not been ported to AArch64
+    qBlendFunctions[QImage::Format_RGB16][QImage::Format_ARGB32_Premultiplied] = qt_blend_argb32_on_rgb16_neon;
+    qBlendFunctions[QImage::Format_ARGB32_Premultiplied][QImage::Format_RGB16] = qt_blend_rgb16_on_argb32_neon;
+    qBlendFunctions[QImage::Format_RGB16][QImage::Format_RGB16] = qt_blend_rgb16_on_rgb16_neon;
+
     qScaleFunctions[QImage::Format_RGB16][QImage::Format_ARGB32_Premultiplied] = qt_scale_image_argb32_on_rgb16_neon;
     qScaleFunctions[QImage::Format_RGB16][QImage::Format_RGB16] = qt_scale_image_rgb16_on_rgb16_neon;
 
@@ -6448,19 +6460,13 @@ void qInitDrawhelperAsm()
 
     qDrawHelper[QImage::Format_RGB16].alphamapBlit = qt_alphamapblit_quint16_neon;
 
-    qt_functionForMode_C[QPainter::CompositionMode_SourceOver] = qt_blend_argb32_on_argb32_scanline_neon;
-    qt_functionForModeSolid_C[QPainter::CompositionMode_SourceOver] = comp_func_solid_SourceOver_neon;
-    qt_functionForMode_C[QPainter::CompositionMode_Plus] = comp_func_Plus_neon;
     destFetchProc[QImage::Format_RGB16] = qt_destFetchRGB16_neon;
     destStoreProc[QImage::Format_RGB16] = qt_destStoreRGB16_neon;
 
     qMemRotateFunctions[QImage::Format_RGB16][0] = qt_memrotate90_16_neon;
     qMemRotateFunctions[QImage::Format_RGB16][2] = qt_memrotate270_16_neon;
+#endif
 
-    extern const uint * QT_FASTCALL qt_fetch_radial_gradient_neon(uint *buffer, const Operator *op, const QSpanData *data,
-                                                                  int y, int x, int length);
-
-    qt_fetch_radial_gradient = qt_fetch_radial_gradient_neon;
 #endif
 
 #if defined(Q_PROCESSOR_MIPS_32) && defined(QT_COMPILER_SUPPORTS_MIPS_DSP)

src/gui/painting/qdrawhelper_neon.cpp

@@ -44,6 +44,7 @@ QT_BEGIN_NAMESPACE
 void qt_memfill32(quint32 *dest, quint32 value, int count)
 {
     const int epilogueSize = count % 16;
+#if !defined(Q_PROCESSOR_ARM_64)
     if (count >= 16) {
         quint32 *const neonEnd = dest + count - epilogueSize;
         register uint32x4_t valueVector1 asm ("q0") = vdupq_n_u32(value);
@@ -58,6 +59,22 @@ void qt_memfill32(quint32 *dest, quint32 value, int count)
             );
         }
     }
+#else
+    if (count >= 16) {
+        quint32 *const neonEnd = dest + count - epilogueSize;
+        register uint32x4_t valueVector1 asm ("v0") = vdupq_n_u32(value);
+        register uint32x4_t valueVector2 asm ("v1") = valueVector1;
+        while (dest != neonEnd) {
+            asm volatile (
+                "st2     { v0.4s, v1.4s }, [%[DST]], #32 \n\t"
+                "st2     { v0.4s, v1.4s }, [%[DST]], #32 \n\t"
+                : [DST]"+r" (dest)
+                : [VALUE1]"w"(valueVector1), [VALUE2]"w"(valueVector2)
+                : "memory"
+            );
+        }
+    }
+#endif
 
     switch (epilogueSize)
     {
@@ -118,6 +135,7 @@ static inline uint16x8_t qvsource_over_u16(uint16x8_t src16, uint16x8_t dst16, u
     return vaddq_u16(src16, qvbyte_mul_u16(dst16, alpha16, half));
 }
 
+#if !defined(Q_PROCESSOR_ARM_64)
 extern "C" void
 pixman_composite_over_8888_0565_asm_neon (int32_t   w,
                                           int32_t   h,
@@ -164,7 +182,6 @@ pixman_composite_src_0565_0565_asm_neon (int32_t   w,
                                          int32_t   dst_stride,
                                          uint16_t *src,
                                          int32_t   src_stride);
-
 // qblendfunctions.cpp
 void qt_blend_argb32_on_rgb16_const_alpha(uchar *destPixels, int dbpl,
                                           const uchar *srcPixels, int sbpl,
@@ -204,6 +221,7 @@ void qt_blend_rgb16_on_rgb16(uchar *dst, int dbpl,
                              int w, int h,
                              int const_alpha);
 
+
 template <int N>
 static inline void scanLineBlit16(quint16 *dst, quint16 *src, int dstride)
 {
@@ -329,11 +347,16 @@ void qt_blend_argb32_on_rgb16_neon(uchar *destPixels, int dbpl,
 
     pixman_composite_over_8888_0565_asm_neon(w, h, dst, dbpl / 2, src, sbpl / 4);
 }
+#endif
 
 void qt_blend_argb32_on_argb32_scanline_neon(uint *dest, const uint *src, int length, uint const_alpha)
 {
     if (const_alpha == 255) {
+#if !defined(Q_PROCESSOR_ARM_64)
         pixman_composite_scanline_over_asm_neon(length, dest, src);
+#else
+        qt_blend_argb32_on_argb32_neon((uchar *)dest, 4 * length, (uchar *)src, 4 * length, length, 1, 256);
+#endif
     } else {
         qt_blend_argb32_on_argb32_neon((uchar *)dest, 4 * length, (uchar *)src, 4 * length, length, 1, (const_alpha * 256) / 255);
     }
@@ -349,7 +372,51 @@ void qt_blend_argb32_on_argb32_neon(uchar *destPixels, int dbpl,
     uint16x8_t half = vdupq_n_u16(0x80);
     uint16x8_t full = vdupq_n_u16(0xff);
     if (const_alpha == 256) {
+#if !defined(Q_PROCESSOR_ARM_64)
         pixman_composite_over_8888_8888_asm_neon(w, h, (uint32_t *)destPixels, dbpl / 4, (uint32_t *)srcPixels, sbpl / 4);
+#else
+        for (int y=0; y<h; ++y) {
+            int x = 0;
+            for (; x < w-3; x += 4) {
+                if (src[x] | src[x+1] | src[x+2] | src[x+3]) {
+                    uint32x4_t src32 = vld1q_u32((uint32_t *)&src[x]);
+                    uint32x4_t dst32 = vld1q_u32((uint32_t *)&dst[x]);
+
+                    const uint8x16_t src8 = vreinterpretq_u8_u32(src32);
+                    const uint8x16_t dst8 = vreinterpretq_u8_u32(dst32);
+
+                    const uint8x8_t src8_low = vget_low_u8(src8);
+                    const uint8x8_t dst8_low = vget_low_u8(dst8);
+
+                    const uint8x8_t src8_high = vget_high_u8(src8);
+                    const uint8x8_t dst8_high = vget_high_u8(dst8);
+
+                    const uint16x8_t src16_low = vmovl_u8(src8_low);
+                    const uint16x8_t dst16_low = vmovl_u8(dst8_low);
+
+                    const uint16x8_t src16_high = vmovl_u8(src8_high);
+                    const uint16x8_t dst16_high = vmovl_u8(dst8_high);
+
+                    const uint16x8_t result16_low = qvsource_over_u16(src16_low, dst16_low, half, full);
+                    const uint16x8_t result16_high = qvsource_over_u16(src16_high, dst16_high, half, full);
+
+                    const uint32x2_t result32_low = vreinterpret_u32_u8(vmovn_u16(result16_low));
+                    const uint32x2_t result32_high = vreinterpret_u32_u8(vmovn_u16(result16_high));
+
+                    vst1q_u32((uint32_t *)&dst[x], vcombine_u32(result32_low, result32_high));
+                }
+            }
+            for (; x<w; ++x) {
+                uint s = src[x];
+                if (s >= 0xff000000)
+                    dst[x] = s;
+                else if (s != 0)
+                    dst[x] = s + BYTE_MUL(dst[x], qAlpha(~s));
+            }
+            dst = (quint32 *)(((uchar *) dst) + dbpl);
+            src = (const quint32 *)(((const uchar *) src) + sbpl);
+        }
+#endif
     } else if (const_alpha != 0) {
         const_alpha = (const_alpha * 255) >> 8;
         uint16x8_t const_alpha16 = vdupq_n_u16(const_alpha);
@@ -463,6 +530,7 @@ void qt_blend_rgb32_on_rgb32_neon(uchar *destPixels, int dbpl,
     }
 }
 
+#if !defined(Q_PROCESSOR_ARM_64)
 void qt_alphamapblit_quint16_neon(QRasterBuffer *rasterBuffer,
                                   int x, int y, const QRgba64 &color,
                                   const uchar *bitmap,
@@ -703,6 +771,7 @@ void QT_FASTCALL qt_destStoreRGB16_neon(QRasterBuffer *rasterBuffer, int x, int
             data[i + j] = dstBuffer[j];
     }
 }
+#endif
 
 void QT_FASTCALL comp_func_solid_SourceOver_neon(uint *destPixels, int length, uint color, uint const_alpha)
 {
@@ -754,16 +823,13 @@ void QT_FASTCALL comp_func_Plus_neon(uint *dst, const uint *src, int length, uin
         uint *const neonEnd = end - 3;
 
         while (dst < neonEnd) {
-            asm volatile (
+            uint8x16_t vs = vld1q_u8((const uint8_t*)src);
-                "vld2.8     { d0, d1 }, [%[SRC]] !\n\t"
+            const uint8x16_t vd = vld1q_u8((uint8_t*)dst);
-                "vld2.8     { d2, d3 }, [%[DST]]\n\t"
+            vs = vqaddq_u8(vs, vd);
-                "vqadd.u8 q0, q0, q1\n\t"
+            vst1q_u8((uint8_t*)dst, vs);
-                "vst2.8     { d0, d1 }, [%[DST]] !\n\t"
+            src += 4;
-                : [DST]"+r" (dst), [SRC]"+r" (src)
+            dst += 4;
-                :
+        };
-                : "memory", "d0", "d1", "d2", "d3", "q0", "q1"
-            );
-        }
 
         while (dst != end) {
             *dst = comp_func_Plus_one_pixel(*dst, *src);
@@ -802,6 +868,7 @@ void QT_FASTCALL comp_func_Plus_neon(uint *dst, const uint *src, int length, uin
     }
 }
 
+#if !defined(Q_PROCESSOR_ARM_64)
 static const int tileSize = 32;
 
 extern "C" void qt_rotate90_16_neon(quint16 *dst, const quint16 *src, int sstride, int dstride, int count);
@@ -945,6 +1012,7 @@ void qt_memrotate270_16_neon(const uchar *srcPixels, int w, int h,
         }
     }
 }
+#endif
 
 class QSimdNeon
 {