From c4792cdfa917911bea91ebf245e02a1a4b9038c6 Mon Sep 17 00:00:00 2001 From: remittor Date: Tue, 6 Oct 2020 17:16:43 +0300 Subject: [PATCH 1/3] Fix: The "inline" specifier do not use for LZ4_wildCopy8 and LZ4_wildCopy32 This problem was reproduced on MSVC 2015 (32-bit). Both functions were called using the operator "call". --- lib/lz4.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lz4.c b/lib/lz4.c index 0290ea2..1d08e10 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -162,7 +162,7 @@ # define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE #else # define LZ4_FORCE_O2_GCC_PPC64LE -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static +# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE LZ4_FORCE_INLINE #endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) From 749bd91a06be385b6485bfc7fe94e8a86b3ed13a Mon Sep 17 00:00:00 2001 From: remittor Date: Wed, 7 Oct 2020 09:51:08 +0300 Subject: [PATCH 2/3] Replace define LZ4_FORCE_O2_INLINE_GCC_PPC64LE to LZ4_FORCE_INLINE There is no reason to separate these two definitions! --- lib/lz4.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 1d08e10..0f53857 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -143,7 +143,7 @@ # endif /* _MSC_VER */ #endif /* LZ4_FORCE_INLINE */ -/* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, * together with a simple 8-byte copy loop as a fall-back path. * However, this optimization hurts the decompression speed by >30%, @@ -158,11 +158,11 @@ * of LZ4_wildCopy8 does not affect the compression speed. */ #if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) -# define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) #else -# define LZ4_FORCE_O2_GCC_PPC64LE -# define LZ4_FORCE_O2_INLINE_GCC_PPC64LE LZ4_FORCE_INLINE +# define LZ4_FORCE_O2 #endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) @@ -381,7 +381,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -LZ4_FORCE_O2_INLINE_GCC_PPC64LE +LZ4_FORCE_INLINE void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; @@ -410,7 +410,7 @@ static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; #if LZ4_FAST_DEC_LOOP -LZ4_FORCE_O2_INLINE_GCC_PPC64LE void +LZ4_FORCE_INLINE void LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { if (offset < 8) { @@ -434,7 +434,7 @@ LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, con /* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd * this version copies two times 16 bytes (instead of one time 32 bytes) * because it must be compatible with offsets >= 16. */ -LZ4_FORCE_O2_INLINE_GCC_PPC64LE void +LZ4_FORCE_INLINE void LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; @@ -447,7 +447,7 @@ LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) /* LZ4_memcpy_using_offset() presumes : * - dstEnd >= dstPtr + MINMATCH * - there is at least 8 bytes available to write after dstEnd */ -LZ4_FORCE_O2_INLINE_GCC_PPC64LE void +LZ4_FORCE_INLINE void LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) { BYTE v[8]; @@ -2145,7 +2145,7 @@ LZ4_decompress_generic( /*===== Instantiate the API decoding functions. =====*/ -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, @@ -2153,7 +2153,7 @@ int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int (BYTE*)dest, NULL, 0); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) { dstCapacity = MIN(targetOutputSize, dstCapacity); @@ -2162,7 +2162,7 @@ int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, noDict, (BYTE*)dst, NULL, 0); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, @@ -2172,7 +2172,7 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) /*===== Instantiate a few more decoding cases, used more than once. =====*/ -LZ4_FORCE_O2_GCC_PPC64LE /* Exported, an obsolete API function. */ +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, @@ -2188,7 +2188,7 @@ int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int origin return LZ4_decompress_fast(source, dest, originalSize); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, size_t prefixSize) { @@ -2197,7 +2197,7 @@ static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, i (BYTE*)dest-prefixSize, NULL, 0); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const void* dictStart, size_t dictSize) @@ -2207,7 +2207,7 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, const void* dictStart, size_t dictSize) { @@ -2296,7 +2296,7 @@ int LZ4_decoderRingBufferSize(int maxBlockSize) If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; @@ -2336,7 +2336,7 @@ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch return result; } -LZ4_FORCE_O2_GCC_PPC64LE +LZ4_FORCE_O2 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; From 0e3933edd435c54cc2e21e38f5d4ba7bf644a24e Mon Sep 17 00:00:00 2001 From: remittor Date: Wed, 7 Oct 2020 09:52:40 +0300 Subject: [PATCH 3/3] Replace "static" to "LZ4_FORCE_INLINE" for small functions The "static" specifier does not guarantee that the function will be inlined. --- lib/lz4.c | 36 ++++++++++++++++++------------------ lib/lz4hc.c | 4 ++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/lz4.c b/lib/lz4.c index 0f53857..6adfafd 100644 --- a/lib/lz4.c +++ b/lib/lz4.c @@ -299,7 +299,7 @@ typedef enum { #define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) #endif -static unsigned LZ4_isLittleEndian(void) +LZ4_FORCE_INLINE unsigned LZ4_isLittleEndian(void) { const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; @@ -309,12 +309,12 @@ static unsigned LZ4_isLittleEndian(void) #if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) /* lie to the compiler about data alignment; use with caution */ -static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } -static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } -static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } +LZ4_FORCE_INLINE U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +LZ4_FORCE_INLINE U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +LZ4_FORCE_INLINE reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } -static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +LZ4_FORCE_INLINE void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +LZ4_FORCE_INLINE void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } #elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) @@ -322,36 +322,36 @@ static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } /* currently only defined for gcc and icc */ typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; -static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } +LZ4_FORCE_INLINE U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +LZ4_FORCE_INLINE U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +LZ4_FORCE_INLINE reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } -static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +LZ4_FORCE_INLINE void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +LZ4_FORCE_INLINE void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } #else /* safe and portable access using memcpy() */ -static U16 LZ4_read16(const void* memPtr) +LZ4_FORCE_INLINE U16 LZ4_read16(const void* memPtr) { U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } -static U32 LZ4_read32(const void* memPtr) +LZ4_FORCE_INLINE U32 LZ4_read32(const void* memPtr) { U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } -static reg_t LZ4_read_ARCH(const void* memPtr) +LZ4_FORCE_INLINE reg_t LZ4_read_ARCH(const void* memPtr) { reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } -static void LZ4_write16(void* memPtr, U16 value) +LZ4_FORCE_INLINE void LZ4_write16(void* memPtr, U16 value) { LZ4_memcpy(memPtr, &value, sizeof(value)); } -static void LZ4_write32(void* memPtr, U32 value) +LZ4_FORCE_INLINE void LZ4_write32(void* memPtr, U32 value) { LZ4_memcpy(memPtr, &value, sizeof(value)); } @@ -359,7 +359,7 @@ static void LZ4_write32(void* memPtr, U32 value) #endif /* LZ4_FORCE_MEMORY_ACCESS */ -static U16 LZ4_readLE16(const void* memPtr) +LZ4_FORCE_INLINE U16 LZ4_readLE16(const void* memPtr) { if (LZ4_isLittleEndian()) { return LZ4_read16(memPtr); @@ -369,7 +369,7 @@ static U16 LZ4_readLE16(const void* memPtr) } } -static void LZ4_writeLE16(void* memPtr, U16 value) +LZ4_FORCE_INLINE void LZ4_writeLE16(void* memPtr, U16 value) { if (LZ4_isLittleEndian()) { LZ4_write16(memPtr, value); diff --git a/lib/lz4hc.c b/lib/lz4hc.c index cd802d8..8e3bd70 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -158,7 +158,7 @@ int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, #endif -static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) +LZ4_FORCE_INLINE U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) { size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; if (bitsToRotate == 0) @@ -223,7 +223,7 @@ LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) * 4 byte MINMATCH would overflow. * @returns true if the match index is okay. */ -static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) +LZ4_FORCE_INLINE int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) { return ((U32)((dictLimit - 1) - matchIndex) >= 3); }