AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #192 from Cyan4973/dev

Browse Source 
updated xxhash
This commit is contained in:
Yann Collet 2016-05-29 00:41:36 +02:00
commit 04fa41b5ce
8 changed files with 535 additions and 406 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
programs/.clang_complete Normal file
View File

@ -0,0 +1,3 @@
-I../lib/common
-I../lib/legacy
-I./legacy

1
programs/.gitignore vendored
View File

@ -9,6 +9,7 @@ zbufftest
zbufftest32 zbufftest32
datagen datagen
paramgrill paramgrill
roundTripCrash
# Object files # Object files
*.o *.o

8
programs/Makefile
View File

@ -134,19 +134,21 @@ zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \
datagen.c xxhash.c zbufftest.c datagen.c xxhash.c zbufftest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
paramgrill : $(ZSTD_FILES) \ paramgrill : $(ZSTD_FILES) datagen.c xxhash.c paramgrill.c
datagen.c xxhash.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT) $(CC) $(FLAGS) $^ -lm -o $@$(EXT)
datagen : datagen.c datagencli.c datagen : datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
clean: clean:
@rm -f core *.o tmp* result* *.gcda dictionary *.zst \ @rm -f core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT)
@echo Cleaning completed @echo Cleaning completed

11
programs/fuzzer.c
View File

@ -42,8 +42,9 @@
#include <time.h> /* clock_t */ #include <time.h> /* clock_t */
#include "zstd_static.h" /* ZSTD_VERSION_STRING */ #include "zstd_static.h" /* ZSTD_VERSION_STRING */
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" /* XXH64 */
#include "mem.h" #include "mem.h"
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
/*-************************************ /*-************************************
@ -463,7 +464,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, totalCSize, totalGenSize; size_t cSize, dSize, totalCSize, totalGenSize;
U32 sampleSizeLog, nbChunks, n; U32 sampleSizeLog, nbChunks, n;
XXH64_CREATESTATE_STATIC(xxh64); XXH64_state_t xxhState;
U64 crcOrig; U64 crcOrig;
BYTE* sampleBuffer; BYTE* sampleBuffer;
const BYTE* dict; const BYTE* dict;
@ -614,7 +615,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx); { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); } CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
} }
XXH64_reset(xxh64, 0); XXH64_reset(&xxhState, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2; nbChunks = (FUZ_rand(&lseed) & 127) + 2;
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
@ -629,7 +630,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
cSize += compressResult; cSize += compressResult;
} }
XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize); XXH64_update(&xxhState, srcBuffer+sampleStart, sampleSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
totalTestSize += sampleSize; totalTestSize += sampleSize;
} }
@ -637,7 +638,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
cSize += flushResult; cSize += flushResult;
} }
crcOrig = XXH64_digest(xxh64); crcOrig = XXH64_digest(&xxhState);
/* streaming decompression test */ /* streaming decompression test */
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);

185
programs/roundTripCrash.c Normal file
View File

@ -0,0 +1,185 @@
/*
roundTripCrash
Copyright (C) Yann Collet 2013-2016
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd homepage : http://www.zstd.net
*/
/*
This program takes a file in input,
performs a zstd round-trip test (compression - decompress)
compares the result with original
and generates a crash (double free) on corruption detection.
*/
/*===========================================
* Dependencies
*==========================================*/
#include <stddef.h> /* size_t */
#include <stdlib.h> /* malloc, free, exit */
#include <stdio.h> /* fprintf */
#include <sys/types.h> /* stat */
#include <sys/stat.h> /* stat */
#include "zstd.h"
/** roundTripTest() :
* Compresses `srcBuff` into `compressedBuff`,
* then decompresses `compressedBuff` into `resultBuff`.
* Compression level used is derived from first content byte.
* @return : result of decompression, which should be == `srcSize`
* or an error code if either compression or decompression fails.
* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`
* for compression to be guaranteed to work */
static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
void* compressedBuff, size_t compressedBuffCapacity,
const void* srcBuff, size_t srcBuffSize)
{
static const int maxClevel = 19;
int const cLevel = (!srcBuffSize) ? 1 : (*(const unsigned char*)srcBuff) % maxClevel;
size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel);
if (ZSTD_isError(cSize)) {
fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize));
return cSize;
}
return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize);
}
static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
{
const char* ip1 = (const char*)buff1;
const char* ip2 = (const char*)buff2;
size_t pos;
for (pos=0; pos<buffSize; pos++)
if (ip1[pos]!=ip2[pos])
break;
return pos;
}
static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
{
size_t const cBuffSize = ZSTD_compressBound(srcBuffSize);
void* cBuff = malloc(cBuffSize);
void* rBuff = malloc(cBuffSize);
#define CRASH { free(cBuff); free(cBuff); } /* double free, to crash program */
if (!cBuff || !rBuff) {
fprintf(stderr, "not enough memory ! \n");
exit (1);
}
{ size_t const result = roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize);
if (ZSTD_isError(result)) {
fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result));
CRASH;
}
if (result != srcBuffSize) {
fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize);
CRASH;
}
if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) {
fprintf(stderr, "Silent decoding corruption !!!");
CRASH;
}
}
free(cBuff);
free(rBuff);
}
static size_t getFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
#endif
return (size_t)statbuf.st_size;
}
static int isDirectory(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
}
/** loadFile() :
* requirement : `buffer` size >= `fileSize` */
static void loadFile(void* buffer, const char* fileName, size_t fileSize)
{
FILE* const f = fopen(fileName, "rb");
if (isDirectory(fileName)) {
fprintf(stderr, "Ignoring %s directory \n", fileName);
exit(2);
}
if (f==NULL) {
fprintf(stderr, "Impossible to open %s \n", fileName);
exit(3);
}
{ size_t const readSize = fread(buffer, 1, fileSize, f);
if (readSize != fileSize) {
fprintf(stderr, "Error reading %s \n", fileName);
exit(5);
} }
fclose(f);
}
static void fileCheck(const char* fileName)
{
size_t const fileSize = getFileSize(fileName);
void* buffer = malloc(fileSize);
if (!buffer) {
fprintf(stderr, "not enough memory \n");
exit(4);
}
loadFile(buffer, fileName, fileSize);
roundTripCheck(buffer, fileSize);
free (buffer);
}
int main(int argCount, const char** argv) {
if (argCount < 2) {
fprintf(stderr, "Error : no argument : need input file \n");
exit(9);
}
fileCheck(argv[1]);
fprintf(stderr, "no pb detected\n");
return 0;
}

593
programs/xxhash.c
View File

@ -1,41 +1,42 @@
/* /*
xxHash - Fast Hash algorithm * xxHash - Fast Hash algorithm
Copyright (C) 2012-2016, Yann Collet * Copyright (C) 2012-2016, Yann Collet
*
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
*
Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are * modification, are permitted provided that the following conditions are
met: * met:
*
* Redistributions of source code must retain the above copyright * * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer. * notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above * * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer * copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the * in the documentation and/or other materials provided with the
distribution. * distribution.
*
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
You can contact the author at : * You can contact the author at :
- xxHash source repository : https://github.com/Cyan4973/xxHash * - xxHash homepage: http://www.xxhash.com
* - xxHash source repository : https://github.com/Cyan4973/xxHash
*/ */
/* ************************************* /* *************************************
* Tuning parameters * Tuning parameters
***************************************/ ***************************************/
/*!XXH_FORCE_MEMORY_ACCESS /*!XXH_FORCE_MEMORY_ACCESS :
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance. * The below switch allow to select different access method for improved performance.
@ -65,24 +66,47 @@ You can contact the author at :
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ /* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
/*!XXH_FORCE_NATIVE_FORMAT : /*!XXH_FORCE_NATIVE_FORMAT :
* By default, xxHash library provides endian-independent Hash values, based on little-endian convention. * By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
* Results are therefore identical for little-endian and big-endian CPU. * Results are therefore identical for little-endian and big-endian CPU.
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
* Should endian-independance be of no importance for your application, you may set the #define below to 1, * Should endian-independance be of no importance for your application, you may set the #define below to 1,
* to improve speed for Big-endian CPU. * to improve speed for Big-endian CPU.
* This option has no impact on Little_Endian CPU. * This option has no impact on Little_Endian CPU.
*/ */
#define XXH_FORCE_NATIVE_FORMAT 0 #ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */
# define XXH_FORCE_NATIVE_FORMAT 0
/*!XXH_USELESS_ALIGN_BRANCH :
* This is a minor performance trick, only useful with lots of very small keys.
* It means : don't check for aligned/unaligned input, because performance will be the same.
* It saves one initial branch per hash.
*/
#if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
# define XXH_USELESS_ALIGN_BRANCH 1
#endif #endif
/*!XXH_FORCE_ALIGN_CHECK :
* This is a minor performance trick, only useful with lots of very small keys.
* It means : check for aligned/unaligned input.
* The check costs one initial branch per hash; set to 0 when the input data
* is guaranteed to be aligned.
*/
#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
# define XXH_FORCE_ALIGN_CHECK 0
# else
# define XXH_FORCE_ALIGN_CHECK 1
# endif
#endif
/* *************************************
* Includes & Memory related functions
***************************************/
/* Modify the local functions below should you wish to use some other memory routines */
/* for malloc(), free() */
#include <stdlib.h>
static void* XXH_malloc(size_t s) { return malloc(s); }
static void XXH_free (void* p) { free(p); }
/* for memcpy() */
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h"
/* ************************************* /* *************************************
* Compiler Specific Options * Compiler Specific Options
@ -103,27 +127,12 @@ You can contact the author at :
#endif #endif
/* *************************************
* Includes & Memory related functions
***************************************/
/* Modify the local functions below should you wish to use some other memory routines */
/* for malloc(), free() */
#include <stdlib.h>
static void* XXH_malloc(size_t s) { return malloc(s); }
static void XXH_free (void* p) { free(p); }
/* for memcpy() */
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
#include "xxhash.h"
/* ************************************* /* *************************************
* Basic Types * Basic Types
***************************************/ ***************************************/
#ifndef MEM_MODULE #ifndef MEM_MODULE
# define MEM_MODULE # define MEM_MODULE
# if !defined (__VMS) && ( defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ ) # if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h> # include <stdint.h>
typedef uint8_t BYTE; typedef uint8_t BYTE;
typedef uint16_t U16; typedef uint16_t U16;
@ -250,6 +259,11 @@ FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
return XXH_readLE32_align(ptr, endian, XXH_unaligned); return XXH_readLE32_align(ptr, endian, XXH_unaligned);
} }
static U32 XXH_readBE32(const void* ptr)
{
return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
}
FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{ {
if (align==XXH_unaligned) if (align==XXH_unaligned)
@ -263,6 +277,11 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
return XXH_readLE64_align(ptr, endian, XXH_unaligned); return XXH_readLE64_align(ptr, endian, XXH_unaligned);
} }
static U64 XXH_readBE64(const void* ptr)
{
return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
}
/* ************************************* /* *************************************
* Macros * Macros
@ -273,17 +292,17 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
/* ************************************* /* *************************************
* Constants * Constants
***************************************/ ***************************************/
#define PRIME32_1 2654435761U static const U32 PRIME32_1 = 2654435761U;
#define PRIME32_2 2246822519U static const U32 PRIME32_2 = 2246822519U;
#define PRIME32_3 3266489917U static const U32 PRIME32_3 = 3266489917U;
#define PRIME32_4 668265263U static const U32 PRIME32_4 = 668265263U;
#define PRIME32_5 374761393U static const U32 PRIME32_5 = 374761393U;
#define PRIME64_1 11400714785074694791ULL static const U64 PRIME64_1 = 11400714785074694791ULL;
#define PRIME64_2 14029467366897019727ULL static const U64 PRIME64_2 = 14029467366897019727ULL;
#define PRIME64_3 1609587929392839161ULL static const U64 PRIME64_3 = 1609587929392839161ULL;
#define PRIME64_4 9650029242287828579ULL static const U64 PRIME64_4 = 9650029242287828579ULL;
#define PRIME64_5 2870177450012600261ULL static const U64 PRIME64_5 = 2870177450012600261ULL;
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
@ -291,6 +310,15 @@ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
/* *************************** /* ***************************
* Simple Hash Functions * Simple Hash Functions
*****************************/ *****************************/
static U32 XXH32_round(U32 seed, U32 input)
{
seed += input * PRIME32_2;
seed = XXH_rotl32(seed, 13);
seed *= PRIME32_1;
return seed;
}
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
{ {
const BYTE* p = (const BYTE*)input; const BYTE* p = (const BYTE*)input;
@ -299,60 +327,40 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH
#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) #define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER #ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (p==NULL) if (p==NULL) {
{
len=0; len=0;
bEnd=p=(const BYTE*)(size_t)16; bEnd=p=(const BYTE*)(size_t)16;
} }
#endif #endif
if (len>=16) if (len>=16) {
{
const BYTE* const limit = bEnd - 16; const BYTE* const limit = bEnd - 16;
U32 v1 = seed + PRIME32_1 + PRIME32_2; U32 v1 = seed + PRIME32_1 + PRIME32_2;
U32 v2 = seed + PRIME32_2; U32 v2 = seed + PRIME32_2;
U32 v3 = seed + 0; U32 v3 = seed + 0;
U32 v4 = seed - PRIME32_1; U32 v4 = seed - PRIME32_1;
do do {
{ v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
v1 += XXH_get32bits(p) * PRIME32_2; v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
v1 = XXH_rotl32(v1, 13); v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
v1 *= PRIME32_1; v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
p+=4; } while (p<=limit);
v2 += XXH_get32bits(p) * PRIME32_2;
v2 = XXH_rotl32(v2, 13);
v2 *= PRIME32_1;
p+=4;
v3 += XXH_get32bits(p) * PRIME32_2;
v3 = XXH_rotl32(v3, 13);
v3 *= PRIME32_1;
p+=4;
v4 += XXH_get32bits(p) * PRIME32_2;
v4 = XXH_rotl32(v4, 13);
v4 *= PRIME32_1;
p+=4;
}
while (p<=limit);
h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
} } else {
else
{
h32 = seed + PRIME32_5; h32 = seed + PRIME32_5;
} }
h32 += (U32) len; h32 += (U32) len;
while (p+4<=bEnd) while (p+4<=bEnd) {
{
h32 += XXH_get32bits(p) * PRIME32_3; h32 += XXH_get32bits(p) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
p+=4; p+=4;
} }
while (p<bEnd) while (p<bEnd) {
{
h32 += (*p) * PRIME32_5; h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
p++; p++;
@ -372,22 +380,20 @@ XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int s
{ {
#if 0 #if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
XXH32_state_t state; XXH32_CREATESTATE_STATIC(state);
XXH32_reset(&state, seed); XXH32_reset(state, seed);
XXH32_update(&state, input, len); XXH32_update(state, input, len);
return XXH32_digest(&state); return XXH32_digest(state);
#else #else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USELESS_ALIGN_BRANCH) if (XXH_FORCE_ALIGN_CHECK) {
if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */ if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
} } }
# endif
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
@ -396,103 +402,77 @@ XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int s
#endif #endif
} }
static U64 XXH64_round(U64 acc, U64 input)
{
acc += input * PRIME64_2;
acc = XXH_rotl64(acc, 31);
acc *= PRIME64_1;
return acc;
}
static U64 XXH64_mergeRound(U64 acc, U64 val)
{
val = XXH64_round(0, val);
acc ^= val;
acc = acc * PRIME64_1 + PRIME64_4;
return acc;
}
FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
{ {
const BYTE* p = (const BYTE*)input; const BYTE* p = (const BYTE*)input;
const BYTE* bEnd = p + len; const BYTE* const bEnd = p + len;
U64 h64; U64 h64;
#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) #define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER #ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (p==NULL) if (p==NULL) {
{
len=0; len=0;
bEnd=p=(const BYTE*)(size_t)32; bEnd=p=(const BYTE*)(size_t)32;
} }
#endif #endif
if (len>=32) if (len>=32) {
{
const BYTE* const limit = bEnd - 32; const BYTE* const limit = bEnd - 32;
U64 v1 = seed + PRIME64_1 + PRIME64_2; U64 v1 = seed + PRIME64_1 + PRIME64_2;
U64 v2 = seed + PRIME64_2; U64 v2 = seed + PRIME64_2;
U64 v3 = seed + 0; U64 v3 = seed + 0;
U64 v4 = seed - PRIME64_1; U64 v4 = seed - PRIME64_1;
do do {
{ v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
v1 += XXH_get64bits(p) * PRIME64_2; v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
p+=8; v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
v1 = XXH_rotl64(v1, 31); v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
v1 *= PRIME64_1; } while (p<=limit);
v2 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
v3 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
v4 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
}
while (p<=limit);
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
h64 = XXH64_mergeRound(h64, v1);
h64 = XXH64_mergeRound(h64, v2);
h64 = XXH64_mergeRound(h64, v3);
h64 = XXH64_mergeRound(h64, v4);
v1 *= PRIME64_2; } else {
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
h64 ^= v1;
h64 = h64 * PRIME64_1 + PRIME64_4;
v2 *= PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
h64 ^= v2;
h64 = h64 * PRIME64_1 + PRIME64_4;
v3 *= PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
h64 ^= v3;
h64 = h64 * PRIME64_1 + PRIME64_4;
v4 *= PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
h64 ^= v4;
h64 = h64 * PRIME64_1 + PRIME64_4;
}
else
{
h64 = seed + PRIME64_5; h64 = seed + PRIME64_5;
} }
h64 += (U64) len; h64 += (U64) len;
while (p+8<=bEnd) while (p+8<=bEnd) {
{ U64 const k1 = XXH64_round(0, XXH_get64bits(p));
U64 k1 = XXH_get64bits(p);
k1 *= PRIME64_2;
k1 = XXH_rotl64(k1,31);
k1 *= PRIME64_1;
h64 ^= k1; h64 ^= k1;
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8; p+=8;
} }
if (p+4<=bEnd) if (p+4<=bEnd) {
{
h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4; p+=4;
} }
while (p<bEnd) while (p<bEnd) {
{
h64 ^= (*p) * PRIME64_5; h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1; h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++; p++;
@ -512,22 +492,20 @@ XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned
{ {
#if 0 #if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
XXH64_state_t state; XXH64_CREATESTATE_STATIC(state);
XXH64_reset(&state, seed); XXH64_reset(state, seed);
XXH64_update(&state, input, len); XXH64_update(state, input, len);
return XXH64_digest(&state); return XXH64_digest(state);
#else #else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USELESS_ALIGN_BRANCH) if (XXH_FORCE_ALIGN_CHECK) {
if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */ if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else else
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
} } }
# endif
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
@ -536,39 +514,13 @@ XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned
#endif #endif
} }
/* ************************************************** /* **************************************************
* Advanced Hash Functions * Advanced Hash Functions
****************************************************/ ****************************************************/
/*** Allocation ***/
struct XXH32_state_s
{
U64 total_len;
U32 seed;
U32 v1;
U32 v2;
U32 v3;
U32 v4;
U32 mem32[4]; /* defined as U32 for alignment */
U32 memsize;
}; /* typedef'd to XXH32_state_t within xxhash.h */
struct XXH64_state_s
{
U64 total_len;
U64 seed;
U64 v1;
U64 v2;
U64 v3;
U64 v4;
U64 mem64[4]; /* defined as U64 for alignment */
U32 memsize;
}; /* typedef'd to XXH64_state_t within xxhash.h */
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
{ {
XXH_STATIC_ASSERT(sizeof(XXH32_stateBody_t) >= sizeof(XXH32_state_t)); /* A compilation error here means XXH32_state_t is not large enough */
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
} }
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
@ -579,7 +531,6 @@ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
{ {
XXH_STATIC_ASSERT(sizeof(XXH64_stateBody_t) >= sizeof(XXH64_state_t)); /* A compilation error here means XXH64_state_t is not large enough */
return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
} }
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
@ -630,67 +581,37 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void
state->total_len += len; state->total_len += len;
if (state->memsize + len < 16) /* fill in tmp buffer */ if (state->memsize + len < 16) { /* fill in tmp buffer */
{
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
state->memsize += (U32)len; state->memsize += (U32)len;
return XXH_OK; return XXH_OK;
} }
if (state->memsize) /* some data left from previous update */ if (state->memsize) { /* some data left from previous update */
{
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
{ { const U32* p32 = state->mem32;
const U32* p32 = state->mem32; state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
state->v1 = XXH_rotl32(state->v1, 13); state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
state->v1 *= PRIME32_1; state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
p32++;
state->v2 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v2 = XXH_rotl32(state->v2, 13);
state->v2 *= PRIME32_1;
p32++;
state->v3 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v3 = XXH_rotl32(state->v3, 13);
state->v3 *= PRIME32_1;
p32++;
state->v4 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v4 = XXH_rotl32(state->v4, 13);
state->v4 *= PRIME32_1;
p32++;
} }
p += 16-state->memsize; p += 16-state->memsize;
state->memsize = 0; state->memsize = 0;
} }
if (p <= bEnd-16) if (p <= bEnd-16) {
{
const BYTE* const limit = bEnd - 16; const BYTE* const limit = bEnd - 16;
U32 v1 = state->v1; U32 v1 = state->v1;
U32 v2 = state->v2; U32 v2 = state->v2;
U32 v3 = state->v3; U32 v3 = state->v3;
U32 v4 = state->v4; U32 v4 = state->v4;
do do {
{ v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
v1 += XXH_readLE32(p, endian) * PRIME32_2; v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
v1 = XXH_rotl32(v1, 13); v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
v1 *= PRIME32_1; v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
p+=4; } while (p<=limit);
v2 += XXH_readLE32(p, endian) * PRIME32_2;
v2 = XXH_rotl32(v2, 13);
v2 *= PRIME32_1;
p+=4;
v3 += XXH_readLE32(p, endian) * PRIME32_2;
v3 = XXH_rotl32(v3, 13);
v3 *= PRIME32_1;
p+=4;
v4 += XXH_readLE32(p, endian) * PRIME32_2;
v4 = XXH_rotl32(v4, 13);
v4 *= PRIME32_1;
p+=4;
}
while (p<=limit);
state->v1 = v1; state->v1 = v1;
state->v2 = v2; state->v2 = v2;
@ -698,8 +619,7 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void
state->v4 = v4; state->v4 = v4;
} }
if (p < bEnd) if (p < bEnd) {
{
XXH_memcpy(state->mem32, p, bEnd-p); XXH_memcpy(state->mem32, p, bEnd-p);
state->memsize = (int)(bEnd-p); state->memsize = (int)(bEnd-p);
} }
@ -722,29 +642,24 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void*
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
{ {
const BYTE * p = (const BYTE*)state->mem32; const BYTE * p = (const BYTE*)state->mem32;
const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32; U32 h32;
if (state->total_len >= 16) if (state->total_len >= 16) {
{
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
} } else {
else
{
h32 = state->seed + PRIME32_5; h32 = state->seed + PRIME32_5;
} }
h32 += (U32) state->total_len; h32 += (U32) state->total_len;
while (p+4<=bEnd) while (p+4<=bEnd) {
{
h32 += XXH_readLE32(p, endian) * PRIME32_3; h32 += XXH_readLE32(p, endian) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4; h32 = XXH_rotl32(h32, 17) * PRIME32_4;
p+=4; p+=4;
} }
while (p<bEnd) while (p<bEnd) {
{
h32 += (*p) * PRIME32_5; h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1; h32 = XXH_rotl32(h32, 11) * PRIME32_1;
p++; p++;
@ -771,6 +686,9 @@ XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
} }
/* **** XXH64 **** */
FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
{ {
const BYTE* p = (const BYTE*)input; const BYTE* p = (const BYTE*)input;
@ -782,67 +700,35 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void
state->total_len += len; state->total_len += len;
if (state->memsize + len < 32) /* fill in tmp buffer */ if (state->memsize + len < 32) { /* fill in tmp buffer */
{
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
state->memsize += (U32)len; state->memsize += (U32)len;
return XXH_OK; return XXH_OK;
} }
if (state->memsize) /* some data left from previous update */ if (state->memsize) { /* tmp buffer is full */
{
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
{ state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
const U64* p64 = state->mem64; state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
state->v1 = XXH_rotl64(state->v1, 31); state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
state->v1 *= PRIME64_1;
p64++;
state->v2 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v2 = XXH_rotl64(state->v2, 31);
state->v2 *= PRIME64_1;
p64++;
state->v3 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v3 = XXH_rotl64(state->v3, 31);
state->v3 *= PRIME64_1;
p64++;
state->v4 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v4 = XXH_rotl64(state->v4, 31);
state->v4 *= PRIME64_1;
p64++;
}
p += 32-state->memsize; p += 32-state->memsize;
state->memsize = 0; state->memsize = 0;
} }
if (p+32 <= bEnd) if (p+32 <= bEnd) {
{
const BYTE* const limit = bEnd - 32; const BYTE* const limit = bEnd - 32;
U64 v1 = state->v1; U64 v1 = state->v1;
U64 v2 = state->v2; U64 v2 = state->v2;
U64 v3 = state->v3; U64 v3 = state->v3;
U64 v4 = state->v4; U64 v4 = state->v4;
do do {
{ v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
v1 += XXH_readLE64(p, endian) * PRIME64_2; v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
v1 = XXH_rotl64(v1, 31); v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
v1 *= PRIME64_1; v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
p+=8; } while (p<=limit);
v2 += XXH_readLE64(p, endian) * PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
p+=8;
v3 += XXH_readLE64(p, endian) * PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
p+=8;
v4 += XXH_readLE64(p, endian) * PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
p+=8;
}
while (p<=limit);
state->v1 = v1; state->v1 = v1;
state->v2 = v2; state->v2 = v2;
@ -850,8 +736,7 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void
state->v4 = v4; state->v4 = v4;
} }
if (p < bEnd) if (p < bEnd) {
{
XXH_memcpy(state->mem64, p, bEnd-p); XXH_memcpy(state->mem64, p, bEnd-p);
state->memsize = (int)(bEnd-p); state->memsize = (int)(bEnd-p);
} }
@ -874,69 +759,40 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void*
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
{ {
const BYTE * p = (const BYTE*)state->mem64; const BYTE * p = (const BYTE*)state->mem64;
const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64; U64 h64;
if (state->total_len >= 32) if (state->total_len >= 32) {
{ U64 const v1 = state->v1;
U64 v1 = state->v1; U64 const v2 = state->v2;
U64 v2 = state->v2; U64 const v3 = state->v3;
U64 v3 = state->v3; U64 const v4 = state->v4;
U64 v4 = state->v4;
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
h64 = XXH64_mergeRound(h64, v1);
v1 *= PRIME64_2; h64 = XXH64_mergeRound(h64, v2);
v1 = XXH_rotl64(v1, 31); h64 = XXH64_mergeRound(h64, v3);
v1 *= PRIME64_1; h64 = XXH64_mergeRound(h64, v4);
h64 ^= v1; } else {
h64 = h64*PRIME64_1 + PRIME64_4;
v2 *= PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
h64 ^= v2;
h64 = h64*PRIME64_1 + PRIME64_4;
v3 *= PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
h64 ^= v3;
h64 = h64*PRIME64_1 + PRIME64_4;
v4 *= PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
h64 ^= v4;
h64 = h64*PRIME64_1 + PRIME64_4;
}
else
{
h64 = state->seed + PRIME64_5; h64 = state->seed + PRIME64_5;
} }
h64 += (U64) state->total_len; h64 += (U64) state->total_len;
while (p+8<=bEnd) while (p+8<=bEnd) {
{ U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
U64 k1 = XXH_readLE64(p, endian);
k1 *= PRIME64_2;
k1 = XXH_rotl64(k1,31);
k1 *= PRIME64_1;
h64 ^= k1; h64 ^= k1;
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8; p+=8;
} }
if (p+4<=bEnd) if (p+4<=bEnd) {
{
h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4; p+=4;
} }
while (p<bEnd) while (p<bEnd) {
{
h64 ^= (*p) * PRIME64_5; h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1; h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++; p++;
@ -963,3 +819,36 @@ XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
} }
/* **************************
* Canonical representation
****************************/
/*! Default XXH result types are basic unsigned 32 and 64 bits.
* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
*/
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
{
XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
memcpy(dst, &hash, sizeof(*dst));
}
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
{
XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
memcpy(dst, &hash, sizeof(*dst));
}
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
{
return XXH_readBE32(src);
}
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
{
return XXH_readBE64(src);
}

93
programs/xxhash.h
View File

@ -64,7 +64,8 @@ XXH64 13.8 GB/s 1.9 GB/s
XXH32 6.8 GB/s 6.0 GB/s XXH32 6.8 GB/s 6.0 GB/s
*/ */
#pragma once #ifndef XXHASH_H_5627135585666179
#define XXHASH_H_5627135585666179 1
#if defined (__cplusplus) #if defined (__cplusplus)
extern "C" { extern "C" {
@ -138,7 +139,7 @@ regular symbol name will be automatically translated by this header.
* Version * Version
***************************************/ ***************************************/
#define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MAJOR 0
#define XXH_VERSION_MINOR 5 #define XXH_VERSION_MINOR 6
#define XXH_VERSION_RELEASE 0 #define XXH_VERSION_RELEASE 0
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
XXH_PUBLIC_API unsigned XXH_versionNumber (void); XXH_PUBLIC_API unsigned XXH_versionNumber (void);
@ -147,9 +148,11 @@ XXH_PUBLIC_API unsigned XXH_versionNumber (void);
/* **************************** /* ****************************
* Simple Hash Functions * Simple Hash Functions
******************************/ ******************************/
typedef unsigned int XXH32_hash_t;
typedef unsigned long long XXH64_hash_t;
XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t length, unsigned int seed); XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
/*! /*!
XXH32() : XXH32() :
@ -165,23 +168,13 @@ XXH64() :
/* **************************** /* ****************************
* Advanced Hash Functions * Streaming Hash Functions
******************************/ ******************************/
typedef struct XXH32_state_s XXH32_state_t; /* incomplete */ typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */
typedef struct XXH64_state_s XXH64_state_t; /* incomplete */ typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
/*! Dynamic allocation of states
/*!Static allocation Compatible with dynamic libraries */
For static linking only, do not use in the context of DLL ! */
typedef struct { long long ll[ 6]; } XXH32_stateBody_t;
typedef struct { long long ll[11]; } XXH64_stateBody_t;
#define XXH32_CREATESTATE_STATIC(name) XXH32_stateBody_t name##xxhbody; void* name##xxhvoid = &(name##xxhbody); XXH32_state_t* name = (XXH32_state_t*)(name##xxhvoid) /* no final ; */
#define XXH64_CREATESTATE_STATIC(name) XXH64_stateBody_t name##xxhbody; void* name##xxhvoid = &(name##xxhbody); XXH64_state_t* name = (XXH64_state_t*)(name##xxhvoid) /* no final ; */
/*!Dynamic allocation
To be preferred in the context of DLL */
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
@ -194,11 +187,11 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* statePtr); XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* statePtr); XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
/*! /*!
These functions generate the xxHash of an input provided in multiple segments, These functions generate the xxHash of an input provided in multiple segments,
@ -213,14 +206,68 @@ Obviously, input must be valid, hence allocated and read accessible.
The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
Finally, a hash value can be produced anytime, by using XXHnn_digest(). Finally, a hash value can be produced anytime, by using XXHnn_digest().
This function returns the nn-bits hash. This function returns the nn-bits hash as an int or long long.
It's nonetheless possible to continue inserting input into the hash state
It's still possible to continue inserting input into the hash state after a digest,
and later on generate some new hashes, by calling again XXHnn_digest(). and later on generate some new hashes, by calling again XXHnn_digest().
When done, free XXH state space if it was allocated dynamically. When done, free XXH state space if it was allocated dynamically.
*/ */
/* **************************
* Canonical representation
****************************/
typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
/*! Default result type for XXH functions are primitive unsigned 32 and 64 bits.
* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
*/
#ifdef XXH_STATIC_LINKING_ONLY
/* This part contains definition which shall only be used with static linking.
The prototypes / types defined here are not guaranteed to remain stable.
They could change in a future version, becoming incompatible with a different version of the library */
struct XXH32_state_s {
unsigned long long total_len;
unsigned seed;
unsigned v1;
unsigned v2;
unsigned v3;
unsigned v4;
unsigned mem32[4]; /* buffer defined as U32 for alignment */
unsigned memsize;
}; /* typedef'd to XXH32_state_t */
struct XXH64_state_s {
unsigned long long total_len;
unsigned long long seed;
unsigned long long v1;
unsigned long long v2;
unsigned long long v3;
unsigned long long v4;
unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
unsigned memsize;
}; /* typedef'd to XXH64_state_t */
#endif
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif
#endif /* XXHASH_H_5627135585666179 */

9
programs/zbufftest.c
View File

@ -44,6 +44,7 @@
#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */ #include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */
#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */ #include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */ #include "xxhash.h" /* XXH64 */
@ -322,7 +323,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
size_t cSize, totalTestSize, totalCSize, totalGenSize; size_t cSize, totalTestSize, totalCSize, totalGenSize;
size_t errorCode; size_t errorCode;
U32 n, nbChunks; U32 n, nbChunks;
XXH64_CREATESTATE_STATIC(xxh64); XXH64_state_t xxhState;
U64 crcOrig; U64 crcOrig;
/* init */ /* init */
@ -366,7 +367,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
} } } }
/* multi-segments compression test */ /* multi-segments compression test */
XXH64_reset(xxh64, 0); XXH64_reset(&xxhState, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2; nbChunks = (FUZ_rand(&lseed) & 127) + 2;
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) { for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
/* compress random chunk into random size dst buffer */ /* compress random chunk into random size dst buffer */
@ -378,7 +379,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize); size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize);
CHECK (ZBUFF_isError(compressionError), "compression error : %s", ZBUFF_getErrorName(compressionError)); CHECK (ZBUFF_isError(compressionError), "compression error : %s", ZBUFF_getErrorName(compressionError));
XXH64_update(xxh64, srcBuffer+srcStart, readChunkSize); XXH64_update(&xxhState, srcBuffer+srcStart, readChunkSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize); memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize);
cSize += dstBuffSize; cSize += dstBuffSize;
totalTestSize += readChunkSize; totalTestSize += readChunkSize;
@ -399,7 +400,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError)); CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
cSize += dstBuffSize; cSize += dstBuffSize;
} }
crcOrig = XXH64_digest(xxh64); crcOrig = XXH64_digest(&xxhState);
/* multi - fragments decompression test */ /* multi - fragments decompression test */
ZBUFF_decompressInitDictionary(zd, dict, dictSize); ZBUFF_decompressInitDictionary(zd, dict, dictSize);