Merge pull request #61 from Cyan4973/dev

Dev

.gitattributes

@@ -8,3 +8,8 @@
 # Denote files that should not be modified.
 *.odt binary
 *.png binary
+# Visual Studio
+*.sln binary
+*.suo binary
+*.vcxproj* binary
+

.travis.yml

@@ -3,6 +3,8 @@ compiler: gcc
 script: make test-travis
 before_install:
   - sudo apt-get update  -qq
+  - sudo apt-get install -qq clang
+  - sudo apt-get install -qq g++-multilib
   - sudo apt-get install -qq gcc-multilib
   - sudo apt-get install -qq valgrind
 
@@ -10,6 +12,9 @@ env:
   - LZ4_TRAVIS_CI_ENV=travis-install
   - LZ4_TRAVIS_CI_ENV=streaming-examples
   - LZ4_TRAVIS_CI_ENV=cmake
+  - LZ4_TRAVIS_CI_ENV=gpptest
+  - LZ4_TRAVIS_CI_ENV=clangtest
+  - LZ4_TRAVIS_CI_ENV=staticAnalyze
   - LZ4_TRAVIS_CI_ENV=dist
   - LZ4_TRAVIS_CI_ENV=test-lz4
   - LZ4_TRAVIS_CI_ENV=test-lz4c

Makefile

@@ -26,16 +26,16 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 # 
 # You can contact the author at :
-#  - LZ4 source repository : http://code.google.com/p/lz4/
+#  - LZ4 source repository : https://github.com/Cyan4973/lz4
 #  - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
 # ################################################################
 
 # Version number
-export VERSION=126
+export VERSION=128
 export RELEASE=r$(VERSION)
 
 DESTDIR?=
-PREFIX ?= /usr
+PREFIX ?= /usr/local
 
 LIBDIR ?= $(PREFIX)/lib
 INCLUDEDIR=$(PREFIX)/include
@@ -50,7 +50,7 @@ TEXT =  $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \
 	Makefile lz4_block_format.txt LZ4_Frame_Format.html NEWS README.md \
 	cmake_unofficial/CMakeLists.txt \
 	$(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \
-	$(PRGDIR)/datagen.c $(PRGDIR)/fuzzer.c \
+	$(PRGDIR)/datagen.c $(PRGDIR)/datagen.h $(PRGDIR)/datagencli.c $(PRGDIR)/fuzzer.c \
 	$(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \
 	$(PRGDIR)/bench.c $(PRGDIR)/bench.h \
 	$(PRGDIR)/lz4.1 $(PRGDIR)/lz4c.1 $(PRGDIR)/lz4cat.1 \
@@ -82,7 +82,7 @@ clean:
 	@rm -f $(DISTRIBNAME) *.sha1
 	@cd $(PRGDIR); $(MAKE) clean
 	@cd $(LZ4DIR); $(MAKE) clean
-	@cd examples; $(MAKE) clean
+	@cd examples;  $(MAKE) clean
 	@echo Cleaning completed
 
 
@@ -127,6 +127,15 @@ test-travis: $(TRAVIS_TARGET)
 cmake:
 	@cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE)
 
+gpptest: clean
+	export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align"; $(MAKE) -e all
+
+clangtest: clean
+	export CC=clang; $(MAKE) all
+
+staticAnalyze: clean
+	export CFLAGS=-g; scan-build -v $(MAKE) all
+
 streaming-examples:
 	cd examples; $(MAKE) -e test
 

NEWS

@@ -1,5 +1,24 @@
+r128:
+New   : lz4cli sparse file support
+New   : command -m, to compress multiple files in a single command
+Fixed : Restored lz4hc compression ratio (was slightly lower since r124)
+New   : lz4 cli supports long commands
+New   : lz4frame & lz4cli frame content size support
+New   : lz4frame supports skippable frames
+Changed:Default "make install" directory is /usr/local
+New   : lz4 cli supports "pass-through" mode
+New   : g++ compatibility tests
+New   : datagen can generate sparse files
+New   : scan-build tests
+Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48)
+Added : Visual project directory
+Updated:Man page & Specification
+
+r127:
+N/A   : added a file on SVN
+
 r126:
-New : lz4frame API is now integrated into liblz4
+New   : lz4frame API is now integrated into liblz4
 Fixed : GCC 4.9 bug on highest performance settings, reported by Greg Slazinski
 Fixed : bug within LZ4 HC streaming mode, reported by James Boyle
 Fixed : older compiler don't like nameless unions, reported by Cheyi Lin

README.md

@@ -9,13 +9,13 @@ A high compression derivative, called LZ4_HC, is also provided. It trades CPU ti
 |master      | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=master)](https://travis-ci.org/Cyan4973/lz4) |
 |dev         | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=dev)](https://travis-ci.org/Cyan4973/lz4) |
 
-This is an official mirror of LZ4 project, [hosted on Google Code](http://code.google.com/p/lz4/).
-The intention is to offer github's capabilities to lz4 users, such as cloning, branch, pull requests or source download.
 
-The "master" branch will reflect, the status of lz4 at its official homepage.
-The "dev" branch is the one where all contributions will be merged. If you plan to propose a patch, please commit into the "dev" branch. Direct commit to "master" are not permitted.
-Feature branches will also exist, typically to introduce new requirements, and be temporarily available for testing before merge into "dev" branch.
+> **Branch Policy:**
 
+> - The "master" branch is considered stable, at all times.
+> - The "dev" branch is the one where all contributions must be merged before being promoted to master.
+>  - If you plan to propose a patch, please commit into the "dev" branch. Direct commit to "master" are not permitted.
+> - Feature branches can also exist, for dedicated testing of larger modifications before merge into "dev" branch.
 
 Benchmarks
 -------------------------

cmake_unofficial/CMakeLists.txt

@@ -2,7 +2,7 @@ PROJECT(LZ4 C)
 set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 compression library")
 set(CPACK_PACKAGE_VERSION_MAJOR 1)
 set(CPACK_PACKAGE_VERSION_MINOR 5)
-set(CPACK_PACKAGE_VERSION_PATCH r126)
+set(CPACK_PACKAGE_VERSION_PATCH r128)
 set(VERSION_STRING	" \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ")
 include(CPack)
 
@@ -15,17 +15,15 @@ IF( ${SIZEOF_VOID_P} STREQUAL  "8" )
 ENDIF()
 
 option(BUILD_TOOLS "Build the command line tools" ON)
-option(BUILD_LIBS  "Build the libraries in addition to the tools" OFF)
+option(BUILD_LIBS  "Build the libraries in addition to the tools" ON)
 
 if(UNIX AND BUILD_LIBS)
-    if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
-        add_definitions(-fPIC)
-    endif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
+    add_definitions(-fPIC)
 endif()
 
 set(LZ4_DIR ../lib/)
 set(PRG_DIR ../programs/)
-set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h)
+set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c)
 set(LZ4_SRCS ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c)
 
 if(BUILD_TOOLS AND NOT BUILD_LIBS)
@@ -34,10 +32,7 @@ endif()
 
 if(BUILD_TOOLS)
     add_executable(lz4 ${LZ4_SRCS})
-    set_target_properties(lz4 PROPERTIES COMPILE_DEFINITIONS DISABLE_LZ4C_LEGACY_OPTIONS)
     install(TARGETS lz4 RUNTIME DESTINATION "bin/")
-    add_executable(lz4c ${LZ4_SRCS})
-    install(TARGETS lz4c RUNTIME DESTINATION "bin/")
 endif()
 
 if(BUILD_LIBS)    
@@ -61,7 +56,6 @@ if(BUILD_LIBS)
     
     if(BUILD_TOOLS)
         target_link_libraries(lz4 liblz4)
-        target_link_libraries(lz4c liblz4)
     endif()
 endif()
 
@@ -69,14 +63,17 @@ endif()
 #warnings
 
 ADD_DEFINITIONS("-Wall")
+if(CMAKE_COMPILER_IS_GNUCXX)
 ADD_DEFINITIONS("-Wextra")
 ADD_DEFINITIONS("-Wundef")
 ADD_DEFINITIONS("-Wshadow")
 ADD_DEFINITIONS("-Wcast-align")
 ADD_DEFINITIONS("-Wstrict-prototypes")
+endif(CMAKE_COMPILER_IS_GNUCXX)
 ADD_DEFINITIONS("-std=c99")
 ADD_DEFINITIONS("-DLZ4_VERSION=\"${CPACK_PACKAGE_VERSION_PATCH}\"")
 INCLUDE_DIRECTORIES (${LZ4_DIR})
 
 
 
+

lib/Makefile

@@ -26,20 +26,19 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 # 
 # You can contact the author at :
-#  - LZ4 source repository : http://code.google.com/p/lz4/
-#  - LZ4 source mirror : https://github.com/Cyan4973/lz4
+#  - LZ4 source repository : https://github.com/Cyan4973/lz4
 #  - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
 # ################################################################
 
 # Version numbers
-VERSION ?= 126
+VERSION ?= 128
 LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
 LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
 LIBVER_PATCH=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h`
 LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
 
 DESTDIR?=
-PREFIX ?= /usr
+PREFIX ?= /usr/local
 CFLAGS ?= -O3
 CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic
 
@@ -67,7 +66,7 @@ all: liblz4
 
 liblz4: lz4.c lz4hc.c lz4frame.c xxhash.c
 	@echo compiling static library
-	@$(CC) $(CPPFLAGS) $(CFLAGS) -c $^
+	@$(CC) $(CPPFLAGS) $(CFLAGS) -fPIC -c $^
 	@$(AR) rcs liblz4.a lz4.o lz4hc.o lz4frame.o xxhash.o
 	@echo compiling dynamic library $(LIBVER)
 	@$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
@@ -112,6 +111,7 @@ uninstall:
 	@[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a
 	@[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h
 	@[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
+	@[ -f $(DESTDIR)$(INCLUDEDIR)/lz4frame.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4frame.h
 	@echo lz4 libraries successfully uninstalled
 
 endif

lib/README.md

@@ -0,0 +1,19 @@
+LZ4 - Library Files
+================================
+
+This directory contains many files, but you don't necessarily need them all.
+
+If you want to integrate LZ4 compression/decompression into your program, you basically need to include "**lz4.c**" and "**lz4.h**" only.
+
+If you want more compression, at the cost of compression speed (but preserving decompression speed), you will also have to include "**lz4hc.c**" and "**lz4hc.h**". Note that lz4hc needs lz4 to work properly.
+
+Next level, if you want to produce files or data streams compatible with lz4 utility, you will have to use and include "**lz4frame.c**" and **lz4frame.h**". This library encapsulate lz4-compressed blocks into the official interoperable frame format. In order to work properly, lz4frame needs lz4 and lz4hc, and also "**xxhash.c**" and "**xxhash.h**", which provide the error detection algorithm.
+
+A more complex "lz4frame_static.h" is also provided, although its usage is not recommended. It contains definitions which are not guaranteed to remain stable within future versions. Use only if you don't plan to update your lz4 version.
+
+The other files are not source code. There are :
+
+ - LICENSE : contains the BSD license text
+ - Makefile : script to compile or install lz4 library (static or dynamic)
+ - liblz4.pc.in : for pkg-config (make install)
+

lib/lz4.c

@@ -1,6 +1,7 @@
 /*
    LZ4 - Fast LZ compression algorithm
    Copyright (C) 2011-2015, Yann Collet.
+
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
@@ -27,8 +28,7 @@
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
-   - LZ4 source repository : http://code.google.com/p/lz4
-   - LZ4 source mirror : https://github.com/Cyan4973/lz4
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
    - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
@@ -64,7 +64,7 @@
  *     Program will crash.
  * If uncommenting results in better performance (case 1)
  * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c)
- * An automatic detection macro will be added to match your case within future versions of the library.
+ * This way, an automatic detection macro can be added to match your case within later versions of the library.
  */
 /* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */
 
@@ -98,7 +98,7 @@
 
 
 /**************************************
-   Compiler Options
+*  Compiler Options
 **************************************/
 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
 /* "restrict" is a known keyword */
@@ -190,7 +190,7 @@ static U16 LZ4_readLE16(const void* memPtr)
         return *(U16*)memPtr;
     else
     {
-        const BYTE* p = memPtr;
+        const BYTE* p = (const BYTE*)memPtr;
         return (U16)((U16)p[0] + (p[1]<<8));
     }
 }
@@ -204,7 +204,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value)
     }
     else
     {
-        BYTE* p = memPtr;
+        BYTE* p = (BYTE*)memPtr;
         p[0] = (BYTE) value;
         p[1] = (BYTE)(value>>8);
     }
@@ -285,9 +285,9 @@ static void LZ4_copy8(void* dstPtr, const void* srcPtr)
 /* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */
 static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
 {
-    BYTE* d = dstPtr;
-    const BYTE* s = srcPtr;
-    BYTE* e = dstEnd;
+    BYTE* d = (BYTE*)dstPtr;
+    const BYTE* s = (const BYTE*)srcPtr;
+    BYTE* e = (BYTE*)dstEnd;
     do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e);
 }
 
@@ -316,14 +316,14 @@ static const int LZ4_minLength = (MFLIMIT+1);
 
 
 /**************************************
-   Common Utils
+*  Common Utils
 **************************************/
 #define LZ4_STATIC_ASSERT(c)    { enum { LZ4_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
 
-/********************************
-   Common functions
-********************************/
+/**************************************
+*  Common functions
+**************************************/
 static unsigned LZ4_NbCommonBytes (register size_t val)
 {
     if (LZ4_isLittleEndian())
@@ -412,7 +412,7 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi
 
 #ifndef LZ4_COMMONDEFS_ONLY
 /**************************************
-   Local Constants
+*  Local Constants
 **************************************/
 #define LZ4_HASHLOG   (LZ4_MEMORY_USAGE-2)
 #define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
@@ -423,14 +423,14 @@ static const U32 LZ4_skipTrigger = 6;  /* Increase this value ==> compression ru
 
 
 /**************************************
-   Local Utils
+*  Local Utils
 **************************************/
 int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
 int LZ4_compressBound(int isize)  { return LZ4_COMPRESSBOUND(isize); }
 
 
 /**************************************
-   Local Structures and types
+*  Local Structures and types
 **************************************/
 typedef struct {
     U32 hashTable[HASH_SIZE_U32];
@@ -453,10 +453,10 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive;
 
 
 /********************************
-   Compression functions
+*  Compression functions
 ********************************/
 
-static U32 LZ4_hashSequence(U32 sequence, tableType_t tableType)
+static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType)
 {
     if (tableType == byU16)
         return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
@@ -466,7 +466,7 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t tableType)
 
 static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); }
 
-static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
 {
     switch (tableType)
     {
@@ -502,7 +502,7 @@ static int LZ4_compress_generic(
                  int inputSize,
                  int maxOutputSize,
                  limitedOutput_directive outputLimited,
-                 tableType_t tableType,
+                 tableType_t const tableType,
                  dict_directive dict,
                  dictIssue_directive dictIssue)
 {
@@ -749,7 +749,7 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in
 
 
 /*****************************************
-   Experimental : Streaming functions
+*  Experimental : Streaming functions
 *****************************************/
 
 /*
@@ -930,9 +930,9 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
 
 
 
-/****************************
-   Decompression functions
-****************************/
+/*******************************
+*  Decompression functions
+*******************************/
 /*
  * This generic decompression function cover all use cases.
  * It shall be instantiated several times, using different sets of directives
@@ -1151,7 +1151,7 @@ typedef struct
  */
 LZ4_streamDecode_t* LZ4_createStreamDecode(void)
 {
-    LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U64), LZ4_STREAMDECODESIZE_U64);
+    LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t));
     return lz4s;
 }
 
@@ -1282,7 +1282,7 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres
 
 
 /***************************************************
-    Obsolete Functions
+*  Obsolete Functions
 ***************************************************/
 /*
 These function names are deprecated and should no longer be used.

lib/lz4.h

@@ -1,7 +1,8 @@
 /*
    LZ4 - Fast LZ compression algorithm
    Header File
-   Copyright (C) 2011-2014, Yann Collet.
+   Copyright (C) 2011-2015, Yann Collet.
+
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
@@ -28,7 +29,7 @@
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
-   - LZ4 source repository : http://code.google.com/p/lz4/
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
    - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 #pragma once
@@ -44,16 +45,16 @@ extern "C" {
 */
 
 /**************************************
-   Version
+*  Version
 **************************************/
 #define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
-#define LZ4_VERSION_MINOR    5    /* for new (non-breaking) interface capabilities */
+#define LZ4_VERSION_MINOR    6    /* for new (non-breaking) interface capabilities */
 #define LZ4_VERSION_RELEASE  0    /* for tweaks, bug-fixes, or development */
 #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
 int LZ4_versionNumber (void);
 
 /**************************************
-   Tuning parameter
+*  Tuning parameter
 **************************************/
 /*
  * LZ4_MEMORY_USAGE :
@@ -66,7 +67,7 @@ int LZ4_versionNumber (void);
 
 
 /**************************************
-   Simple Functions
+*  Simple Functions
 **************************************/
 
 int LZ4_compress        (const char* source, char* dest, int sourceSize);
@@ -95,7 +96,7 @@ LZ4_decompress_safe() :
 
 
 /**************************************
-   Advanced Functions
+*  Advanced Functions
 **************************************/
 #define LZ4_MAX_INPUT_SIZE        0x7E000000   /* 2 113 929 216 bytes */
 #define LZ4_COMPRESSBOUND(isize)  ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
@@ -169,7 +170,7 @@ int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedS
 
 
 /***********************************************
-   Streaming Compression Functions
+*  Streaming Compression Functions
 ***********************************************/
 
 #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
@@ -211,6 +212,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictS
  * LZ4_compress_continue
  * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio
  * Previous data blocks are assumed to still be present at their previous location.
+ * dest buffer must be already allocated, and sized to at least LZ4_compressBound(inputSize)
  */
 int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
 
@@ -227,14 +229,13 @@ int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char
  * save it into a safer place (char* safeBuffer)
  * Note : you don't need to call LZ4_loadDict() afterwards,
  *        dictionary is immediately usable, you can therefore call again LZ4_compress_continue()
- * Return : dictionary size in bytes, or 0 if error
- * Note : any dictSize > 64 KB will be interpreted as 64KB.
+ * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
  */
 int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize);
 
 
 /************************************************
-   Streaming Decompression Functions
+*  Streaming Decompression Functions
 ************************************************/
 
 #define LZ4_STREAMDECODESIZE_U64  4
@@ -285,7 +286,7 @@ int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalS
 
 
 /**************************************
-   Obsolete Functions
+*  Obsolete Functions
 **************************************/
 /*
 Obsolete decompression functions

lib/lz4frame.c

@@ -1,6 +1,7 @@
 /*
 LZ4 auto-framing library
-Copyright (C) 2011-2014, Yann Collet.
+Copyright (C) 2011-2015, Yann Collet.
+
 BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
 Redistribution and use in source and binary forms, with or without
@@ -27,13 +28,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 You can contact the author at :
-- LZ4 source repository : http://code.google.com/p/lz4/
+- LZ4 source repository : https://github.com/Cyan4973/lz4
 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
 /* LZ4F is a stand-alone API to create LZ4-compressed Frames
-* fully conformant to specification v1.4.1.
-* All related operations, including memory management, are handled by the library.
+*  in full conformance with specification v1.5.0
+*  All related operations, including memory management, are handled by the library.
 * */
 
 
@@ -44,15 +45,9 @@ Compiler Options
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #endif
 
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#ifdef __GNUC__
-#  pragma GCC diagnostic ignored "-Wmissing-braces"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#  pragma GCC diagnostic ignored "-Wmissing-field-initializers"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#endif
-
 
 /**************************************
-Memory routines
+*  Memory routines
 **************************************/
 #include <stdlib.h>   /* malloc, calloc, free */
 #define ALLOCATOR(s)   calloc(1,s)
@@ -62,7 +57,7 @@ Memory routines
 
 
 /**************************************
-Includes
+*  Includes
 **************************************/
 #include "lz4frame_static.h"
 #include "lz4.h"
@@ -71,7 +66,7 @@ Includes
 
 
 /**************************************
-Basic Types
+*  Basic Types
 **************************************/
 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
 # include <stdint.h>
@@ -90,7 +85,7 @@ typedef unsigned long long  U64;
 
 
 /**************************************
-Constants
+*  Constants
 **************************************/
 #define KB *(1<<10)
 #define MB *(1<<20)
@@ -102,36 +97,39 @@ Constants
 #define _4BITS 0x0F
 #define _8BITS 0xFF
 
+#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
 #define LZ4F_MAGICNUMBER 0x184D2204U
 #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
-#define LZ4F_MAXHEADERFRAME_SIZE 7
+#define LZ4F_MAXHEADERFRAME_SIZE 15
 #define LZ4F_BLOCKSIZEID_DEFAULT max64KB
 
+static const size_t minFHSize = 5;
 static const U32 minHClevel = 3;
 
 /**************************************
-Structures and local types
+*  Structures and local types
 **************************************/
 typedef struct
 {
     LZ4F_preferences_t prefs;
-    U32 version;
-    U32 cStage;
+    U32    version;
+    U32    cStage;
     size_t maxBlockSize;
     size_t maxBufferSize;
     BYTE*  tmpBuff;
     BYTE*  tmpIn;
     size_t tmpInSize;
+    U64    totalInSize;
     XXH32_state_t xxh;
-    void* lz4CtxPtr;
-    U32 lz4CtxLevel;     /* 0: unallocated;  1: LZ4_stream_t;  3: LZ4_streamHC_t */
+    void*  lz4CtxPtr;
+    U32    lz4CtxLevel;     /* 0: unallocated;  1: LZ4_stream_t;  3: LZ4_streamHC_t */
 } LZ4F_cctx_internal_t;
 
 typedef struct
 {
     LZ4F_frameInfo_t frameInfo;
-    unsigned version;
-    unsigned dStage;
+    U32    version;
+    U32    dStage;
     size_t maxBlockSize;
     size_t maxBufferSize;
     const BYTE* srcExpect;
@@ -145,23 +143,18 @@ typedef struct
     size_t tmpOutSize;
     size_t tmpOutStart;
     XXH32_state_t xxh;
-    BYTE   header[8];
+    BYTE   header[16];
 } LZ4F_dctx_internal_t;
 
 
 /**************************************
-Macros
-**************************************/
-
-
-/**************************************
-Error management
+*  Error management
 **************************************/
 #define LZ4F_GENERATE_STRING(STRING) #STRING,
 static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) };
 
 
-U32 LZ4F_isError(LZ4F_errorCode_t code)
+unsigned LZ4F_isError(LZ4F_errorCode_t code)
 {
     return (code > (LZ4F_errorCode_t)(-ERROR_maxCode));
 }
@@ -175,7 +168,7 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code)
 
 
 /**************************************
-Private functions
+*  Private functions
 **************************************/
 static size_t LZ4F_getBlockSize(unsigned blockSizeID)
 {
@@ -189,14 +182,6 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID)
 
 
 /* unoptimized version; solves endianess & alignment issues */
-static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32)
-{
-    dstPtr[0] = (BYTE)value32;
-    dstPtr[1] = (BYTE)(value32 >> 8);
-    dstPtr[2] = (BYTE)(value32 >> 16);
-    dstPtr[3] = (BYTE)(value32 >> 24);
-}
-
 static U32 LZ4F_readLE32 (const BYTE* srcPtr)
 {
     U32 value32 = srcPtr[0];
@@ -206,41 +191,78 @@ static U32 LZ4F_readLE32 (const BYTE* srcPtr)
     return value32;
 }
 
-
-static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length)
+static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32)
 {
-    U32 xxh = XXH32(header, (U32)length, 0);
+    dstPtr[0] = (BYTE)value32;
+    dstPtr[1] = (BYTE)(value32 >> 8);
+    dstPtr[2] = (BYTE)(value32 >> 16);
+    dstPtr[3] = (BYTE)(value32 >> 24);
+}
+
+static U64 LZ4F_readLE64 (const BYTE* srcPtr)
+{
+    U64 value64 = srcPtr[0];
+    value64 += (srcPtr[1]<<8);
+    value64 += (srcPtr[2]<<16);
+    value64 += (srcPtr[3]<<24);
+    value64 += ((U64)srcPtr[4]<<32);
+    value64 += ((U64)srcPtr[5]<<40);
+    value64 += ((U64)srcPtr[6]<<48);
+    value64 += ((U64)srcPtr[7]<<56);
+    return value64;
+}
+
+static void LZ4F_writeLE64 (BYTE* dstPtr, U64 value64)
+{
+    dstPtr[0] = (BYTE)value64;
+    dstPtr[1] = (BYTE)(value64 >> 8);
+    dstPtr[2] = (BYTE)(value64 >> 16);
+    dstPtr[3] = (BYTE)(value64 >> 24);
+    dstPtr[4] = (BYTE)(value64 >> 32);
+    dstPtr[5] = (BYTE)(value64 >> 40);
+    dstPtr[6] = (BYTE)(value64 >> 48);
+    dstPtr[7] = (BYTE)(value64 >> 56);
+}
+
+
+static BYTE LZ4F_headerChecksum (const void* header, size_t length)
+{
+    U32 xxh = XXH32(header, length, 0);
     return (BYTE)(xxh >> 8);
 }
 
 
 /**************************************
-Simple compression functions
+*  Simple compression functions
 **************************************/
+static blockSizeID_t LZ4F_optimalBSID(const blockSizeID_t requestedBSID, const size_t srcSize)
+{
+    blockSizeID_t proposedBSID = max64KB;
+    size_t maxBlockSize = 64 KB;
+    while (requestedBSID > proposedBSID)
+    {
+        if (srcSize <= maxBlockSize)
+            return proposedBSID;
+        proposedBSID = (blockSizeID_t)((int)proposedBSID + 1);
+        maxBlockSize <<= 2;
+    }
+    return requestedBSID;
+}
+
+
 size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
 {
-    LZ4F_preferences_t prefs = { 0 };
+    LZ4F_preferences_t prefs;
     size_t headerSize;
     size_t streamSize;
 
     if (preferencesPtr!=NULL) prefs = *preferencesPtr;
-    {
-        blockSizeID_t proposedBSID = max64KB;
-        size_t maxBlockSize = 64 KB;
-        while (prefs.frameInfo.blockSizeID > proposedBSID)
-        {
-            if (srcSize <= maxBlockSize)
-            {
-                prefs.frameInfo.blockSizeID = proposedBSID;
-                break;
-            }
-            proposedBSID++;
-            maxBlockSize <<= 2;
-        }
-    }
+    else memset(&prefs, 0, sizeof(prefs));
+
+    prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
     prefs.autoFlush = 1;
 
-    headerSize = 7;      /* basic header size (no option) including magic number */
+    headerSize = 15;      /* header size, including magic number and frame content size*/
     streamSize = LZ4F_compressBound(srcSize, &prefs);
 
     return headerSize + streamSize;
@@ -248,43 +270,47 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
 
 
 /* LZ4F_compressFrame()
-* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step.
+* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step.
 * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
 * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound()
 * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
-* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default.
 * The result of the function is the number of bytes written into dstBuffer.
 * The function outputs an error code if it fails (can be tested using LZ4F_isError())
 */
 size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
 {
-    LZ4F_cctx_internal_t cctxI = { 0 };   /* works because no allocation */
-    LZ4F_preferences_t prefs = { 0 };
-    LZ4F_compressOptions_t options = { 0 };
+    LZ4F_cctx_internal_t cctxI;
+    LZ4_stream_t lz4ctx;
+    LZ4F_preferences_t prefs;
+    LZ4F_compressOptions_t options;
     LZ4F_errorCode_t errorCode;
     BYTE* const dstStart = (BYTE*) dstBuffer;
     BYTE* dstPtr = dstStart;
     BYTE* const dstEnd = dstStart + dstMaxSize;
 
+    memset(&cctxI, 0, sizeof(cctxI));   /* works because no allocation */
+    memset(&options, 0, sizeof(options));
 
     cctxI.version = LZ4F_VERSION;
     cctxI.maxBufferSize = 5 MB;   /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */
 
     if (preferencesPtr!=NULL) prefs = *preferencesPtr;
+    else
     {
-        blockSizeID_t proposedBSID = max64KB;
-        size_t maxBlockSize = 64 KB;
-        while (prefs.frameInfo.blockSizeID > proposedBSID)
-        {
-            if (srcSize <= maxBlockSize)
-            {
-                prefs.frameInfo.blockSizeID = proposedBSID;
-                break;
-            }
-            proposedBSID++;
-            maxBlockSize <<= 2;
-        }
+        memset(&prefs, 0, sizeof(prefs));
+        prefs.frameInfo.frameOSize = (U64)srcSize;
     }
+    if (prefs.frameInfo.frameOSize != 0)
+        prefs.frameInfo.frameOSize = (U64)srcSize;   /* correct frame size if selected (!=0) */
+
+    if (prefs.compressionLevel < minHClevel)
+    {
+        cctxI.lz4CtxPtr = &lz4ctx;
+        cctxI.lz4CtxLevel = 1;
+    }
+
+    prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
     prefs.autoFlush = 1;
     if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
         prefs.frameInfo.blockMode = blockIndependent;   /* no need for linked blocks */
@@ -298,8 +324,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
     if (LZ4F_isError(errorCode)) return errorCode;
     dstPtr += errorCode;   /* header size */
 
-    dstMaxSize -= errorCode;
-    errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstMaxSize, srcBuffer, srcSize, &options);
+    errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
     if (LZ4F_isError(errorCode)) return errorCode;
     dstPtr += errorCode;
 
@@ -307,7 +332,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
     if (LZ4F_isError(errorCode)) return errorCode;
     dstPtr += errorCode;
 
-    FREEMEM(cctxI.lz4CtxPtr);
+    if (prefs.compressionLevel >= minHClevel)   /* no allocation necessary with lz4 fast */
+        FREEMEM(cctxI.lz4CtxPtr);
 
     return (dstPtr - dstStart);
 }
@@ -361,7 +387,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 */
 size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr)
 {
-    LZ4F_preferences_t prefNull = { 0 };
+    LZ4F_preferences_t prefNull;
     LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
     BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
@@ -370,20 +396,21 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
 
     if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall;
     if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC;
+    memset(&prefNull, 0, sizeof(prefNull));
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
     cctxPtr->prefs = *preferencesPtr;
 
     /* ctx Management */
     {
-        U32 targetCtxLevel = cctxPtr->prefs.compressionLevel<minHClevel ? 1 : 2;
-        if (cctxPtr->lz4CtxLevel < targetCtxLevel)
+        U32 tableID = cctxPtr->prefs.compressionLevel<minHClevel ? 1 : 2;  /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
+        if (cctxPtr->lz4CtxLevel < tableID)
         {
             FREEMEM(cctxPtr->lz4CtxPtr);
             if (cctxPtr->prefs.compressionLevel<minHClevel)
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
             else
                 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
-            cctxPtr->lz4CtxLevel = targetCtxLevel;
+            cctxPtr->lz4CtxLevel = tableID;
         }
     }
 
@@ -405,7 +432,7 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
     cctxPtr->tmpIn = cctxPtr->tmpBuff;
     cctxPtr->tmpInSize = 0;
     XXH32_reset(&(cctxPtr->xxh), 0);
-    if (cctxPtr->prefs.compressionLevel<minHClevel)
+    if (cctxPtr->prefs.compressionLevel < minHClevel)
         LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
     else
         LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
@@ -418,13 +445,22 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
     /* FLG Byte */
     *dstPtr++ = ((1 & _2BITS) << 6)    /* Version('01') */
         + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)    /* Block mode */
-        + (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2);   /* Stream checksum */
+        + (BYTE)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)   /* Frame checksum */
+        + (BYTE)((cctxPtr->prefs.frameInfo.frameOSize > 0) << 3);   /* Frame content size */
     /* BD Byte */
-    *dstPtr++ = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
+    *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
+    /* Optional Frame content size field */
+    if (cctxPtr->prefs.frameInfo.frameOSize)
+    {
+        LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.frameOSize);
+        dstPtr += 8;
+        cctxPtr->totalInSize = 0;
+    }
     /* CRC Byte */
-    *dstPtr++ = LZ4F_headerChecksum(headerStart, 2);
+    *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
+    dstPtr++;
 
-    cctxPtr->cStage = 1;   /* header written, wait for data block */
+    cctxPtr->cStage = 1;   /* header written, now request input data block */
 
     return (dstPtr - dstStart);
 }
@@ -436,17 +472,19 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
 * */
 size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
 {
-    const LZ4F_preferences_t prefsNull = { 0 };
-    const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
-    blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID;
-    size_t blockSize = LZ4F_getBlockSize(bid);
-    unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1;
-    size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize;
-    size_t blockInfo = 4;   /* default, without block CRC option */
-    size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
-    size_t result = (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;
+    LZ4F_preferences_t prefsNull;
+    memset(&prefsNull, 0, sizeof(prefsNull));
+    {
+        const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
+        blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID;
+        size_t blockSize = LZ4F_getBlockSize(bid);
+        unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1;
+        size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize;
+        size_t blockInfo = 4;   /* default, without block CRC option */
+        size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
 
-    return result;
+        return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;;
+    }
 }
 
 
@@ -518,7 +556,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
 */
 size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr)
 {
-    LZ4F_compressOptions_t cOptionsNull = { 0 };
+    LZ4F_compressOptions_t cOptionsNull;
     LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
     size_t blockSize = cctxPtr->maxBlockSize;
     const BYTE* srcPtr = (const BYTE*)srcBuffer;
@@ -531,6 +569,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
 
     if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC;
     if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall;
+    memset(&cOptionsNull, 0, sizeof(cOptionsNull));
     if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
 
     /* select compression function */
@@ -611,8 +650,9 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
     }
 
     if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled)
-        XXH32_update(&(cctxPtr->xxh), srcBuffer, (unsigned)srcSize);
+        XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
 
+    cctxPtr->totalInSize += srcSize;
     return dstPtr - dstStart;
 }
 
@@ -627,7 +667,6 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
 */
 size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
 {
-    LZ4F_compressOptions_t cOptionsNull = { 0 };
     LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext;
     BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
@@ -637,7 +676,6 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer,
     if (cctxPtr->tmpInSize == 0) return 0;   /* nothing to flush */
     if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC;
     if (dstMaxSize < (cctxPtr->tmpInSize + 16)) return (size_t)-ERROR_dstMaxSize_tooSmall;
-    if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
     (void)compressOptionsPtr;   /* not yet useful */
 
     /* select compression function */
@@ -691,13 +729,19 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB
 
     cctxPtr->cStage = 0;   /* state is now re-usable (with identical preferences) */
 
+    if (cctxPtr->prefs.frameInfo.frameOSize)
+    {
+        if (cctxPtr->prefs.frameInfo.frameOSize != cctxPtr->totalInSize)
+            return (size_t)-ERROR_frameSize_wrong;
+    }
+
     return dstPtr - dstStart;
 }
 
 
-/***********************************
-* Decompression functions
-* *********************************/
+/**********************************
+*  Decompression functions
+**********************************/
 
 /* Resource management */
 
@@ -708,11 +752,11 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB
 * If the result LZ4F_errorCode_t is not zero, there was an error during context creation.
 * Object can release its memory using LZ4F_freeDecompressionContext();
 */
-LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber)
+LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber)
 {
     LZ4F_dctx_internal_t* dctxPtr;
 
-    dctxPtr = ALLOCATOR(sizeof(LZ4F_dctx_internal_t));
+    dctxPtr = (LZ4F_dctx_internal_t*)ALLOCATOR(sizeof(LZ4F_dctx_internal_t));
     if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC;
 
     dctxPtr->version = versionNumber;
@@ -720,7 +764,7 @@ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F
     return OK_NoError;
 }
 
-LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext)
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_decompressionContext)
 {
     LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)LZ4F_decompressionContext;
     FREEMEM(dctxPtr->tmpIn);
@@ -730,51 +774,104 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_de
 }
 
 
-/* Decompression */
+/* ******************************************************************** */
+/* ********************* Decompression ******************************** */
+/* ******************************************************************** */
 
-static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPtr, size_t srcSize)
+typedef enum { dstage_getHeader=0, dstage_storeHeader,
+    dstage_getCBlockSize, dstage_storeCBlockSize,
+    dstage_copyDirect,
+    dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock,
+    dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut,
+    dstage_getSuffix, dstage_storeSuffix,
+    dstage_getSFrameSize, dstage_storeSFrameSize,
+    dstage_skipSkippable
+} dStage_t;
+
+
+/* LZ4F_decodeHeader
+   return : nb Bytes read from srcVoidPtr (necessarily <= srcSize)
+            or an error code (testable with LZ4F_isError())
+   output : set internal values of dctx, such as
+            dctxPtr->frameInfo and dctxPtr->dStage.
+*/
+static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize)
 {
     BYTE FLG, BD, HC;
-    unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID;
+    unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID;
     size_t bufferNeeded;
+    size_t frameHeaderSize;
+    const BYTE* srcPtr = (const BYTE*)srcVoidPtr;
 
     /* need to decode header to get frameInfo */
-    if (srcSize < 7) return (size_t)-ERROR_GENERIC;   /* minimal header size */
+    if (srcSize < minFHSize) return (size_t)-ERROR_GENERIC;   /* minimal header size */
+    memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo));
+
+    /* skippable frames */
+    if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
+    {
+        dctxPtr->frameInfo.frameType = skippableFrame;
+        if (srcVoidPtr == (void*)(dctxPtr->header))
+        {
+            dctxPtr->tmpInSize = srcSize;
+            dctxPtr->tmpInTarget = 8;
+            dctxPtr->dStage = dstage_storeSFrameSize;
+            return srcSize;
+        }
+        else
+        {
+            dctxPtr->dStage = dstage_getSFrameSize;
+            return 4;
+        }
+    }
 
     /* control magic number */
-    if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_GENERIC;
-    srcPtr += 4;
+    if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_frameType_unknown;
+    dctxPtr->frameInfo.frameType = LZ4F_frame;
 
     /* Flags */
-    FLG = srcPtr[0];
-    version = (FLG>>6)&_2BITS;
+    FLG = srcPtr[4];
+    version = (FLG>>6) & _2BITS;
     blockMode = (FLG>>5) & _1BIT;
     blockChecksumFlag = (FLG>>4) & _1BIT;
     contentSizeFlag = (FLG>>3) & _1BIT;
     contentChecksumFlag = (FLG>>2) & _1BIT;
-    dictFlag = (FLG>>0) & _1BIT;
-    BD = srcPtr[1];
+
+    /* Frame Header Size */
+    frameHeaderSize = contentSizeFlag ? 15 : 7;
+
+    if (srcSize < frameHeaderSize)
+    {
+        if (srcPtr != dctxPtr->header)
+            memcpy(dctxPtr->header, srcPtr, srcSize);
+        dctxPtr->tmpInSize = srcSize;
+        dctxPtr->tmpInTarget = frameHeaderSize;
+        dctxPtr->dStage = dstage_storeHeader;
+        return srcSize;
+    }
+
+    BD = srcPtr[5];
     blockSizeID = (BD>>4) & _3BITS;
 
-    /* check */
-    HC = LZ4F_headerChecksum(srcPtr, 2);
-    if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC;   /* Bad header checksum error */
-
     /* validate */
-    if (version != 1) return (size_t)-ERROR_GENERIC;   /* Version Number, only supported value */
-    if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC;   /* Only supported value for the time being */
-    if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC;   /* Only supported value for the time being */
-    if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC;   /* Reserved bit */
-    if (dictFlag != 0) return (size_t)-ERROR_GENERIC;   /* Only supported value for the time being */
+    if (version != 1) return (size_t)-ERROR_GENERIC;           /* Version Number, only supported value */
+    if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
+    if (((FLG>>0)&_2BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */
     if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC;   /* Reserved bit */
-    if (blockSizeID < 4) return (size_t)-ERROR_GENERIC;   /* Only supported values for the time being */
-    if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC;   /* Reserved bits */
+    if (blockSizeID < 4) return (size_t)-ERROR_GENERIC;        /* 4-7 only supported values for the time being */
+    if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC;  /* Reserved bits */
+
+    /* check */
+    HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
+    if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-ERROR_GENERIC;   /* Bad header checksum error */
 
     /* save */
-    dctxPtr->frameInfo.blockMode = blockMode;
-    dctxPtr->frameInfo.contentChecksumFlag = contentChecksumFlag;
-    dctxPtr->frameInfo.blockSizeID = blockSizeID;
+    dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode;
+    dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag;
+    dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID;
     dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
+    if (contentSizeFlag)
+        dctxPtr->frameInfo.frameOSize = LZ4F_readLE64(srcPtr+6);
 
     /* init */
     if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0);
@@ -786,9 +883,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt
         FREEMEM(dctxPtr->tmpIn);
         FREEMEM(dctxPtr->tmpOutBuffer);
         dctxPtr->maxBufferSize = bufferNeeded;
-        dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize);
+        dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize);
         if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC;
-        dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize);
+        dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize);
         if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC;
     }
     dctxPtr->tmpInSize = 0;
@@ -799,19 +896,12 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt
     dctxPtr->tmpOutStart = 0;
     dctxPtr->tmpOutSize = 0;
 
-    return 7;
+    dctxPtr->dStage = dstage_getCBlockSize;
+
+    return frameHeaderSize;
 }
 
 
-typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_decodeHeader,
-    dstage_getCBlockSize, dstage_storeCBlockSize, dstage_decodeCBlockSize,
-    dstage_copyDirect,
-    dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock,
-    dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut,
-    dstage_getSuffix, dstage_storeSuffix, dstage_checkSuffix
-} dStage_t;
-
-
 /* LZ4F_getFrameInfo()
 * This function decodes frame header information, such as blockSize.
 * It is optional : you could start by calling directly LZ4F_decompress() instead.
@@ -830,11 +920,10 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont
     {
         LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcBuffer, *srcSizePtr);
         if (LZ4F_isError(errorCode)) return errorCode;
-        *srcSizePtr = errorCode;
+        *srcSizePtr = errorCode;   /* nb Bytes consumed */
         *frameInfoPtr = dctxPtr->frameInfo;
         dctxPtr->srcExpect = NULL;
-        dctxPtr->dStage = dstage_getCBlockSize;
-        return 4;
+        return 4;   /* nextSrcSizeHint : 4 == block header size */
     }
 
     /* frameInfo already decoded */
@@ -844,15 +933,14 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont
 }
 
 
+/* redirector, with common prototype */
 static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
 {
-    (void)dictStart;
-    (void)dictSize;
+    (void)dictStart; (void)dictSize;
     return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize);
 }
 
 
-
 static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
 {
     if (dctxPtr->dictSize==0)
@@ -880,15 +968,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s
 
     if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
     {
-#if 0
-        size_t savedDictSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
-        memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart- savedDictSize, savedDictSize);
-        dctxPtr->dict = dctxPtr->tmpOutBuffer;
-        dctxPtr->dictSize = savedDictSize + dctxPtr->tmpOutStart + dstSize;
-        return;
-
-#else
-
         size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
         size_t copySize = 64 KB - dctxPtr->tmpOutSize;
         BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
@@ -900,7 +979,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s
         dctxPtr->dict = dctxPtr->tmpOutBuffer;
         dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize;
         return;
-#endif
     }
 
     if (dctxPtr->dict == dctxPtr->tmpOutBuffer)     /* copy dst into tmp to complete dict */
@@ -952,18 +1030,19 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                        const LZ4F_decompressOptions_t* decompressOptionsPtr)
 {
     LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext;
-    static const LZ4F_decompressOptions_t optionsNull = { 0 };
+    LZ4F_decompressOptions_t optionsNull;
     const BYTE* const srcStart = (const BYTE*)srcBuffer;
     const BYTE* const srcEnd = srcStart + *srcSizePtr;
     const BYTE* srcPtr = srcStart;
     BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* const dstEnd = dstStart + *dstSizePtr;
     BYTE* dstPtr = dstStart;
-    const BYTE* selectedIn=NULL;
+    const BYTE* selectedIn = NULL;
     unsigned doAnotherStage = 1;
     size_t nextSrcSizeHint = 1;
 
 
+    memset(&optionsNull, 0, sizeof(optionsNull));
     if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
     *srcSizePtr = 0;
     *dstSizePtr = 0;
@@ -971,7 +1050,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
     /* expect to continue decoding src buffer where it left previously */
     if (dctxPtr->srcExpect != NULL)
     {
-        if (srcStart != dctxPtr->srcExpect) return (size_t)-ERROR_GENERIC;
+        if (srcStart != dctxPtr->srcExpect) return (size_t)-ERROR_wrongSrcPtr;
     }
 
     /* programmed as a state machine */
@@ -986,39 +1065,31 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
             {
                 if (srcEnd-srcPtr >= 7)
                 {
-                    selectedIn = srcPtr;
-                    srcPtr += 7;
-                    dctxPtr->dStage = dstage_decodeHeader;
+                    LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr);
+                    if (LZ4F_isError(errorCode)) return errorCode;
+                    srcPtr += errorCode;
                     break;
                 }
                 dctxPtr->tmpInSize = 0;
+                dctxPtr->tmpInTarget = 7;
                 dctxPtr->dStage = dstage_storeHeader;
-                break;
             }
 
         case dstage_storeHeader:
             {
-                size_t sizeToCopy = 7 - dctxPtr->tmpInSize;
+                size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
                 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy =  srcEnd - srcPtr;
                 memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
                 dctxPtr->tmpInSize += sizeToCopy;
                 srcPtr += sizeToCopy;
-                if (dctxPtr->tmpInSize < 7)
+                if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget)
                 {
-                    nextSrcSizeHint = (7 - dctxPtr->tmpInSize) + 4;
-                    doAnotherStage = 0;   /* no enough src, wait to get some more */
+                    nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4;
+                    doAnotherStage = 0;   /* not enough src data, ask for some more */
                     break;
                 }
-                selectedIn = dctxPtr->header;
-                dctxPtr->dStage = dstage_decodeHeader;
-                break;
-            }
-
-        case dstage_decodeHeader:
-            {
-                LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, selectedIn, 7);
+                LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget);
                 if (LZ4F_isError(errorCode)) return errorCode;
-                dctxPtr->dStage = dstage_getCBlockSize;
                 break;
             }
 
@@ -1028,15 +1099,16 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                 {
                     selectedIn = srcPtr;
                     srcPtr += 4;
-                    dctxPtr->dStage = dstage_decodeCBlockSize;
-                    break;
                 }
+                else
+                {
                 /* not enough input to read cBlockSize field */
-                dctxPtr->tmpInSize = 0;
-                dctxPtr->dStage = dstage_storeCBlockSize;
-                break;
+                    dctxPtr->tmpInSize = 0;
+                    dctxPtr->dStage = dstage_storeCBlockSize;
+                }
             }
 
+            if (dctxPtr->dStage == dstage_storeCBlockSize)
         case dstage_storeCBlockSize:
             {
                 size_t sizeToCopy = 4 - dctxPtr->tmpInSize;
@@ -1051,11 +1123,9 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                     break;
                 }
                 selectedIn = dctxPtr->tmpIn;
-                dctxPtr->dStage = dstage_decodeCBlockSize;
-                break;
             }
 
-        case dstage_decodeCBlockSize:
+        /* case dstage_decodeCBlockSize: */   /* no more direct access, to prevent scan-build warning */
             {
                 size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
                 if (nextCBlockSize==0)   /* frameEnd signal, no more CBlock */
@@ -1243,18 +1313,19 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                     doAnotherStage = 0;
                     break;
                 }
-                if ((srcEnd - srcPtr) >= 4)   /* CRC present */
+                if ((srcEnd - srcPtr) < 4)   /* not enough size for entire CRC */
+                {
+                    dctxPtr->tmpInSize = 0;
+                    dctxPtr->dStage = dstage_storeSuffix;
+                }
+                else
                 {
                     selectedIn = srcPtr;
                     srcPtr += 4;
-                    dctxPtr->dStage = dstage_checkSuffix;
-                    break;
                 }
-                dctxPtr->tmpInSize = 0;
-                dctxPtr->dStage = dstage_storeSuffix;
-                break;
             }
 
+            if (dctxPtr->dStage == dstage_storeSuffix)
         case dstage_storeSuffix:
             {
                 size_t sizeToCopy = 4 - dctxPtr->tmpInSize;
@@ -1269,11 +1340,9 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                     break;
                 }
                 selectedIn = dctxPtr->tmpIn;
-                dctxPtr->dStage = dstage_checkSuffix;
-                break;
             }
 
-        case dstage_checkSuffix:
+        /* case dstage_checkSuffix: */   /* no direct call, to avoid scan-build warning */
             {
                 U32 readCRC = LZ4F_readLE32(selectedIn);
                 U32 resultCRC = XXH32_digest(&(dctxPtr->xxh));
@@ -1283,6 +1352,61 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                 doAnotherStage = 0;
                 break;
             }
+
+        case dstage_getSFrameSize:
+            {
+                if ((srcEnd - srcPtr) >= 4)
+                {
+                    selectedIn = srcPtr;
+                    srcPtr += 4;
+                }
+                else
+                {
+                /* not enough input to read cBlockSize field */
+                    dctxPtr->tmpInSize = 4;
+                    dctxPtr->tmpInTarget = 8;
+                    dctxPtr->dStage = dstage_storeSFrameSize;
+                }
+            }
+
+            if (dctxPtr->dStage == dstage_storeSFrameSize)
+        case dstage_storeSFrameSize:
+            {
+                size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
+                if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
+                memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
+                srcPtr += sizeToCopy;
+                dctxPtr->tmpInSize += sizeToCopy;
+                if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* not enough input to get full sBlockSize; wait for more */
+                {
+                    nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
+                    doAnotherStage = 0;
+                    break;
+                }
+                selectedIn = dctxPtr->header + 4;
+            }
+
+        /* case dstage_decodeSBlockSize: */   /* no direct access */
+            {
+                size_t SFrameSize = LZ4F_readLE32(selectedIn);
+                dctxPtr->frameInfo.frameOSize = SFrameSize;
+                dctxPtr->tmpInTarget = SFrameSize;
+                dctxPtr->dStage = dstage_skipSkippable;
+                break;
+            }
+
+        case dstage_skipSkippable:
+            {
+                size_t skipSize = dctxPtr->tmpInTarget;
+                if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr;
+                srcPtr += skipSize;
+                dctxPtr->tmpInTarget -= skipSize;
+                doAnotherStage = 0;
+                nextSrcSizeHint = dctxPtr->tmpInTarget;
+                if (nextSrcSizeHint) break;
+                dctxPtr->dStage = dstage_getHeader;
+                break;
+            }
         }
     }
 
@@ -1320,10 +1444,12 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
         }
     }
 
-    if (srcPtr<srcEnd)   /* function must be called again with following source data */
+    /* require function to be called again from position where it stopped */
+    if (srcPtr<srcEnd)
         dctxPtr->srcExpect = srcPtr;
     else
         dctxPtr->srcExpect = NULL;
+
     *srcSizePtr = (srcPtr - srcStart);
     *dstSizePtr = (dstPtr - dstStart);
     return nextSrcSizeHint;

lib/lz4frame.h

@@ -28,8 +28,7 @@
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
-   - LZ4 source repository : http://code.google.com/p/lz4/
-   - LZ4 source mirror : https://github.com/Cyan4973/lz4
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
    - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
@@ -66,19 +65,22 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /* return error code str
 typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7 } blockSizeID_t;
 typedef enum { blockLinked=0, blockIndependent} blockMode_t;
 typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t;
+typedef enum { LZ4F_frame=0, skippableFrame } frameType_t;
 
 typedef struct {
-  blockSizeID_t     blockSizeID;           /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
-  blockMode_t       blockMode;             /* blockLinked, blockIndependent ; 0 == default */
-  contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
-  unsigned          reserved[5];
+  blockSizeID_t      blockSizeID;           /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
+  blockMode_t        blockMode;             /* blockLinked, blockIndependent ; 0 == default */
+  contentChecksum_t  contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
+  frameType_t        frameType;             /* LZ4F_frame, skippableFrame ; 0 == default */
+  unsigned long long frameOSize;            /* Size of uncompressed (original) content ; 0 == unknown */
+  unsigned           reserved[2];           /* must be zero for forward compatibility */
 } LZ4F_frameInfo_t;
 
 typedef struct {
   LZ4F_frameInfo_t frameInfo;
   unsigned         compressionLevel;       /* 0 == default (fast mode); values above 16 count as 16 */
-  unsigned         autoFlush;              /* 1 == always flush : reduce need for tmp buffer */
-  unsigned         reserved[4];
+  unsigned         autoFlush;              /* 1 == always flush (reduce need for tmp buffer) */
+  unsigned         reserved[4];            /* must be zero for forward compatibility */
 } LZ4F_preferences_t;
 
 
@@ -130,7 +132,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
 size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr);
 /* LZ4F_compressBegin() :
  * will write the frame header into dstBuffer.
- * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 19 bytes.
+ * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes.
  * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default.
  * The result of the function is the number of bytes written into dstBuffer for the header
  * or an error code (can be tested using LZ4F_isError())
@@ -215,7 +217,7 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx,
  * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
  * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr)
  * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call,
- * or an error code which can be tested using LZ4F_isError().
+ *                        or an error code which can be tested using LZ4F_isError().
  */
 
 size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx,

lib/lz4frame_static.h

@@ -29,8 +29,7 @@
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
-   - LZ4 source repository : http://code.google.com/p/lz4/
-   - LZ4 source mirror : https://github.com/Cyan4973/lz4
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
    - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
@@ -53,6 +52,9 @@ extern "C" {
         ITEM(ERROR_compressionLevel_invalid) \
         ITEM(ERROR_allocation_failed) \
         ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
+        ITEM(ERROR_frameSize_wrong) \
+        ITEM(ERROR_frameType_unknown) \
+        ITEM(ERROR_wrongSrcPtr) \
         ITEM(ERROR_decompressionFailed) \
         ITEM(ERROR_checksum_invalid) \
         ITEM(ERROR_maxCode)

lib/lz4hc.c

@@ -1,6 +1,7 @@
 /*
 LZ4 HC - High Compression Mode of LZ4
-Copyright (C) 2011-2014, Yann Collet.
+Copyright (C) 2011-2015, Yann Collet.
+
 BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
 Redistribution and use in source and binary forms, with or without
@@ -27,8 +28,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 You can contact the author at :
-- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
-- LZ4 source repository : http://code.google.com/p/lz4/
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
 

lib/lz4hc.h

@@ -1,7 +1,7 @@
 /*
    LZ4 HC - High Compression Mode of LZ4
    Header File
-   Copyright (C) 2011-2014, Yann Collet.
+   Copyright (C) 2011-2015, Yann Collet.
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without
@@ -28,8 +28,8 @@
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
-   - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
-   - LZ4 source repository : http://code.google.com/p/lz4/
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 #pragma once
 

lib/xxhash.c

@@ -28,8 +28,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 You can contact the author at :
-- xxHash source repository : http://code.google.com/p/xxhash/
-- xxHash source mirror : https://github.com/Cyan4973/xxHash
+- xxHash source repository : https://github.com/Cyan4973/xxHash
 - public discussion board : https://groups.google.com/forum/#!forum/lz4c
 */
 

lib/xxhash.h

@@ -1,7 +1,8 @@
 /*
    xxHash - Extremely Fast Hash algorithm
    Header File
-   Copyright (C) 2012-2014, Yann Collet.
+   Copyright (C) 2012-2015, Yann Collet.
+
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
    Redistribution and use in source and binary forms, with or without

programs/Makefile

@@ -19,8 +19,8 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # You can contact the author at :
-#  - LZ4 source repository : http://code.google.com/p/lz4/
-#  - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c
+#  - LZ4 source repository : https://github.com/Cyan4973/lz4
+#  - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 # ##########################################################################
 # lz4 : Command Line Utility, supporting gzip-like arguments
 # lz4c  : CLU, supporting also legacy lz4demo arguments
@@ -31,10 +31,10 @@
 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
-RELEASE?= r126
+RELEASE?= r128
 
 DESTDIR?=
-PREFIX ?= /usr
+PREFIX ?= /usr/local
 CFLAGS ?= -O3
 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\"
 FLAGS   = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
@@ -92,10 +92,9 @@ frametest: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxha
 frametest32: $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c frametest.c
 	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
 
-datagen : datagen.c
+datagen : datagen.c datagencli.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-
 clean:
 	@rm -f core *.o *.test \
         lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
@@ -113,22 +112,26 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
 install: lz4 lz4c
 	@echo Installing binaries
 	@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
-	@install -m 755 lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4$(EXT)
-	@ln -sf lz4$(EXT) $(DESTDIR)$(BINDIR)/lz4cat
-	@install -m 755 lz4c$(EXT) $(DESTDIR)$(BINDIR)/lz4c$(EXT)
+	@install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4
+	@ln -sf lz4 $(DESTDIR)$(BINDIR)/lz4cat
+	@ln -sf lz4 $(DESTDIR)$(BINDIR)/unlz4
+	@install -m 755 lz4c $(DESTDIR)$(BINDIR)/lz4c
 	@echo Installing man pages
 	@install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1
-	@install -m 644 lz4c.1 $(DESTDIR)$(MANDIR)/lz4c.1
-	@install -m 644 lz4cat.1 $(DESTDIR)$(MANDIR)/lz4cat.1
+	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4c.1
+	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/lz4cat.1
+	@ln -sf lz4.1 $(DESTDIR)$(MANDIR)/unlz4.1
 	@echo lz4 installation completed
 
 uninstall:
 	rm -f $(DESTDIR)$(BINDIR)/lz4cat
-	[ -x $(DESTDIR)$(BINDIR)/lz4$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4$(EXT)
-	[ -x $(DESTDIR)$(BINDIR)/lz4c$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/lz4c$(EXT)
+	rm -f $(DESTDIR)$(BINDIR)/unlz4
+	[ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4
+	[ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c
 	[ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1
-	[ -f $(DESTDIR)$(MANDIR)/lz4c.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4c.1
-	[ -f $(DESTDIR)$(MANDIR)/lz4cat.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
+	rm -f $(DESTDIR)$(MANDIR)/lz4c.1
+	rm -f $(DESTDIR)$(MANDIR)/lz4cat.1
+	rm -f $(DESTDIR)$(MANDIR)/unlz4.1
 	@echo lz4 programs successfully uninstalled
 
 test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-mem
@@ -139,13 +142,32 @@ test-all: test test32
 
 test-travis: $(TRAVIS_TARGET)
 
-test-lz4: lz4 datagen
-	./datagen -g16KB  | ./lz4 -9     | ./lz4 -vdq > $(VOID)
-	./datagen         | ./lz4        | ./lz4 -vdq > $(VOID)
-	./datagen -g6M -p100 | ./lz4 -9BD | ./lz4 -vdq > $(VOID)
-	./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -vdq > $(VOID)
-	./datagen -g6GB   | ./lz4 -vqB5D | ./lz4 -vdq > $(VOID)
-# test frame concatenation with null-length frame
+test-lz4-sparse: lz4 datagen
+	@echo ---- test sparse file support ----
+	./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dv --sparse-support > tmpB4
+	./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dv --sparse-support > tmpB5
+	./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dv --sparse-support > tmpB6
+	./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dv --sparse-support > tmpB7
+	ls -ls tmp*
+	./datagen -g50M -P100 | diff -s - tmpB4
+	./datagen -g50M -P100 | diff -s - tmpB5
+	./datagen -g50M -P100 | diff -s - tmpB6
+	./datagen -g50M -P100 | diff -s - tmpB7
+	./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse-support > tmpOdd   # Odd size file (to not finish on an exact nb of blocks)
+	./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd
+	ls -ls tmpOdd
+	@rm tmp*
+
+test-lz4-contentSize: lz4 datagen
+	@echo ---- test original size support ----
+	./datagen -g15M > tmp
+	./lz4 -v tmp | ./lz4 -t
+	./lz4 -v --frame-content-size tmp | ./lz4 -d > tmp2
+	diff -s tmp tmp2
+	@rm tmp*
+
+test-lz4-frame-concatenation: lz4 datagen
+	@echo ---- test frame concatenation ----
 	@echo -n > empty.test
 	@echo hi > nonempty.test
 	cat nonempty.test empty.test nonempty.test > orig.test
@@ -156,20 +178,40 @@ test-lz4: lz4 datagen
 	sdiff orig.test result.test
 	@rm *.test
 	@echo frame concatenation test completed
-# test frame concatenation with null-length frame
 
+test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
+	@echo ---- test lz4 basic compression/decompression ----
+	./datagen -g0     | ./lz4 -v     | ./lz4 -t
+	./datagen -g16KB  | ./lz4 -9     | ./lz4 -t
+	./datagen         | ./lz4        | ./lz4 -t
+	./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t
+	./datagen -g17M   | ./lz4 -9v    | ./lz4 -tq
+	./datagen -g33M   | ./lz4 --no-frame-crc | ./lz4 -t
+	./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t
+	./datagen -g6GB   | ./lz4 -vqB5D | ./lz4 -t
+	./datagen -g6GB   | ./lz4 -vq9BD | ./lz4 -t
+	@echo ---- test multiple input files ----
+	@./datagen -s1 > file1
+	@./datagen -s2 > file2
+	@./datagen -s3 > file3
+	./lz4 -f -m file1 file2 file3
+	ls -l file*
+	@rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4
+	@echo ---- test pass-through ----
+	./datagen | ./lz4 -tf
 
 test-lz4c: lz4c datagen
-	./datagen -g256MB | ./lz4c -l -v    | ./lz4c   -vdq > $(VOID)
+	./datagen -g256MB | ./lz4c -l -v    | ./lz4c   -t
 
-test-lz4c32: lz4 lz4c32 lz4 datagen
-	./datagen -g16KB  | ./lz4c32 -9     | ./lz4c32 -vdq > $(VOID)
-	./datagen -g16KB  | ./lz4c32 -9     | ./lz4    -vdq > $(VOID)
-	./datagen         | ./lz4c32        | ./lz4c32 -vdq > $(VOID)
-	./datagen         | ./lz4c32        | ./lz4    -vdq > $(VOID)
-	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -vdq > $(VOID)
-	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4    -vdq > $(VOID)
-	./datagen -g6GB   | ./lz4c32 -vqB5D | ./lz4c32 -vdq > $(VOID)
+test-lz4c32: lz4 lz4c32 datagen
+	./datagen -g16KB  | ./lz4c32 -9     | ./lz4c32 -t
+	./datagen -g16KB  | ./lz4c32 -9     | ./lz4    -t
+	./datagen         | ./lz4c32        | ./lz4c32 -t
+	./datagen         | ./lz4c32        | ./lz4    -t
+	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -t
+	./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4    -t
+	./datagen -g6GB   | ./lz4c32 -vqB5D | ./lz4c32 -t
+	./datagen -g6GB   | ./lz4c32 -vq9BD | ./lz4    -t
 
 test-fullbench: fullbench
 	./fullbench --no-prompt $(TEST_FILES)
@@ -190,13 +232,15 @@ test-frametest32: frametest32
 	./frametest32
 
 test-mem: lz4 datagen fuzzer frametest
+	valgrind --leak-check=yes ./datagen -g50M > $(VOID)
 	./datagen -g16KB > tmp
-	valgrind --leak-check=yes ./lz4 -9 -BD -f tmp /dev/null
+	valgrind --leak-check=yes ./lz4 -9 -BD -f tmp $(VOID)
 	./datagen -g16MB > tmp
-	valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp /dev/null
+	valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp tmp2
 	./datagen -g256MB > tmp
-	valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp /dev/null
-	rm tmp
+	valgrind --leak-check=yes ./lz4 -t tmp2
+	valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp $(VOID)
+	rm tmp*
 	valgrind --leak-check=yes ./fuzzer -i64 -t1
 	valgrind --leak-check=yes ./frametest -i256
 

programs/bench.c

@@ -1,6 +1,7 @@
 /*
-    bench.c - Demo program to benchmark open-source compression algorithm
-    Copyright (C) Yann Collet 2012-2014
+    bench.c - Demo program to benchmark open-source compression algorithms
+    Copyright (C) Yann Collet 2012-2015
+
     GPL v2 License
 
     This program is free software; you can redistribute it and/or modify
@@ -18,16 +19,18 @@
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
     You can contact the author at :
-    - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
-    - LZ4 source repository : http://code.google.com/p/lz4/
+    - LZ4 source repository : https://github.com/Cyan4973/lz4
+    - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
 /**************************************
 *  Compiler Options
 ***************************************/
-/* Disable some Visual warning messages */
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE     /* VS2005 */
+#if defined(_MSC_VER) || defined(_WIN32)
+#  define _CRT_SECURE_NO_WARNINGS
+#  define _CRT_SECURE_NO_DEPRECATE     /* VS2005 */
+#  define BMK_LEGACY_TIMER 1           /* S_ISREG & gettimeofday() are not supported by MSVC */
+#endif
 
 /* Unix Large Files support (>4GB) */
 #define _FILE_OFFSET_BITS 64
@@ -37,11 +40,6 @@
 #  define _LARGEFILE64_SOURCE
 #endif
 
-/* S_ISREG & gettimeofday() are not supported by MSVC */
-#if defined(_MSC_VER) || defined(_WIN32)
-#  define BMK_LEGACY_TIMER 1
-#endif
-
 
 /**************************************
 *  Includes
@@ -141,15 +139,15 @@ static int chunkSize = DEFAULT_CHUNKSIZE;
 static int nbIterations = NBLOOPS;
 static int BMK_pause = 0;
 
-void BMK_SetBlocksize(int bsize) { chunkSize = bsize; }
+void BMK_setBlocksize(int bsize) { chunkSize = bsize; }
 
-void BMK_SetNbIterations(int nbLoops)
+void BMK_setNbIterations(int nbLoops)
 {
     nbIterations = nbLoops;
     DISPLAY("- %i iterations -\n", nbIterations);
 }
 
-void BMK_SetPause(void) { BMK_pause = 1; }
+void BMK_setPause(void) { BMK_pause = 1; }
 
 
 /*********************************************************
@@ -206,16 +204,21 @@ static size_t BMK_findMaxMem(U64 requiredMem)
 
     while (!testmem)
     {
-        requiredMem -= step;
+        if (requiredMem > step) requiredMem -= step;
+        else requiredMem >>= 1;
         testmem = (BYTE*) malloc ((size_t)requiredMem);
     }
-
     free (testmem);
-    return (size_t) (requiredMem - step);
+
+    /* keep some space available */
+    if (requiredMem > step) requiredMem -= step;
+    else requiredMem >>= 1;
+
+    return (size_t)requiredMem;
 }
 
 
-static U64 BMK_GetFileSize(char* infilename)
+static U64 BMK_GetFileSize(const char* infilename)
 {
     int r;
 #if defined(_MSC_VER)
@@ -234,7 +237,7 @@ static U64 BMK_GetFileSize(char* infilename)
 *  Public function
 **********************************************************/
 
-int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
+int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel)
 {
   int fileIdx=0;
   char* orig_buff;
@@ -265,7 +268,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
   while (fileIdx<nbFiles)
   {
       FILE*  inFile;
-      char*  inFileName;
+      const char*  inFileName;
       U64    inFileSize;
       size_t benchedSize;
       int nbChunks;
@@ -286,7 +289,9 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
 
       /* Memory allocation & restrictions */
       inFileSize = BMK_GetFileSize(inFileName);
+      if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; }
       benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;
+      if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; }
       if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
       if (benchedSize < inFileSize)
       {
@@ -295,11 +300,11 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
 
       /* Alloc */
       chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters));
-      orig_buff = (char*)malloc((size_t )benchedSize);
+      orig_buff = (char*)malloc((size_t)benchedSize);
       nbChunks = (int) ((int)benchedSize / chunkSize) + 1;
       maxCompressedChunkSize = LZ4_compressBound(chunkSize);
       compressedBuffSize = nbChunks * maxCompressedChunkSize;
-      compressedBuffer = (char*)malloc((size_t )compressedBuffSize);
+      compressedBuffer = (char*)malloc((size_t)compressedBuffSize);
 
 
       if (!orig_buff || !compressedBuffer)

programs/bench.h

@@ -1,6 +1,6 @@
 /*
     bench.h - Demo program to benchmark open-source compression algorithm
-    Copyright (C) Yann Collet 2012-2014
+    Copyright (C) Yann Collet 2012-2015
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -17,26 +17,17 @@
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
     You can contact the author at :
-    - LZ4 source repository : http://code.google.com/p/lz4/
-    - LZ4 public forum : https://group.google.com/forum/#!forum/lz4c
+    - LZ4 source repository : https://github.com/Cyan4973/lz4
+    - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 #pragma once
 
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
 
 /* Main function */
-int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel);
+int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel);
 
 /* Set Parameters */
-void BMK_SetBlocksize(int bsize);
-void BMK_SetNbIterations(int nbLoops);
-void BMK_SetPause(void);
+void BMK_setBlocksize(int bsize);
+void BMK_setNbIterations(int nbLoops);
+void BMK_setPause(void);
 
-
-
-#if defined (__cplusplus)
-}
-#endif

programs/datagen.c

@@ -19,26 +19,20 @@
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
     You can contact the author at :
-   - LZ4 source repository : http://code.google.com/p/lz4
-   - LZ4 source mirror : https://github.com/Cyan4973/lz4
-   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+   - ZSTD source repository : https://github.com/Cyan4973/zstd
+   - Public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
 /**************************************
- Remove Visual warning messages
+*  Includes
 **************************************/
-#define _CRT_SECURE_NO_WARNINGS   // fgets
+#include <stdlib.h>    /* malloc */
+#include <stdio.h>     /* FILE, fwrite */
+#include <string.h>    /* memcpy */
 
 
 /**************************************
- Includes
-**************************************/
-#include <stdio.h>      // fgets, sscanf
-#include <string.h>     // strcmp
-
-
-/**************************************
-   Basic Types
+*  Basic Types
 **************************************/
 #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
 # include <stdint.h>
@@ -57,230 +51,173 @@
 
 
 /**************************************
- Constants
+*  OS-specific Includes
 **************************************/
-#ifndef LZ4_VERSION
-#  define LZ4_VERSION "r125"
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+#  include <fcntl.h>   /* _O_BINARY */
+#  include <io.h>      /* _setmode, _isatty */
+#  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+#else
+#  define SET_BINARY_MODE(file)
 #endif
 
-#define KB *(1 <<10)
-#define MB *(1 <<20)
-#define GB *(1U<<30)
 
-#define CDG_SIZE_DEFAULT (64 KB)
-#define CDG_SEED_DEFAULT 0
-#define CDG_COMPRESSIBILITY_DEFAULT 50
+/**************************************
+*  Constants
+**************************************/
+#define KB *(1 <<10)
+
 #define PRIME1   2654435761U
 #define PRIME2   2246822519U
 
 
-/**************************************
-  Macros
-**************************************/
-#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-
-
-/**************************************
-  Local Parameters
-**************************************/
-static unsigned no_prompt = 0;
-static char*    programName;
-static unsigned displayLevel = 2;
-
-
 /*********************************************************
-  functions
+*  Local Functions
 *********************************************************/
-
-#define CDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-static unsigned int CDG_rand(U32* src)
+#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static unsigned int RDG_rand(U32* src)
 {
     U32 rand32 = *src;
     rand32 *= PRIME1;
-    rand32 += PRIME2;
-    rand32  = CDG_rotl32(rand32, 13);
+    rand32 ^= PRIME2;
+    rand32  = RDG_rotl32(rand32, 13);
     *src = rand32;
     return rand32;
 }
 
 
-#define CDG_RAND15BITS  ((CDG_rand(seed) >> 3) & 32767)
-#define CDG_RANDLENGTH  ( ((CDG_rand(seed) >> 7) & 3) ? (CDG_rand(seed) % 14) : (CDG_rand(seed) & 511) + 15)
-#define CDG_RANDCHAR    (((CDG_rand(seed) >> 9) & 63) + '0')
-static void CDG_generate(U64 size, U32* seed, double proba)
+#define LTSIZE 8192
+#define LTMASK (LTSIZE-1)
+static void* RDG_createLiteralDistrib(double ld)
 {
-    BYTE fullbuff[32 KB + 128 KB + 1];
-    BYTE* buff = fullbuff + 32 KB;
-    U64 total=0;
-    U32 P32 = (U32)(32768 * proba);
-    U32 pos=1;
-    U32 genBlockSize = 128 KB;
+    BYTE* lt = (BYTE*)malloc(LTSIZE);
+    U32 i = 0;
+    BYTE character = '0';
+    BYTE firstChar = '(';
+    BYTE lastChar = '}';
 
-    // Build initial prefix
-    fullbuff[0] = CDG_RANDCHAR;
-    while (pos<32 KB)
+    if (ld==0.0)
     {
-        // Select : Literal (char) or Match (within 32K)
-        if (CDG_RAND15BITS < P32)
+        character = 0;
+        firstChar = 0;
+        lastChar =255;
+    }
+    while (i<LTSIZE)
+    {
+        U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
+        U32 end;
+        if (weight + i > LTSIZE) weight = LTSIZE-i;
+        end = i + weight;
+        while (i < end) lt[i++] = character;
+        character++;
+        if (character > lastChar) character = firstChar;
+    }
+    return lt;
+}
+
+static char RDG_genChar(U32* seed, const void* ltctx)
+{
+    const BYTE* lt = (const BYTE*)ltctx;
+    U32 id = RDG_rand(seed) & LTMASK;
+    return lt[id];
+}
+
+#define RDG_DICTSIZE    (32 KB)
+#define RDG_RAND15BITS  ((RDG_rand(seed) >> 3) & 32767)
+#define RDG_RANDLENGTH  ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
+void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, void* litTable, unsigned* seedPtr)
+{
+    BYTE* buffPtr = (BYTE*)buffer;
+    const U32 matchProba32 = (U32)(32768 * matchProba);
+    size_t pos = prefixSize;
+    void* ldctx = litTable;
+    U32* seed = seedPtr;
+
+    /* special case */
+    while (matchProba >= 1.0)
+    {
+        size_t size0 = RDG_rand(seed) & 3;
+        size0  = (size_t)1 << (16 + size0 * 2);
+        size0 += RDG_rand(seed) & (size0-1);   /* because size0 is power of 2*/
+        if (buffSize < pos + size0)
         {
-            // Copy (within 64K)
-            U32 d;
-            int ref;
-            int length = CDG_RANDLENGTH + 4;
-            U32 offset = CDG_RAND15BITS + 1;
-            if (offset > pos) offset = pos;
-            ref = pos - offset;
+            memset(buffPtr+pos, 0, buffSize-pos);
+            return;
+        }
+        memset(buffPtr+pos, 0, size0);
+        pos += size0;
+        buffPtr[pos-1] = RDG_genChar(seed, ldctx);
+    }
+
+    /* init */
+    if (pos==0) buffPtr[0] = RDG_genChar(seed, ldctx), pos=1;
+
+    /* Generate compressible data */
+    while (pos < buffSize)
+    {
+        /* Select : Literal (char) or Match (within 32K) */
+        if (RDG_RAND15BITS < matchProba32)
+        {
+            /* Copy (within 32K) */
+            size_t match;
+            size_t d;
+            int length = RDG_RANDLENGTH + 4;
+            U32 offset = RDG_RAND15BITS + 1;
+            if (offset > pos) offset = (U32)pos;
+            match = pos - offset;
             d = pos + length;
-            while (pos < d) fullbuff[pos++] = fullbuff[ref++];
+            if (d > buffSize) d = buffSize;
+            while (pos < d) buffPtr[pos++] = buffPtr[match++];
         }
         else
         {
-            // Literal (noise)
-            U32 d = pos + CDG_RANDLENGTH;
-            while (pos < d) fullbuff[pos++] = CDG_RANDCHAR;
+            /* Literal (noise) */
+            size_t d;
+            size_t length = RDG_RANDLENGTH;
+            d = pos + length;
+            if (d > buffSize) d = buffSize;
+            while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldctx);
         }
     }
+}
 
-    // Generate compressible data
-    pos = 0;
+
+void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
+{
+    void* ldctx;
+    if (litProba==0.0) litProba = matchProba / 4.5;
+    ldctx = RDG_createLiteralDistrib(litProba);
+    RDG_genBlock(buffer, size, 0, matchProba, ldctx, &seed);
+    free(ldctx);
+}
+
+
+#define RDG_BLOCKSIZE (128 KB)
+void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed)
+{
+    BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE];
+    U64 total = 0;
+    size_t genBlockSize = RDG_BLOCKSIZE;
+    void* ldctx;
+
+    /* init */
+    if (litProba==0.0) litProba = matchProba / 4.5;
+    ldctx = RDG_createLiteralDistrib(litProba);
+    SET_BINARY_MODE(stdout);
+
+    /* Generate dict */
+    RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldctx, &seed);
+
+    /* Generate compressible data */
     while (total < size)
     {
-        if (size-total < 128 KB) genBlockSize = (U32)(size-total);
+        RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldctx, &seed);
+        if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total);
         total += genBlockSize;
-        buff[genBlockSize] = 0;
-        pos = 0;
-        while (pos<genBlockSize)
-        {
-            // Select : Literal (char) or Match (within 32K)
-            if (CDG_RAND15BITS < P32)
-            {
-                // Copy (within 64K)
-                int ref;
-                U32 d;
-                int length = CDG_RANDLENGTH + 4;
-                U32 offset = CDG_RAND15BITS + 1;
-                if (pos + length > genBlockSize ) length = genBlockSize - pos;
-                ref = pos - offset;
-                d = pos + length;
-                while (pos < d) buff[pos++] = buff[ref++];
-            }
-            else
-            {
-                // Literal (noise)
-                U32 d;
-                int length = CDG_RANDLENGTH;
-                if (pos + length > genBlockSize) length = genBlockSize - pos;
-                d = pos + length;
-                while (pos < d) buff[pos++] = CDG_RANDCHAR;
-            }
-        }
-        // output datagen
-        pos=0;
-        for (;pos+512<=genBlockSize;pos+=512)
-            printf("%512.512s", buff+pos);
-        for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
-        // Regenerate prefix
-        memcpy(fullbuff, buff + 96 KB, 32 KB);
-    }
-}
-
-
-int CDG_usage(void)
-{
-    DISPLAY( "Compressible data generator\n");
-    DISPLAY( "Usage :\n");
-    DISPLAY( "      %s [size] [args]\n", programName);
-    DISPLAY( "\n");
-    DISPLAY( "Arguments :\n");
-    DISPLAY( " -g#    : generate # data (default:%i)\n", CDG_SIZE_DEFAULT);
-    DISPLAY( " -s#    : Select seed (default:%i)\n", CDG_SEED_DEFAULT);
-    DISPLAY( " -p#    : Select compressibility in %% (default:%i%%)\n", CDG_COMPRESSIBILITY_DEFAULT);
-    DISPLAY( " -h     : display help and exit\n");
-    return 0;
-}
-
-
-int main(int argc, char** argv)
-{
-    int argNb;
-    int proba = CDG_COMPRESSIBILITY_DEFAULT;
-    U64 size = CDG_SIZE_DEFAULT;
-    U32 seed = CDG_SEED_DEFAULT;
-
-    // Check command line
-    programName = argv[0];
-    for(argNb=1; argNb<argc; argNb++)
-    {
-        char* argument = argv[argNb];
-
-        if(!argument) continue;   // Protection if argument empty
-
-        // Decode command (note : aggregated commands are allowed)
-        if (*argument=='-')
-        {
-            if (!strcmp(argument, "--no-prompt")) { no_prompt=1; continue; }
-
-            argument++;
-            while (*argument!=0)
-            {
-                switch(*argument)
-                {
-                case 'h':
-                    return CDG_usage();
-                case 'g':
-                    argument++;
-                    size=0;
-                    while ((*argument>='0') && (*argument<='9'))
-                    {
-                        size *= 10;
-                        size += *argument - '0';
-                        argument++;
-                    }
-                    if (*argument=='K') { size <<= 10; argument++; }
-                    if (*argument=='M') { size <<= 20; argument++; }
-                    if (*argument=='G') { size <<= 30; argument++; }
-                    if (*argument=='B') { argument++; }
-                    break;
-                case 's':
-                    argument++;
-                    seed=0;
-                    while ((*argument>='0') && (*argument<='9'))
-                    {
-                        seed *= 10;
-                        seed += *argument - '0';
-                        argument++;
-                    }
-                    break;
-                case 'p':
-                    argument++;
-                    proba=0;
-                    while ((*argument>='0') && (*argument<='9'))
-                    {
-                        proba *= 10;
-                        proba += *argument - '0';
-                        argument++;
-                    }
-                    if (proba<0) proba=0;
-                    if (proba>100) proba=100;
-                    break;
-                case 'v':
-                    displayLevel = 4;
-                    argument++;
-                    break;
-                default: ;
-                }
-            }
-
-        }
+        fwrite(buff, 1, genBlockSize, stdout);
+        /* update dict */
+        memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
     }
 
-    // Get Seed
-    DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION);
-    DISPLAYLEVEL(3, "Seed = %u \n", seed);
-    if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
-
-    CDG_generate(size, &seed, ((double)proba) / 100);
-
-    return 0;
+    free(ldctx);
 }

programs/datagen.h

@@ -0,0 +1,40 @@
+/*
+    datagen.h - compressible data generator header
+    Copyright (C) Yann Collet 2012-2015
+
+    GPL v2 License
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+   - ZSTD source repository : https://github.com/Cyan4973/zstd
+   - Public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+
+#include <stddef.h>   /* size_t */
+
+void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed);
+void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed);
+/* RDG_genOut
+   Generate 'size' bytes of compressible data into stdout.
+   Compressibility can be controlled using 'matchProba'.
+   'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used.
+   Generated data can be selected using 'seed'.
+   If (matchProba, litProba and seed) are equal, the function always generate the same content.
+
+   RDG_genBuffer
+   Same as RDG_genOut, but generate data into provided buffer
+*/

programs/datagencli.c

@@ -0,0 +1,193 @@
+/*
+    datagencli.c
+    compressible data command line generator
+    Copyright (C) Yann Collet 2012-2015
+
+    GPL v2 License
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    You can contact the author at :
+   - ZSTD source repository : https://github.com/Cyan4973/zstd
+   - Public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+/**************************************
+*  Includes
+**************************************/
+#include <stdio.h>     /* fprintf, stderr */
+#include "datagen.h"   /* RDG_generate */
+
+
+/**************************************
+*  Basic Types
+**************************************/
+#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
+# include <stdint.h>
+  typedef  uint8_t BYTE;
+  typedef uint16_t U16;
+  typedef uint32_t U32;
+  typedef  int32_t S32;
+  typedef uint64_t U64;
+#else
+  typedef unsigned char       BYTE;
+  typedef unsigned short      U16;
+  typedef unsigned int        U32;
+  typedef   signed int        S32;
+  typedef unsigned long long  U64;
+#endif
+
+
+/**************************************
+*  Constants
+**************************************/
+#ifndef ZSTD_VERSION
+#  define ZSTD_VERSION "r1"
+#endif
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define SIZE_DEFAULT (64 KB)
+#define SEED_DEFAULT 0
+#define COMPRESSIBILITY_DEFAULT 50
+
+
+/**************************************
+*  Macros
+**************************************/
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static unsigned displayLevel = 2;
+
+
+/*********************************************************
+*  Command line
+*********************************************************/
+static int usage(char* programName)
+{
+    DISPLAY( "Compressible data generator\n");
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [size] [args]\n", programName);
+    DISPLAY( "\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -g#    : generate # data (default:%i)\n", SIZE_DEFAULT);
+    DISPLAY( " -s#    : Select seed (default:%i)\n", SEED_DEFAULT);
+    DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT);
+    DISPLAY( " -h     : display help and exit\n");
+    DISPLAY( "Special values :\n");
+    DISPLAY( " -P0    : generate incompressible noise\n");
+    DISPLAY( " -P100  : generate sparse files\n");
+    return 0;
+}
+
+
+int main(int argc, char** argv)
+{
+    int argNb;
+    double proba = (double)COMPRESSIBILITY_DEFAULT / 100;
+    double litProba = 0.0;
+    U64 size = SIZE_DEFAULT;
+    U32 seed = SEED_DEFAULT;
+    char* programName;
+
+    /* Check command line */
+    programName = argv[0];
+    for(argNb=1; argNb<argc; argNb++)
+    {
+        char* argument = argv[argNb];
+
+        if(!argument) continue;   /* Protection if argument empty */
+
+        /* Handle commands. Aggregated commands are allowed */
+        if (*argument=='-')
+        {
+            argument++;
+            while (*argument!=0)
+            {
+                switch(*argument)
+                {
+                case 'h':
+                    return usage(programName);
+                case 'g':
+                    argument++;
+                    size=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        size *= 10;
+                        size += *argument - '0';
+                        argument++;
+                    }
+                    if (*argument=='K') { size <<= 10; argument++; }
+                    if (*argument=='M') { size <<= 20; argument++; }
+                    if (*argument=='G') { size <<= 30; argument++; }
+                    if (*argument=='B') { argument++; }
+                    break;
+                case 's':
+                    argument++;
+                    seed=0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        seed *= 10;
+                        seed += *argument - '0';
+                        argument++;
+                    }
+                    break;
+                case 'P':
+                    argument++;
+                    proba=0.0;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        proba *= 10;
+                        proba += *argument - '0';
+                        argument++;
+                    }
+                    if (proba>100.) proba=100.;
+                    proba /= 100.;
+                    break;
+                case 'L':   /* hidden argument : Literal distribution probability */
+                    argument++;
+                    litProba=0.;
+                    while ((*argument>='0') && (*argument<='9'))
+                    {
+                        litProba *= 10;
+                        litProba += *argument - '0';
+                        argument++;
+                    }
+                    if (litProba>100.) litProba=100.;
+                    litProba /= 100.;
+                    break;
+                case 'v':
+                    displayLevel = 4;
+                    argument++;
+                    break;
+                default:
+                    return usage(programName);
+                }
+            }
+
+        }
+    }
+
+    DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
+    DISPLAYLEVEL(3, "Seed = %u \n", seed);
+    if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
+
+    RDG_genOut(size, proba, litProba, seed);
+    DISPLAYLEVEL(1, "\n");
+
+    return 0;
+}

programs/frametest.c

@@ -1,6 +1,7 @@
 /*
     frameTest - test tool for lz4frame
-    Copyright (C) Yann Collet 2014
+    Copyright (C) Yann Collet 2014-2015
+
     GPL v2 License
 
     This program is free software; you can redistribute it and/or modify
@@ -18,38 +19,43 @@
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
     You can contact the author at :
-    - LZ4 source repository : http://code.google.com/p/lz4/
+    - LZ4 source repository : https://github.com/Cyan4973/lz4
     - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
 /**************************************
-  Compiler specific
+*  Compiler specific
 **************************************/
-#define _CRT_SECURE_NO_WARNINGS   // fgets
 #ifdef _MSC_VER    /* Visual Studio */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 #  pragma warning(disable : 4146)        /* disable: C4146: minus unsigned expression */
 #endif
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#ifdef __GNUC__
-#  pragma GCC diagnostic ignored "-Wmissing-braces"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#  pragma GCC diagnostic ignored "-Wmissing-field-initializers"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
+
+/* S_ISREG & gettimeofday() are not supported by MSVC */
+#if defined(_MSC_VER) || defined(_WIN32)
+#  define FUZ_LEGACY_TIMER 1
 #endif
 
 
 /**************************************
-  Includes
+*  Includes
 **************************************/
-#include <stdlib.h>     // free
-#include <stdio.h>      // fgets, sscanf
-#include <sys/timeb.h>  // timeb
-#include <string.h>     // strcmp
+#include <stdlib.h>     /* malloc, free */
+#include <stdio.h>      /* fprintf */
+#include <string.h>     /* strcmp */
 #include "lz4frame_static.h"
-#include "xxhash.h"     // XXH64
+#include "xxhash.h"     /* XXH64 */
+
+/* Use ftime() if gettimeofday() is not available on your target */
+#if defined(FUZ_LEGACY_TIMER)
+#  include <sys/timeb.h>   /* timeb, ftime */
+#else
+#  include <sys/time.h>    /* gettimeofday */
+#endif
 
 
 /**************************************
-   Basic Types
+*  Basic Types
 **************************************/
 #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
 # include <stdint.h>
@@ -67,13 +73,26 @@ typedef unsigned long long  U64;
 #endif
 
 
+/* unoptimized version; solves endianess & alignment issues */
+static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
+{
+    BYTE* dstPtr = (BYTE*)dstVoidPtr;
+    dstPtr[0] = (BYTE)value32;
+    dstPtr[1] = (BYTE)(value32 >> 8);
+    dstPtr[2] = (BYTE)(value32 >> 16);
+    dstPtr[3] = (BYTE)(value32 >> 24);
+}
+
+
 /**************************************
- Constants
+*  Constants
 **************************************/
 #ifndef LZ4_VERSION
 #  define LZ4_VERSION ""
 #endif
 
+#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
+
 #define KB *(1U<<10)
 #define MB *(1U<<20)
 #define GB *(1U<<30)
@@ -87,7 +106,7 @@ static const U32 prime2 = 2246822519U;
 
 
 /**************************************
-  Macros
+*  Macros
 **************************************/
 #define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...)  if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
@@ -100,7 +119,7 @@ static U32 g_time = 0;
 
 
 /*****************************************
-  Local Parameters
+*  Local Parameters
 *****************************************/
 static U32 no_prompt = 0;
 static char* programName;
@@ -109,8 +128,10 @@ static U32 pause = 0;
 
 
 /*********************************************************
-  Fuzzer functions
+*  Fuzzer functions
 *********************************************************/
+#if defined(FUZ_LEGACY_TIMER)
+
 static U32 FUZ_GetMilliStart(void)
 {
     struct timeb tb;
@@ -120,6 +141,19 @@ static U32 FUZ_GetMilliStart(void)
     return nCount;
 }
 
+#else
+
+static U32 FUZ_GetMilliStart(void)
+{
+    struct timeval tv;
+    U32 nCount;
+    gettimeofday(&tv, NULL);
+    nCount = (U32) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
+    return nCount;
+}
+
+#endif
+
 
 static U32 FUZ_GetMilliSpan(U32 nTimeStart)
 {
@@ -151,15 +185,15 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, d
     unsigned pos = 0;
     U32 P32 = (U32)(32768 * proba);
 
-    // First Byte
+    /* First Byte */
     BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
 
     while (pos < bufferSize)
     {
-        // Select : Literal (noise) or copy (within 64K)
+        /* Select : Literal (noise) or copy (within 64K) */
         if (FUZ_RAND15BITS < P32)
         {
-            // Copy (within 64K)
+            /* Copy (within 64K) */
             unsigned match, end;
             unsigned length = FUZ_RANDLENGTH + 4;
             unsigned offset = FUZ_RAND15BITS + 1;
@@ -171,7 +205,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, unsigned bufferSize, d
         }
         else
         {
-            // Literal (noise)
+            /* Literal (noise) */
             unsigned end;
             unsigned length = FUZ_RANDLENGTH;
             if (pos + length > bufferSize) length = bufferSize - pos;
@@ -203,11 +237,12 @@ int basicTests(U32 seed, double compressibility)
     void* decodedBuffer;
     U32 randState = seed;
     size_t cSize, testSize;
-    LZ4F_preferences_t prefs = { 0 };
+    LZ4F_preferences_t prefs;
     LZ4F_decompressionContext_t dCtx;
     U64 crcOrig;
 
     // Create compressible test buffer
+    memset(&prefs, 0, sizeof(prefs));
     CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
     compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL));
     decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
@@ -236,6 +271,7 @@ int basicTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(3, "Single Block : \n");
         errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
+        if (LZ4F_isError(errorCode)) goto _output_error;
         crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
         if (crcDest != crcOrig) goto _output_error;
         DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize);
@@ -345,6 +381,131 @@ int basicTests(U32 seed, double compressibility)
     if (LZ4F_isError(cSize)) goto _output_error;
     DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
 
+    {
+        size_t errorCode;
+        BYTE* const ostart = (BYTE*)compressedBuffer;
+        BYTE* op = ostart;
+        LZ4F_compressionContext_t cctx;
+        errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+
+        DISPLAYLEVEL(3, "compress without frameSize : \n");
+        memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
+        errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+        op += errorCode;
+        errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+        op += errorCode;
+        errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+        DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
+
+        DISPLAYLEVEL(3, "compress with frameSize : \n");
+        prefs.frameInfo.frameOSize = testSize;
+        op = ostart;
+        errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+        op += errorCode;
+        errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+        op += errorCode;
+        errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+        DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
+
+        DISPLAYLEVEL(3, "compress with wrong frameSize : \n");
+        prefs.frameInfo.frameOSize = testSize+1;
+        op = ostart;
+        errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+        op += errorCode;
+        errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+        op += errorCode;
+        errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL);
+        if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); }
+        else
+            goto _output_error;
+
+        errorCode = LZ4F_freeCompressionContext(cctx);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+    }
+
+    DISPLAYLEVEL(3, "Skippable frame test : \n");
+    {
+        size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
+        unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
+        BYTE* op = (BYTE*)decodedBuffer;
+        BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
+        BYTE* ip = (BYTE*)compressedBuffer;
+        BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
+
+        LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+
+        /* generate skippable frame */
+        FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
+        FUZ_writeLE32(ip+4, (U32)cSize);
+
+        DISPLAYLEVEL(3, "random segment sizes : \n");
+        while (ip < iend)
+        {
+            unsigned nbBits = FUZ_rand(&randState) % maxBits;
+            size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
+            size_t oSize = oend-op;
+            if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
+            errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
+            if (LZ4F_isError(errorCode)) goto _output_error;
+            op += oSize;
+            ip += iSize;
+        }
+        DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
+
+        /* generate zero-size skippable frame */
+        DISPLAYLEVEL(3, "zero-size skippable frame\n");
+        ip = (BYTE*)compressedBuffer;
+        op = (BYTE*)decodedBuffer;
+        FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1);
+        FUZ_writeLE32(ip+4, 0);
+        iend = ip+8;
+
+        while (ip < iend)
+        {
+            unsigned nbBits = FUZ_rand(&randState) % maxBits;
+            size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
+            size_t oSize = oend-op;
+            if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
+            errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
+            if (LZ4F_isError(errorCode)) goto _output_error;
+            op += oSize;
+            ip += iSize;
+        }
+        DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
+
+        DISPLAYLEVEL(3, "Skippable frame header complete in first call \n");
+        ip = (BYTE*)compressedBuffer;
+        op = (BYTE*)decodedBuffer;
+        FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2);
+        FUZ_writeLE32(ip+4, 10);
+        iend = ip+18;
+        while (ip < iend)
+        {
+            size_t iSize = 10;
+            size_t oSize = 10;
+            if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
+            errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
+            if (LZ4F_isError(errorCode)) goto _output_error;
+            op += oSize;
+            ip += iSize;
+        }
+        DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
+
+        /* release memory */
+        errorCode = LZ4F_freeDecompressionContext(dCtx);
+        if (LZ4F_isError(errorCode)) goto _output_error;
+    }
+
     DISPLAY("Basic tests completed \n");
 _end:
     free(CNBuffer);
@@ -391,7 +552,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
 #   define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
                             DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }
 
-    // Create buffers
+    /* Create buffers */
     result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
     CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
     result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
@@ -400,14 +561,14 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
     CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
     compressedBuffer = malloc(LZ4F_compressFrameBound(srcDataLength, NULL));
     CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
-    decodedBuffer = malloc(srcDataLength);
+    decodedBuffer = calloc(1, srcDataLength);   /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
     CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
     FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
 
-    // jump to requested testNb
+    /* jump to requested testNb */
     for (testNb =0; testNb < startTest; testNb++) (void)FUZ_rand(&coreRand);   // sync randomizer
 
-    // main fuzzer loop
+    /* main fuzzer test loop */
     for ( ; testNb < nbTests; testNb++)
     {
         U32 randState = coreRand ^ prime1;
@@ -415,28 +576,41 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         unsigned BMId   = FUZ_rand(&randState) & 1;
         unsigned CCflag = FUZ_rand(&randState) & 1;
         unsigned autoflush = (FUZ_rand(&randState) & 7) == 2;
-        LZ4F_preferences_t prefs = { 0 };
-        LZ4F_compressOptions_t cOptions = { 0 };
-        LZ4F_decompressOptions_t dOptions = { 0 };
+        LZ4F_preferences_t prefs;
+        LZ4F_compressOptions_t cOptions;
+        LZ4F_decompressOptions_t dOptions;
         unsigned nbBits = (FUZ_rand(&randState) % (FUZ_highbit(srcDataLength-1) - 1)) + 1;
         size_t srcSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
         size_t srcStart = FUZ_rand(&randState) % (srcDataLength - srcSize);
+        U64 frameContentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
         size_t cSize;
         U64 crcOrig, crcDecoded;
         LZ4F_preferences_t* prefsPtr = &prefs;
 
-        (void)FUZ_rand(&coreRand);   // update rand seed
+        (void)FUZ_rand(&coreRand);   /* update seed */
+        memset(&prefs, 0, sizeof(prefs));
+        memset(&cOptions, 0, sizeof(cOptions));
+        memset(&dOptions, 0, sizeof(dOptions));
         prefs.frameInfo.blockMode = (blockMode_t)BMId;
         prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId;
         prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag;
+        prefs.frameInfo.frameOSize = frameContentSize;
         prefs.autoFlush = autoflush;
         prefs.compressionLevel = FUZ_rand(&randState) % 5;
-        if ((FUZ_rand(&randState)&0xF) == 1) prefsPtr = NULL;
+        if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
 
         DISPLAYUPDATE(2, "\r%5u   ", testNb);
-        crcOrig = XXH64((BYTE*)srcBuffer+srcStart, (U32)srcSize, 1);
+        crcOrig = XXH64((BYTE*)srcBuffer+srcStart, srcSize, 1);
 
-        if ((FUZ_rand(&randState)&0xF) == 2)
+        if ((FUZ_rand(&randState) & 0xFFF) == 0)
+        {
+            /* create a skippable frame (rare case) */
+            BYTE* op = (BYTE*)compressedBuffer;
+            FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15));
+            FUZ_writeLE32(op+4, srcSize);
+            cSize = srcSize+8;
+        }
+        else if ((FUZ_rand(&randState) & 0xF) == 2)
         {
             cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), (char*)srcBuffer + srcStart, srcSize, prefsPtr);
             CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
@@ -483,6 +657,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
             const BYTE* const iend = ip + cSize;
             BYTE* op = (BYTE*)decodedBuffer;
             BYTE* const oend = op + srcDataLength;
+            size_t totalOut = 0;
             unsigned maxBits = FUZ_highbit((U32)cSize);
             unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1;
             nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst;   /* 0=>0; 1=>1,2 */
@@ -497,22 +672,23 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 if (oSize > (size_t)(oend-op)) oSize = oend-op;
                 dOptions.stableDst = FUZ_rand(&randState) & 1;
                 if (nonContiguousDst==2) dOptions.stableDst = 0;
-                //if (ip == compressedBuffer+62073)                    DISPLAY("oSize : %i : pos %i \n", (int)oSize, (int)(op-(BYTE*)decodedBuffer));
                 result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
-                //if (op+oSize >= (BYTE*)decodedBuffer+94727)                    DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
-                //if ((int)result<0)                    DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
                 if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
                 CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result));
                 XXH64_update(&xxh64, op, (U32)oSize);
+                totalOut += oSize;
                 op += oSize;
                 ip += iSize;
                 op += nonContiguousDst;
-                if (nonContiguousDst==2) op = decodedBuffer;   // overwritten destination
+                if (nonContiguousDst==2) op = (BYTE*)decodedBuffer;   /* overwritten destination */
             }
             CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
-            crcDecoded = XXH64_digest(&xxh64);
-            if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
-            CHECK(crcDecoded != crcOrig, "Decompression corruption");
+            if (totalOut)   /* otherwise, it's a skippable frame */
+            {
+                crcDecoded = XXH64_digest(&xxh64);
+                if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
+                CHECK(crcDecoded != crcOrig, "Decompression corruption");
+            }
         }
     }
 

programs/fullbench.c

@@ -19,17 +19,16 @@
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
     You can contact the author at :
-    - LZ4 source repository : http://code.google.com/p/lz4
-    - LZ4 source mirror : https://github.com/Cyan4973/lz4
+    - LZ4 source repository : https://github.com/Cyan4973/lz4
     - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 
-//**************************************
-// Compiler Options
-//**************************************
-// Disable some Visual warning messages
+/**************************************
+*  Compiler Options
+**************************************/
+/* Disable some Visual warning messages */
 #define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE     // VS2005
+#define _CRT_SECURE_NO_DEPRECATE     /* VS2005 */
 
 // Unix Large Files support (>4GB)
 #if (defined(__sun__) && (!defined(__LP64__)))   // Sun Solaris 32-bits requires specific definitions
@@ -45,9 +44,9 @@
 #endif
 
 
-//**************************************
-// Includes
-//**************************************
+/**************************************
+*  Includes
+**************************************/
 #include <stdlib.h>      // malloc
 #include <stdio.h>       // fprintf, fopen, ftello64
 #include <sys/types.h>   // stat64
@@ -68,10 +67,10 @@
 #include "xxhash.h"
 
 
-//**************************************
-// Compiler Options
-//**************************************
-// S_ISREG & gettimeofday() are not supported by MSVC
+/**************************************
+*  Compiler Options
+**************************************/
+/* S_ISREG & gettimeofday() are not supported by MSVC */
 #if !defined(S_ISREG)
 #  define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
 #endif
@@ -82,9 +81,9 @@
 #endif
 
 
-//**************************************
-// Basic Types
-//**************************************
+/**************************************
+*  Basic Types
+**************************************/
 #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   // C99
 # include <stdint.h>
   typedef uint8_t  BYTE;
@@ -101,9 +100,9 @@
 #endif
 
 
-//****************************
-// Constants
-//****************************
+/**************************************
+*  Constants
+**************************************/
 #define PROGRAM_DESCRIPTION "LZ4 speed analyzer"
 #ifndef LZ4_VERSION
 #  define LZ4_VERSION ""
@@ -114,17 +113,21 @@
 #define NBLOOPS    6
 #define TIMELOOP   2500
 
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
 #define KNUTH      2654435761U
-#define MAX_MEM    (1984<<20)
-#define DEFAULT_CHUNKSIZE   (4<<20)
+#define MAX_MEM    (1984 MB)
+#define DEFAULT_CHUNKSIZE   (4 MB)
 
 #define ALL_COMPRESSORS 0
 #define ALL_DECOMPRESSORS 0
 
 
-//**************************************
-// Local structures
-//**************************************
+/**************************************
+*  Local structures
+**************************************/
 struct chunkParameters
 {
     U32   id;
@@ -135,9 +138,9 @@ struct chunkParameters
 };
 
 
-//**************************************
-// MACRO
-//**************************************
+/**************************************
+*  MACRO
+**************************************/
 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
 #define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__)
 
@@ -217,21 +220,26 @@ static int BMK_GetMilliSpan( int nTimeStart )
 
 static size_t BMK_findMaxMem(U64 requiredMem)
 {
-    size_t step = (64U<<20);   // 64 MB
+    size_t step = 64 MB;
     BYTE* testmem=NULL;
 
-    requiredMem = (((requiredMem >> 25) + 1) << 26);
+    requiredMem = (((requiredMem >> 26) + 1) << 26);
+    requiredMem += 2*step;
     if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
 
-    requiredMem += 2*step;
     while (!testmem)
     {
-        requiredMem -= step;
+        if (requiredMem > step) requiredMem -= step;
+        else requiredMem >>= 1;
         testmem = (BYTE*) malloc ((size_t)requiredMem);
     }
-
     free (testmem);
-    return (size_t) (requiredMem - step);
+
+    /* keep some space available */
+    if (requiredMem > step) requiredMem -= step;
+    else requiredMem >>= 1;
+
+    return (size_t)requiredMem;
 }
 
 
@@ -251,8 +259,127 @@ static U64 BMK_GetFileSize(char* infilename)
 
 
 /*********************************************************
-  Benchmark function
+*  Benchmark function
 *********************************************************/
+#ifdef __SSSE3__
+
+#include <tmmintrin.h>
+
+/* Idea proposed by Terje Mathisen */
+static BYTE stepSize16[17] = {16,16,16,15,16,15,12,14,16,9,10,11,12,13,14,15,16};
+static __m128i replicateTable[17] = {
+    {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+    {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+    {0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1},
+    {0,1,2,0,1,2,0,1,2,0,1,2,0,1,2,0},
+    {0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3},
+    {0,1,2,3,4,0,1,2,3,4,0,1,2,3,4,0},
+    {0,1,2,3,4,5,0,1,2,3,4,5,0,1,2,3},
+    {0,1,2,3,4,5,6,0,1,2,3,4,5,6,0,1},
+    {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7},
+    {0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6},
+    {0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5},
+    {0,1,2,3,4,5,6,7,8,9,10,0,1,2,3,4},
+    {0,1,2,3,4,5,6,7,8,9,10,11,0,1,2,3},
+    {0,1,2,3,4,5,6,7,8,9,10,11,12,0,1,2},
+    {0,1,2,3,4,5,6,7,8,9,10,11,12,13,0,1},
+    {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0},
+    {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}};
+static BYTE stepSize32[17] = {32,32,32,30,32,30,30,28,32,27,30,22,24,26,28,30,16};
+static __m128i replicateTable2[17] = {
+    {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+    {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},
+    {0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1},
+    {1,2,0,1,2,0,1,2,0,1,2,0,1,2,0,1},
+    {0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3},
+    {1,2,3,4,0,1,2,3,4,0,1,2,3,4,0,1},
+    {4,5,0,1,2,3,4,5,0,1,2,3,4,5,0,1},
+    {2,3,4,5,6,0,1,2,3,4,5,6,0,1,2,3},
+    {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7},
+    {7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4},
+    {6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1},
+    {5,6,7,8,9,10,0,1,2,3,4,5,6,7,8,9},
+    {4,5,6,7,8,9,10,11,0,1,2,3,4,5,6,7},
+    {3,4,5,6,7,8,9,10,11,12,0,1,2,3,4,5},
+    {2,3,4,5,6,7,8,9,10,11,12,13,0,1,2,3},
+    {1,2,3,4,5,6,7,8,9,10,11,12,13,14,0,1},
+    {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}};
+
+U32 lz4_decode_sse(BYTE* dest, BYTE* src, U32 srcLength)
+{
+    BYTE* d = dest, *e = src+srcLength;
+    unsigned token, lit_len, mat_len;
+    __m128i a;
+    BYTE* dstore, *msrc;
+
+    if (!srcLength) return 0;
+    goto start;
+
+    do {
+        U32 step;
+        unsigned mat_offset = src[0] + (src[1] << 8);
+        src += 2;
+        msrc = d - mat_offset;
+        if (mat_len == 15) {
+            do {
+                token = *src++;
+                mat_len += token;
+            } while (token == 255);
+        }
+        mat_len += 4;
+
+        dstore = d;
+        d += mat_len;
+
+        if (mat_offset <= 16)
+        { // Bulk store only!
+            __m128i a2;
+            a = _mm_loadu_si128((const __m128i *)msrc);
+            a2 = _mm_shuffle_epi8(a, replicateTable2[mat_offset]);
+            a = _mm_shuffle_epi8(a, replicateTable[mat_offset]);
+            step = stepSize32[mat_offset];
+            do {
+                _mm_storeu_si128((__m128i *)dstore, a);
+                _mm_storeu_si128((__m128i *)(dstore+16), a2);
+                dstore += step;
+            } while (dstore < d);
+        }
+        else
+        {
+            do
+            {
+                a = _mm_loadu_si128((const __m128i *)msrc);
+                _mm_storeu_si128((__m128i *)dstore, a);
+                msrc += sizeof(a);
+                dstore += sizeof(a);
+            } while (dstore < d);
+        }
+start:
+        token = *src++;
+        lit_len = token >> 4;
+        mat_len = token & 15;
+        if (token >= 0xf0) { // lit_len == 15
+            do {
+                token = *src++;
+                lit_len += token;
+            } while (token == 255);
+        }
+        dstore = d;
+        msrc = src;
+        d += lit_len;
+        src += lit_len;
+        do {
+            a = _mm_loadu_si128((const __m128i *)msrc);
+            _mm_storeu_si128((__m128i *)dstore, a);
+            msrc += sizeof(a);
+            dstore += sizeof(a);
+        } while (dstore < d);
+    } while (src < e);
+
+    return (U32)(d-dest);
+}
+#endif // __SSSE3__
+
 
 static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
 {
@@ -345,6 +472,7 @@ static int local_LZ4_saveDictHC(const char* in, char* out, int inSize)
 static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
+    //lz4_decode_sse((BYTE*)out, (BYTE*)in, inSize);
     LZ4_decompress_fast(in, out, outSize);
     return outSize;
 }
@@ -446,7 +574,9 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
 
       // Memory allocation & restrictions
       inFileSize = BMK_GetFileSize(inFileName);
+      if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; }
       benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2;
+      if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; }
       if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
       if (benchedSize < inFileSize)
       {
@@ -567,7 +697,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
                 milliTime = BMK_GetMilliStart();
                 while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
                 {
-                    if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer);
+                    if (initFunction!=NULL) ctx = (LZ4_stream_t*)initFunction(chunkP[0].origBuffer);
                     for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
                     {
                         chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
@@ -677,7 +807,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles)
 
                 PROGRESS("%1i- %-29.29s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
 
-                // CRC Checking
+                /* CRC Checking */
                 crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
                 if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }
             }

programs/fuzzer.c

@@ -25,7 +25,7 @@
 */
 
 /**************************************
-* Remove Visual warning messages
+*  Compiler options
 **************************************/
 #ifdef _MSC_VER    /* Visual Studio */
 #  define _CRT_SECURE_NO_WARNINGS    /* fgets */
@@ -34,21 +34,32 @@
 #  pragma warning(disable : 4310)    /* disable: C4310: constant char value > 127 */
 #endif
 
+/* S_ISREG & gettimeofday() are not supported by MSVC */
+#if defined(_MSC_VER) || defined(_WIN32)
+#  define FUZ_LEGACY_TIMER 1
+#endif
+
 
 /**************************************
-* Includes
+*  Includes
 **************************************/
 #include <stdlib.h>
 #include <stdio.h>      /* fgets, sscanf */
-#include <sys/timeb.h>  /* timeb */
 #include <string.h>     /* strcmp */
 #include "lz4.h"
 #include "lz4hc.h"
 #include "xxhash.h"
 
+/* Use ftime() if gettimeofday() is not available on your target */
+#if defined(FUZ_LEGACY_TIMER)
+#  include <sys/timeb.h>   /* timeb, ftime */
+#else
+#  include <sys/time.h>    /* gettimeofday */
+#endif
+
 
 /**************************************
-* Basic Types
+*  Basic Types
 **************************************/
 #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
 # include <stdint.h>
@@ -67,7 +78,7 @@ typedef unsigned long long  U64;
 
 
 /**************************************
-* Constants
+*  Constants
 **************************************/
 #ifndef LZ4_VERSION
 #  define LZ4_VERSION ""
@@ -88,7 +99,7 @@ typedef unsigned long long  U64;
 
 
 /*****************************************
-* Macros
+*  Macros
 *****************************************/
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
@@ -98,8 +109,10 @@ static U32 g_time = 0;
 
 
 /*********************************************************
-  Fuzzer functions
+*  Fuzzer functions
 *********************************************************/
+#if defined(FUZ_LEGACY_TIMER)
+
 static U32 FUZ_GetMilliStart(void)
 {
     struct timeb tb;
@@ -109,6 +122,20 @@ static U32 FUZ_GetMilliStart(void)
     return nCount;
 }
 
+#else
+
+static U32 FUZ_GetMilliStart(void)
+{
+    struct timeval tv;
+    U32 nCount;
+    gettimeofday(&tv, NULL);
+    nCount = (U32) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
+    return nCount;
+}
+
+#endif
+
+
 static U32 FUZ_GetMilliSpan(U32 nTimeStart)
 {
     U32 nCurrent = FUZ_GetMilliStart();
@@ -177,7 +204,7 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou
 #define BLOCKSIZE_I134   (32 MB)
 static int FUZ_AddressOverflow(void)
 {
-    char* buffers[MAX_NB_BUFF_I134+1] = {0};
+    char* buffers[MAX_NB_BUFF_I134+1];
     int i, nbBuff=0;
     int highAddress = 0;
 
@@ -296,6 +323,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
     U32 crcOrig, crcCheck;
     U32 coreRandState = seed;
     U32 randState = coreRandState ^ PRIME3;
+    int result = 0;
 
 
     // init
@@ -661,7 +689,6 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do
 
     // unalloc
     {
-        int result = 0;
 _exit:
         free(CNBuffer);
         free(compressedBuffer);
@@ -753,7 +780,7 @@ static void FUZ_unitTests(void)
                 FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
 
                 XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                crcNew = crcOrig = XXH64_digest(&xxhNew);
+                crcNew = XXH64_digest(&xxhNew);
                 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
 
                 // prepare next message
@@ -916,7 +943,7 @@ static void FUZ_unitTests(void)
                 FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
 
                 XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                crcNew = crcOrig = XXH64_digest(&xxhNew);
+                crcNew = XXH64_digest(&xxhNew);
                 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
 
                 // prepare next message
@@ -959,10 +986,10 @@ static void FUZ_unitTests(void)
                 FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
 
                 XXH64_update(&xxhNew, testVerify + dNext, messageSize);
-                crcNew = crcOrig = XXH64_digest(&xxhNew);
+                crcNew = XXH64_digest(&xxhNew);
                 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
 
-                // prepare next message
+                /* prepare next message */
                 dNext += messageSize;
                 totalMessageSize += messageSize;
                 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
@@ -970,62 +997,6 @@ static void FUZ_unitTests(void)
                 if (dNext + messageSize > dBufferSize) dNext = 0;
             }
         }
-
-        // long stream test ; Warning : very long test !
-        if (1)
-        {
-            XXH64_state_t crcOrigState;
-            XXH64_state_t crcNewState;
-            const U64 totalTestSize = 6ULL << 30;
-            U64 totalTestDone = 0;
-            size_t oldStart = 0;
-            size_t oldSize  = 0;
-            U32 segNb = 1;
-
-            DISPLAY("Long HC streaming test (%u MB)\n", (U32)(totalTestSize >> 20));
-            LZ4_resetStreamHC(&sHC, 0);
-
-            XXH64_reset(&crcOrigState, 0);
-            XXH64_reset(&crcNewState, 0);
-
-            while (totalTestDone < totalTestSize)
-            {
-                size_t testSize = (FUZ_rand(&randState) & 65535) + 1;
-                size_t testStart = FUZ_rand(&randState) & 65535;
-
-                FUZ_displayUpdate((U32)(totalTestDone >> 20));
-
-                if (testStart == oldStart + oldSize)   // Corner case not covered by this test (LZ4_decompress_safe_usingDict() limitation)
-                    testStart++;
-
-                XXH64_update(&crcOrigState, testInput + testStart, testSize);
-                crcOrig = XXH64_digest(&crcOrigState);
-
-                result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + testStart, testCompressed, (int)testSize, LZ4_compressBound((int)testSize));
-                FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
-
-                result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, (int)testSize, testInput + oldStart, (int)oldSize);
-                FUZ_CHECKTEST(result!=(int)testSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %u failed", segNb);
-
-                XXH64_update(&crcNewState, testVerify, testSize);
-                crcNew = XXH64_digest(&crcNewState);
-                if (crcOrig!=crcNew)
-                {
-                    size_t c=0;
-                    while (testVerify[c] == testInput[testStart+c]) c++;
-                    DISPLAY("Bad decompression at %u / %u \n", (U32)c, (U32)testSize);
-                }
-                FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %u corruption", segNb);
-
-                oldStart = testStart;
-                oldSize = testSize;
-                totalTestDone += testSize;
-
-                segNb ++;
-            }
-
-            DISPLAY("\r");
-        }
     }
 
     printf("All unit tests completed successfully \n");

programs/lz4.1

@@ -1,66 +1,165 @@
 \"
 \" lz4.1: This is a manual page for 'lz4' program. This file is part of the
 \" lz4 <https://code.google.com/p/lz4/> project.
+\" Author: Yann Collet
 \"
 
 \" No hyphenation
 .hy 0
 .nr HY 0
 
-.TH lz4 "1" "2014-02-27" "lz4" "User Commands"
+.TH lz4 "1" "2015-03-21" "lz4" "User Commands"
 .SH NAME
-\fBlz4\fR - Extremely fast compression algorithm
+\fBlz4, unlz4, lz4cat\fR \- Compress or decompress .lz4 files
 
 .SH SYNOPSIS
 .TP 5
 \fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
+.PP
+.B unlz4
+is equivalent to
+.BR "lz4 \-d" 
+.br
+.B lz4cat
+is equivalent to
+.BR "lz4 \-dc" 
+.br
+.PP
+When writing scripts that need to decompress files,
+it is recommended to always use the name
+.B lz4
+with appropriate arguments
+.RB ( "lz4 \-d"
+or
+.BR "lz4 \-dc" )
+instead of the names
+.B unlz4
+and
+.BR lz4cat .
+
 
 .SH DESCRIPTION
 .PP
-\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on
-the \fBLZ77\fR family of compression scheme. At the compression speed of 400
-MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features
-an extremely fast decoder, with speed in multiple GB/s per core, typically
-reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports
-following options
+\fBlz4\fR is an extremely fast lossless compression algorithm,
+based on \fBbyte-aligned LZ77\fR family of compression scheme.
+\fBlz4\fR offers compression speeds of 400 MB/s per core, linearly scalable with multi-core CPUs. 
+It features an extremely fast decoder, with speed in multiple GB/s per core,
+typically reaching RAM speed limit on multi-core systems. 
+.B lz4
+supports a command line syntax similar to
+.BR gzip (1).
+The native file format is the
+.B .lz4
+format.
+
+.SS "Concatenation of .lz4 files"
+It is possible to concatenate
+.B .lz4
+files as is.
+.B lz4
+will decompress such files as if they were a single
+.B .lz4
+file.
+.PP
 
 .SH OPTIONS
+.
+.SS "Short command concatenation"
+In some cases, some options can be expressed using short command
+.B "-x"
+or long command
+.B "--long-word" .
+Short commands can be concatenated together. For example,
+.B "-d -c"
+is equivalent to
+.B "-dc" .
+Long commands cannot be concatenated.
+They must be clearly separated by a space.
+.
+.SS "Operation mode"
+If multiple operation mode options are given,
+the last one takes effect.
+.TP
+.BR \-z ", " \-\-compress
+Compress.
+This is the default operation mode 
+when no operation mode option is specified ,
+no other operation mode is implied from the command name 
+(for example,
+.B unlz4
+implies
+.B \-\-decompress ),
+nor from the input file name
+(for example, a file extension
+.B .lz4
+implies
+.B \-\-decompress
+by default).
+.B -z
+can also be used to force compression of an already compressed
+.B .lz4
+file.
+.TP
+.BR \-d ", " \-\-decompress ", " \-\-uncompress
+Decompress.
+.B --decompress
+is also the default operation when the input filename has an
+.B .lz4
+extensionq
+.TP
+.BR \-t ", " \-\-test
+Test the integrity of compressed
+.B .lz4
+files.
+The decompressed data is discarded.
+No files are created or removed.
+.
+.SS "Operation modifiers"
 .TP
 .B \-1
  fast compression (default)
 .TP
 .B \-9
  high compression
+
 .TP
-.B \-d
- decompression
+.BR \-f ", " --force
+ This option has several effects:
+.RS
+.IP \(bu 3
+If the target file already exists,
+overwrite it without prompting.
+.IP \(bu 3
+When used with
+.B \-\-decompress
+and
+.B lz4
+cannot recognize the type of the source file,
+copy the source file as is to standard output.
+This allows
+.B lz4cat
+.B \-\-force
+to be used like
+.BR cat (1)
+for files that have not been compressed with
+.BR lz4 .
+.RE
+
 .TP
-.B \-f
- overwrite output without prompting
-.TP
-.B \-h/\-H
- display help/long help and exit
-.TP
-.B \-V
- display Version number and exit
-.TP
-.B \-v
- verbose mode
-.TP
-.B \-q
- suppress warnings; specify twice to suppress errors too
-.TP
-.B \-c
+.BR \-c ", " \--stdout ", " \--to-stdout
  force write to standard output, even if it is the console
+
 .TP
-.B \-t
- test compressed file integrity
-.TP
-.B \-z
- force compression
-.TP
-.B \-l
- use Legacy format (useful for Linux Kernel compression)
+.BR \-m
+ Multiple file names.
+ By default, the second filename is used as the output filename for the compressed file.
+ With
+.B -m
+, you can specify any number of input filenames, each of them will be compressed
+with the resulting compressed file named
+.B filename.lz4
+.
+
 .TP
 .B \-B#
  block size [4-7](default : 7)
@@ -69,12 +168,36 @@ following options
 .B \-BD
  block dependency (improve compression ratio)
 .TP
-.B \-BX
- enable block checksum (default:disabled)
-.TP
-.B \-Sx
+.B \--no-frame-crc
  disable stream checksum (default:enabled)
 .TP
+.B \--frame-content-size
+ compressed frame includes original size (default:not present)
+.TP
+.B \--sparse-support 
+ enable sparse file (default:disabled)(experimental)
+.TP
+.B \-l
+ use Legacy format (useful for Linux Kernel compression)
+.
+.SS "Other options"
+.TP
+.BR \-v ", " --verbose
+ verbose mode
+.TP
+.BR \-q ", " --quiet
+ suppress warnings; specify twice to suppress errors too
+.TP
+.B \-h/\-H
+ display help/long help and exit
+.TP
+.BR \-V ", " \--version
+ display Version number and exit
+.TP
+.BR \-k ", " \--keep
+ Don't delete source file.
+This is default behavior anyway, so this option is just for compatibility with gzip/xz.
+.TP
 .B \-b
  benchmark file(s)
 .TP
@@ -82,7 +205,7 @@ following options
  iteration loops [1-9](default : 3), benchmark mode only
 
 .SH BUGS
-Report bugs at:- https://code.google.com/p/lz4/
+Report bugs at: https://github.com/Cyan4973/lz4
 
 .SH AUTHOR
 Yann Collet

programs/lz4c.1

@@ -1,33 +0,0 @@
-\"
-\" lz4c.1: This is a manual page for 'lz4c' program. This file is part of the
-\" lz4 <https://code.google.com/p/lz4/> project.
-\"
-
-\" No hyphenation
-.hy 0
-.nr HY 0
-
-.TH lz4c "1" "2014-04-15" "lz4c" "User Commands"
-.SH NAME
-\fBlz4\fR - Extremely fast compression algorithm
-
-.SH SYNOPSIS
-.TP 5
-\fBlz4c\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
-
-.SH DESCRIPTION
-.PP
-\fBlz4c\fR is the legacy version of \fBlz4\fR.
-As such, it supports older supplementary legacy commands.
-\fBlz4c\fR is now deprecated.
-It is recommended to use \fBlz4\fR instead whenever possible.
-
-To get a list of commands specific to lz4c, do :
-lz4c -h
-
-
-.SH BUGS
-Report bugs at:- https://code.google.com/p/lz4/
-
-.SH AUTHOR
-Yann Collet

programs/lz4cat.1

@@ -1,32 +0,0 @@
-\"
-\" lz4cat.1: This is a manual page for 'lz4cat' program. This file is part of
-\" the lz4 <https://code.google.com/p/lz4/> project.
-\"
-
-\" No hyphenation
-.hy 0
-.nr HY 0
-
-.TH lz4cat "1" "2014-06-20" "lz4cat" "User Commands"
-.SH NAME
-\fBlz4cat\fR - Utility based on LZ4
-
-.SH SYNOPSIS
-.TP 5
-\fBlz4cat\fR [\fBOPTIONS\fR] [-|INPUT-FILE]
-
-.SH DESCRIPTION
-.PP
-\fBlz4cat\fR is an utility based on \fBlz4\fR, an extremely fast lossless compression algorithm.
-
-\fBlz4cat\fR decompress input file or stream, redirecting its output to the console.
-It is equivalent to \fBlz4 -cd\fR,
-
-Available options are the same as \fBlz4\fR ones (man lz4).
-
-
-.SH BUGS
-Report bugs at:- https://code.google.com/p/lz4/
-
-.SH AUTHOR
-Yann Collet

programs/lz4cli.c

@@ -1,6 +1,6 @@
 /*
   LZ4cli - LZ4 Command Line Interface
-  Copyright (C) Yann Collet 2011-2014
+  Copyright (C) Yann Collet 2011-2015
 
   GPL v2 License
 
@@ -19,7 +19,7 @@
   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
   You can contact the author at :
-  - LZ4 source repository : http://code.google.com/p/lz4/
+  - LZ4 source repository : https://github.com/Cyan4973/lz4
   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 /*
@@ -59,23 +59,20 @@
 #include <stdlib.h>   /* exit, calloc, free */
 #include <string.h>   /* strcmp, strlen */
 #include "bench.h"    /* BMK_benchFile, BMK_SetNbIterations, BMK_SetBlocksize, BMK_SetPause */
-#include "lz4io.h"
+#include "lz4io.h"    /* LZ4IO_compressFilename, LZ4IO_decompressFilename, LZ4IO_compressMultipleFilenames */
 
 
 /****************************
 *  OS-specific Includes
 *****************************/
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-#  include <fcntl.h>    /* _O_BINARY */
-#  include <io.h>       /* _setmode, _isatty */
+#  include <io.h>       /* _isatty */
 #  ifdef __MINGW32__
    int _fileno(FILE *stream);   /* MINGW somehow forgets to include this prototype into <stdio.h> */
 #  endif
-#  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
 #else
 #  include <unistd.h>   /* isatty */
-#  define SET_BINARY_MODE(file)
 #  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
 #endif
 
@@ -85,12 +82,13 @@
 ******************************/
 #define COMPRESSOR_NAME "LZ4 command line interface"
 #ifndef LZ4_VERSION
-#  define LZ4_VERSION "r126"
+#  define LZ4_VERSION "r128"
 #endif
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
 #define LZ4_EXTENSION ".lz4"
-#define LZ4_CAT "lz4cat"
+#define LZ4CAT "lz4cat"
+#define UNLZ4 "unlz4"
 
 #define KB *(1U<<10)
 #define MB *(1U<<20)
@@ -136,10 +134,10 @@ static char* programName;
 #define EXTENDED_FORMAT
 #define DEFAULT_COMPRESSOR   LZ4IO_compressFilename
 #define DEFAULT_DECOMPRESSOR LZ4IO_decompressFilename
-int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel);   /* hidden function */
+int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel);   /* hidden function */
 
 
-/****************************
+/*****************************
 *  Functions
 *****************************/
 static int usage(void)
@@ -170,11 +168,14 @@ static int usage_advanced(void)
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
     DISPLAY( " -t     : test compressed file integrity\n");
+    DISPLAY( " -m     : multiple input files (implies automatic output filenames)\n");
     DISPLAY( " -l     : compress using Legacy format (Linux kernel compression)\n");
     DISPLAY( " -B#    : Block size [4-7](default : 7)\n");
     DISPLAY( " -BD    : Block dependency (improve compression ratio)\n");
     /* DISPLAY( " -BX    : enable block checksum (default:disabled)\n");   *//* Option currently inactive */
-    DISPLAY( " -Sx    : disable stream checksum (default:enabled)\n");
+    DISPLAY( "--no-frame-crc       : disable stream checksum (default:enabled)\n");
+    DISPLAY( "--frame-content-size : compressed frame includes original size (default:not present)\n");
+    DISPLAY( "--sparse-support     : enable sparse file (default:disabled)(experimental)\n");
     DISPLAY( "Benchmark arguments :\n");
     DISPLAY( " -b     : benchmark file(s)\n");
     DISPLAY( " -i#    : iteration loops [1-9](default : 3), benchmark mode only\n");
@@ -184,7 +185,6 @@ static int usage_advanced(void)
     DISPLAY( " -c1    : high compression\n");
     DISPLAY( " -hc    : high compression\n");
     DISPLAY( " -y     : overwrite output without prompting \n");
-    DISPLAY( " -s     : suppress warnings \n");
 #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
     EXTENDED_HELP;
     return 0;
@@ -261,25 +261,28 @@ int main(int argc, char** argv)
         cLevel=0,
         decode=0,
         bench=0,
-        filenamesStart=2,
         legacy_format=0,
         forceStdout=0,
         forceCompress=0,
-        main_pause=0;
-    char* input_filename=0;
-    char* output_filename=0;
+        main_pause=0,
+        multiple_inputs=0;
+    const char* input_filename=0;
+    const char* output_filename=0;
     char* dynNameSpace=0;
+    const char** inFileNames = NULL;
+    unsigned ifnIdx=0;
     char nullOutput[] = NULL_OUTPUT;
     char extension[] = LZ4_EXTENSION;
-    int blockSize;
+    int  blockSize;
 
     /* Init */
     programName = argv[0];
     LZ4IO_setOverwrite(0);
     blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
 
-    /* lz4cat behavior */
-    if (!strcmp(programName, LZ4_CAT)) { decode=1; forceStdout=1; output_filename=stdoutmark; displayLevel=1; }
+    /* lz4cat predefined behavior */
+    if (!strcmp(programName, LZ4CAT)) { decode=1; forceStdout=1; output_filename=stdoutmark; displayLevel=1; }
+    if (!strcmp(programName, UNLZ4)) { decode=1; }
 
     /* command switches */
     for(i=1; i<argc; i++)
@@ -288,7 +291,23 @@ int main(int argc, char** argv)
 
         if(!argument) continue;   /* Protection if argument empty */
 
-        /* Decode command (note : aggregated commands are allowed) */
+        /* long commands (--long-word) */
+        if (!strcmp(argument, "--compress")) { forceCompress = 1; continue; }
+        if (!strcmp(argument, "--decompress")) { decode = 1; continue; }
+        if (!strcmp(argument, "--uncompress")) { decode = 1; continue; }
+        if (!strcmp(argument, "--test")) { decode = 1; LZ4IO_setOverwrite(1); output_filename=nulmark; continue; }
+        if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; }
+        if (!strcmp(argument, "--stdout")) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
+        if (!strcmp(argument, "--to-stdout")) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; }
+        if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; }
+        if (!strcmp(argument, "--frame-content-size")) { LZ4IO_setContentSize(1); continue; }
+        if (!strcmp(argument, "--sparse-support")) { LZ4IO_setSparseFile(1); continue; }
+        if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
+        if (!strcmp(argument, "--quiet")) { if (displayLevel) displayLevel--; continue; }
+        if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; }
+        if (!strcmp(argument, "--keep")) { continue; } /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */
+
+        /* Short commands (note : aggregated short commands are allowed) */
         if (argument[0]=='-')
         {
             /* '-' means stdin/stdout */
@@ -308,7 +327,6 @@ int main(int argc, char** argv)
                 if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; }  /* -c1 (high compression) */
                 if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; }  /* -hc (high compression) */
                 if (*argument=='y') { LZ4IO_setOverwrite(1); continue; }                           /* -y (answer 'yes' to overwrite permission) */
-                if (*argument=='s') { displayLevel=1; continue; }                                  /* -s (silent mode) */
 #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
 
                 if ((*argument>='0') && (*argument<='9'))
@@ -353,7 +371,7 @@ int main(int argc, char** argv)
                 case 'v': displayLevel=4; break;
 
                     /* Quiet mode */
-                case 'q': displayLevel--; break;
+                case 'q': if (displayLevel) displayLevel--; break;
 
                     /* keep source file (default anyway, so useless) (for xz/lzma compatibility) */
                 case 'k': break;
@@ -372,36 +390,42 @@ int main(int argc, char** argv)
                         {
                             int B = argument[1] - '0';
                             blockSize = LZ4IO_setBlockSizeID(B);
-                            BMK_SetBlocksize(blockSize);
+                            BMK_setBlocksize(blockSize);
                             argument++;
                             break;
                         }
                         case 'D': LZ4IO_setBlockMode(LZ4IO_blockLinked); argument++; break;
-                        case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break;   /* currently disables */
+                        case 'X': LZ4IO_setBlockChecksumMode(1); argument ++; break;   /* currently disabled */
                         default : exitBlockProperties=1;
                         }
                         if (exitBlockProperties) break;
                     }
                     break;
 
-                    /* Modify Stream properties */
-                case 'S': if (argument[1]=='x') { LZ4IO_setStreamChecksumMode(0); argument++; break; } else { badusage(); }
-
                     /* Benchmark */
-                case 'b': bench=1; break;
+                case 'b': bench=1; multiple_inputs=1;
+                    if (inFileNames == NULL)
+                        inFileNames = (const char**) malloc(argc * sizeof(char*));
+                    break;
+
+                    /* Treat non-option args as input files.  See https://code.google.com/p/lz4/issues/detail?id=151 */
+                case 'm': multiple_inputs=1;
+                    if (inFileNames == NULL)
+                        inFileNames = (const char**) malloc(argc * sizeof(char*));
+                    break;
 
                     /* Modify Nb Iterations (benchmark only) */
                 case 'i':
                     if ((argument[1] >='1') && (argument[1] <='9'))
                     {
                         int iters = argument[1] - '0';
-                        BMK_SetNbIterations(iters);
+                        BMK_setNbIterations(iters);
                         argument++;
                     }
                     break;
 
                     /* Pause at the end (hidden option) */
-                case 'p': main_pause=1; BMK_SetPause(); break;
+                case 'p': main_pause=1; BMK_setPause(); break;
 
                     /* Specific commands for customized versions */
                 EXTENDED_ARGUMENTS;
@@ -413,29 +437,36 @@ int main(int argc, char** argv)
             continue;
         }
 
-        /* first provided filename is input */
-        if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
+        /* Store in *inFileNames[] if -m is used. */
+        if (multiple_inputs) { inFileNames[ifnIdx++]=argument; continue; }
 
-        /* second provided filename is output */
+        /* Store first non-option arg in input_filename to preserve original cli logic. */
+        if (!input_filename) { input_filename=argument; continue; }
+
+        /* Second non-option arg in output_filename to preserve original cli logic. */
         if (!output_filename)
         {
             output_filename=argument;
             if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
             continue;
         }
+
+        /* 3rd non-option arg should not exist */
+        DISPLAYLEVEL(1, "Warning : %s won't be used ! Do you want multiple input files (-m) ? \n", argument);
     }
 
     DISPLAYLEVEL(3, WELCOME_MESSAGE);
     if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10);
 
     /* No input filename ==> use stdin */
+    if (multiple_inputs) input_filename = inFileNames[0], output_filename = (char*)(inFileNames[0]);
     if(!input_filename) { input_filename=stdinmark; }
 
     /* Check if input or output are defined as console; trigger an error in this case */
     if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
 
     /* Check if benchmark is selected */
-    if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel);
+    if (bench) return BMK_benchFiles(inFileNames, ifnIdx, cLevel);
 
     /* No output filename ==> try to select one automatically (when possible) */
     while (!output_filename)
@@ -450,9 +481,9 @@ int main(int argc, char** argv)
         {
             size_t l = strlen(input_filename);
             dynNameSpace = (char*)calloc(1,l+5);
+            strcpy(dynNameSpace, input_filename);
+            strcat(dynNameSpace, LZ4_EXTENSION);
             output_filename = dynNameSpace;
-            strcpy(output_filename, input_filename);
-            strcpy(output_filename+l, LZ4_EXTENSION);
             DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
             break;
         }
@@ -461,12 +492,12 @@ int main(int argc, char** argv)
             size_t outl;
             size_t inl = strlen(input_filename);
             dynNameSpace = (char*)calloc(1,inl+1);
-            output_filename = dynNameSpace;
-            strcpy(output_filename, input_filename);
+            strcpy(dynNameSpace, input_filename);
             outl = inl;
             if (inl>4)
-                while ((outl >= inl-4) && (input_filename[outl] ==  extension[outl-inl+4])) output_filename[outl--]=0;
+                while ((outl >= inl-4) && (input_filename[outl] ==  extension[outl-inl+4])) dynNameSpace[outl--]=0;
             if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); }
+            output_filename = dynNameSpace;
             DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
         }
     }
@@ -491,11 +522,15 @@ int main(int argc, char** argv)
         }
         else
         {
-            DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
+            if (multiple_inputs)
+                LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel);
+            else
+                DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
         }
     }
 
     if (main_pause) waitEnter();
     free(dynNameSpace);
+    free((void*)inFileNames);
     return 0;
 }

programs/lz4io.c

@@ -1,6 +1,7 @@
 /*
   LZ4io.c - LZ4 File/Stream Interface
-  Copyright (C) Yann Collet 2011-2014
+  Copyright (C) Yann Collet 2011-2015
+
   GPL v2 License
 
   This program is free software; you can redistribute it and/or modify
@@ -18,7 +19,7 @@
   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
   You can contact the author at :
-  - LZ4 source repository : http://code.google.com/p/lz4/
+  - LZ4 source repository : https://github.com/Cyan4973/lz4
   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 /*
@@ -31,56 +32,51 @@
 
 /**************************************
 *  Compiler Options
-***************************************/
+**************************************/
 #ifdef _MSC_VER    /* Visual Studio */
 #  define _CRT_SECURE_NO_WARNINGS
 #  define _CRT_SECURE_NO_DEPRECATE     /* VS2005 */
 #  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
 #endif
 
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#ifdef __GNUC__
-#  pragma GCC diagnostic ignored "-Wmissing-braces"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#  pragma GCC diagnostic ignored "-Wmissing-field-initializers"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
-#endif
-
 #define _LARGE_FILES           /* Large file support on 32-bits AIX */
 #define _FILE_OFFSET_BITS 64   /* Large file support on 32-bits unix */
-#define _POSIX_SOURCE 1        /* for fileno() within <stdio.h> on unix */
 
 
-/****************************
+/*****************************
 *  Includes
 *****************************/
-#include <stdio.h>    /* fprintf, fopen, fread, _fileno, stdin, stdout */
-#include <stdlib.h>   /* malloc, free */
-#include <string.h>   /* strcmp, strlen */
-#include <time.h>     /* clock */
+#include <stdio.h>     /* fprintf, fopen, fread, stdin, stdout */
+#include <stdlib.h>    /* malloc, free */
+#include <string.h>    /* strcmp, strlen */
+#include <time.h>      /* clock */
+#include <sys/types.h> /* stat64 */
+#include <sys/stat.h>  /* stat64 */
 #include "lz4io.h"
 #include "lz4.h"      /* still required for legacy format */
 #include "lz4hc.h"    /* still required for legacy format */
 #include "lz4frame.h"
 
 
-/****************************
+/******************************
 *  OS-specific Includes
-*****************************/
+******************************/
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
 #  include <fcntl.h>   /* _O_BINARY */
-#  include <io.h>      /* _setmode, _isatty */
-#  ifdef __MINGW32__
-   int _fileno(FILE *stream);   /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
-#  endif
+#  include <io.h>      /* _setmode, _fileno, _get_osfhandle */
 #  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
-#  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#  include <Windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
+#  define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
+#  if defined(_MSC_VER) && (_MSC_VER >= 1400)  /* Avoid MSVC fseek()'s 2GiB barrier */
+#    define fseek _fseeki64
+#  endif
 #else
-#  include <unistd.h>  /* isatty */
 #  define SET_BINARY_MODE(file)
-#  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#  define SET_SPARSE_FILE_MODE(file)
 #endif
 
 
-/****************************
+/*****************************
 *  Constants
 *****************************/
 #define KB *(1 <<10)
@@ -93,43 +89,46 @@
 #define _4BITS 0x0F
 #define _8BITS 0xFF
 
-#define MAGICNUMBER_SIZE   4
-#define LZ4S_MAGICNUMBER   0x184D2204
-#define LZ4S_SKIPPABLE0    0x184D2A50
-#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0
-#define LEGACY_MAGICNUMBER 0x184C2102
+#define MAGICNUMBER_SIZE    4
+#define LZ4IO_MAGICNUMBER   0x184D2204
+#define LZ4IO_SKIPPABLE0    0x184D2A50
+#define LZ4IO_SKIPPABLEMASK 0xFFFFFFF0
+#define LEGACY_MAGICNUMBER  0x184C2102
 
 #define CACHELINE 64
 #define LEGACY_BLOCKSIZE   (8 MB)
 #define MIN_STREAM_BUFSIZE (192 KB)
-#define LZ4S_BLOCKSIZEID_DEFAULT 7
-#define LZ4S_CHECKSUM_SEED 0
-#define LZ4S_EOS 0
-#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1)
+#define LZ4IO_BLOCKSIZEID_DEFAULT 7
+
+#define sizeT sizeof(size_t)
+#define maskT (sizeT - 1)
 
 
 /**************************************
 *  Macros
-***************************************/
+**************************************/
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
-            if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (displayLevel>=4)) \
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static int g_displayLevel = 0;   /* 0 : no display  ; 1: errors  ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+            if ((LZ4IO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
             { g_time = clock(); DISPLAY(__VA_ARGS__); \
-            if (displayLevel>=4) fflush(stdout); } }
+            if (g_displayLevel>=4) fflush(stdout); } }
 static const unsigned refreshRate = 150;
 static clock_t g_time = 0;
 
 
 /**************************************
 *  Local Parameters
-***************************************/
-static int displayLevel = 0;   /* 0 : no display  ; 1: errors  ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
-static int overwrite = 1;
-static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
-static int blockChecksum = 0;
-static int streamChecksum = 1;
-static int blockIndependence = 1;
+**************************************/
+static int g_overwrite = 1;
+static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT;
+static int g_blockChecksum = 0;
+static int g_streamChecksum = 1;
+static int g_blockIndependence = 1;
+static int g_sparseFileSupport = 0;
+static int g_contentSizeFlag = 0;
 
 static const int minBlockSizeID = 4;
 static const int maxBlockSizeID = 7;
@@ -152,11 +151,10 @@ static const int maxBlockSizeID = 7;
 
 /**************************************
 *  Version modifiers
-***************************************/
+**************************************/
 #define EXTENDED_ARGUMENTS
 #define EXTENDED_HELP
 #define EXTENDED_FORMAT
-#define DEFAULT_COMPRESSOR   compress_file
 #define DEFAULT_DECOMPRESSOR decodeLZ4S
 
 
@@ -167,8 +165,8 @@ static const int maxBlockSizeID = 7;
 /* Default setting : overwrite = 1; return : overwrite mode (0/1) */
 int LZ4IO_setOverwrite(int yes)
 {
-   overwrite = (yes!=0);
-   return overwrite;
+   g_overwrite = (yes!=0);
+   return g_overwrite;
 }
 
 /* blockSizeID : valid values : 4-5-6-7 */
@@ -176,35 +174,49 @@ int LZ4IO_setBlockSizeID(int bsid)
 {
     static const int blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB };
     if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return -1;
-    globalBlockSizeId = bsid;
-    return blockSizeTable[globalBlockSizeId-minBlockSizeID];
+    g_blockSizeId = bsid;
+    return blockSizeTable[g_blockSizeId-minBlockSizeID];
 }
 
 int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode)
 {
-    blockIndependence = (blockMode == LZ4IO_blockIndependent);
-    return blockIndependence;
+    g_blockIndependence = (blockMode == LZ4IO_blockIndependent);
+    return g_blockIndependence;
 }
 
 /* Default setting : no checksum */
 int LZ4IO_setBlockChecksumMode(int xxhash)
 {
-    blockChecksum = (xxhash != 0);
-    return blockChecksum;
+    g_blockChecksum = (xxhash != 0);
+    return g_blockChecksum;
 }
 
 /* Default setting : checksum enabled */
 int LZ4IO_setStreamChecksumMode(int xxhash)
 {
-    streamChecksum = (xxhash != 0);
-    return streamChecksum;
+    g_streamChecksum = (xxhash != 0);
+    return g_streamChecksum;
 }
 
 /* Default setting : 0 (no notification) */
 int LZ4IO_setNotificationLevel(int level)
 {
-    displayLevel = level;
-    return displayLevel;
+    g_displayLevel = level;
+    return g_displayLevel;
+}
+
+/* Default setting : 0 (disabled) */
+int LZ4IO_setSparseFile(int enable)
+{
+    g_sparseFileSupport = (enable!=0);
+    return g_sparseFileSupport;
+}
+
+/* Default setting : 0 (disabled) */
+int LZ4IO_setContentSize(int enable)
+{
+    g_contentSizeFlag = (enable!=0);
+    return g_contentSizeFlag;
 }
 
 static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious)
@@ -214,16 +226,30 @@ static unsigned LZ4IO_GetMilliSpan(clock_t nPrevious)
     return nSpan;
 }
 
-
-/* ************************************************************************ */
-/* ********************** LZ4 File / Pipe compression ********************* */
-/* ************************************************************************ */
-
-static int          LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
-static int          LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }
+static unsigned long long LZ4IO_GetFileSize(const char* infilename)
+{
+    int r;
+#if defined(_MSC_VER)
+    struct _stat64 statbuf;
+    r = _stat64(infilename, &statbuf);
+#else
+    struct stat statbuf;
+    r = stat(infilename, &statbuf);
+#endif
+    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+    return (unsigned long long)statbuf.st_size;
+}
 
 
-static int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)
+/* ************************************************************************ **
+** ********************** LZ4 File / Pipe compression ********************* **
+** ************************************************************************ */
+
+static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
+static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; }
+
+
+static int get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput)
 {
 
     if (!strcmp (input_filename, stdinmark))
@@ -251,12 +277,12 @@ static int get_fileHandle(char* input_filename, char* output_filename, FILE** pf
         if (*pfoutput!=0)
         {
             fclose(*pfoutput);
-            if (!overwrite)
+            if (!g_overwrite)
             {
                 char ch;
                 DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
                 DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
-                if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);   /* No interaction possible */
+                if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);   /* No interaction possible */
                 ch = (char)getchar();
                 if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
             }
@@ -272,15 +298,14 @@ static int get_fileHandle(char* input_filename, char* output_filename, FILE** pf
 
 
 
-
 /***************************************
- *   Legacy Compression
- * *************************************/
+*   Legacy Compression
+***************************************/
 
 /* unoptimized version; solves endianess & alignment issues */
 static void LZ4IO_writeLE32 (void* p, unsigned value32)
 {
-    unsigned char* dstPtr = p;
+    unsigned char* dstPtr = (unsigned char*)p;
     dstPtr[0] = (unsigned char)value32;
     dstPtr[1] = (unsigned char)(value32 >> 8);
     dstPtr[2] = (unsigned char)(value32 >> 16);
@@ -290,7 +315,7 @@ static void LZ4IO_writeLE32 (void* p, unsigned value32)
 /* LZ4IO_compressFilename_Legacy :
  * This function is intentionally "hidden" (not published in .h)
  * It generates compressed streams using the old 'legacy' format */
-int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, int compressionlevel)
+int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel)
 {
     int (*compressionFunction)(const char*, char*, int);
     unsigned long long filesize = 0;
@@ -308,7 +333,7 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i
     if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
 
     get_fileHandle(input_filename, output_filename, &finput, &foutput);
-    if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3;
+    if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3;
 
     /* Allocate Memory */
     in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
@@ -360,11 +385,11 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i
 }
 
 
-/***********************************************
- *   Compression using Frame format
- * ********************************************/
+/*********************************************
+*  Compression using Frame format
+*********************************************/
 
-int LZ4IO_compressFilename(char* input_filename, char* output_filename, int compressionLevel)
+int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel)
 {
     unsigned long long filesize = 0;
     unsigned long long compressedfilesize = 0;
@@ -377,23 +402,29 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
     size_t sizeCheck, headerSize, readSize, outBuffSize;
     LZ4F_compressionContext_t ctx;
     LZ4F_errorCode_t errorCode;
-    LZ4F_preferences_t prefs = {0};
+    LZ4F_preferences_t prefs;
 
 
     /* Init */
     start = clock();
-    if ((displayLevel==2) && (compressionLevel>=3)) displayLevel=3;
+    memset(&prefs, 0, sizeof(prefs));
+    if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3;
     errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
     if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
     get_fileHandle(input_filename, output_filename, &finput, &foutput);
-    blockSize = LZ4S_GetBlockSize_FromBlockId (globalBlockSizeId);
+    blockSize = LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId);
 
     /* Set compression parameters */
     prefs.autoFlush = 1;
     prefs.compressionLevel = compressionLevel;
-    prefs.frameInfo.blockMode = blockIndependence;
-    prefs.frameInfo.blockSizeID = globalBlockSizeId;
-    prefs.frameInfo.contentChecksumFlag = streamChecksum;
+    prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence;
+    prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId;
+    prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum;
+    if (g_contentSizeFlag)
+    {
+      unsigned long long fileSize = LZ4IO_GetFileSize(input_filename);
+      prefs.frameInfo.frameOSize = fileSize;   /* == 0 if input == stdin */
+    }
 
     /* Allocate Memory */
     in_buff  = (char*)malloc(blockSize);
@@ -462,13 +493,34 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp
 }
 
 
+#define FNSPACE 30
+int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel)
+{
+    int i;
+    char* outFileName = (char*)malloc(FNSPACE);
+    size_t ofnSize = FNSPACE;
+    const size_t suffixSize = strlen(suffix);
+
+    for (i=0; i<ifntSize; i++)
+    {
+        size_t ifnSize = strlen(inFileNamesTable[i]);
+        if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); }
+        strcpy(outFileName, inFileNamesTable[i]);
+        strcat(outFileName, suffix);
+        LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel);
+    }
+    free(outFileName);
+    return 0;
+}
+
+
 /* ********************************************************************* */
-/* ********************** LZ4 File / Stream decoding ******************* */
+/* ********************** LZ4 file-stream Decompression **************** */
 /* ********************************************************************* */
 
 static unsigned LZ4IO_readLE32 (const void* s)
 {
-    const unsigned char* srcPtr = s;
+    const unsigned char* srcPtr = (const unsigned char*)s;
     unsigned value32 = srcPtr[0];
     value32 += (srcPtr[1]<<8);
     value32 += (srcPtr[2]<<16);
@@ -506,15 +558,16 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
 
         /* Read Block */
         sizeCheck = fread(in_buff, 1, blockSize, finput);
+        if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !");
 
         /* Decode Block */
         decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
-        if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !");
+        if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
         filesize += decodeSize;
 
         /* Write Block */
         sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);
-        if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n");
+        if (sizeCheck != (size_t)decodeSize) EXM_THROW(54, "Write error : cannot write decoded block into output\n");
     }
 
     /* Free */
@@ -528,57 +581,131 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
 static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
 {
     unsigned long long filesize = 0;
-    char* inBuff;
-    char* outBuff;
+    void* inBuff;
+    void* outBuff;
 #   define HEADERMAX 20
     char  headerBuff[HEADERMAX];
-    size_t sizeCheck, nextToRead, outBuffSize, inBuffSize;
+    size_t sizeCheck;
+    const size_t inBuffSize = 256 KB;
+    const size_t outBuffSize = 256 KB;
     LZ4F_decompressionContext_t ctx;
     LZ4F_errorCode_t errorCode;
-    LZ4F_frameInfo_t frameInfo;
+    unsigned storedSkips = 0;
 
     /* init */
     errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
-    if (LZ4F_isError(errorCode)) EXM_THROW(60, "Allocation error : can't create context : %s", LZ4F_getErrorName(errorCode));
-    LZ4IO_writeLE32(headerBuff, LZ4S_MAGICNUMBER);   /* regenerated here, as it was already read from finput */
-
-    /* Decode stream descriptor */
-    outBuffSize = 0; inBuffSize = 0; sizeCheck = MAGICNUMBER_SIZE;
-    nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL);
-    if (LZ4F_isError(nextToRead)) EXM_THROW(61, "Decompression error : %s", LZ4F_getErrorName(nextToRead));
-    if (nextToRead > HEADERMAX) EXM_THROW(62, "Header too large (%i>%i)", (int)nextToRead, HEADERMAX);
-    sizeCheck = fread(headerBuff, 1, nextToRead, finput);
-    if (sizeCheck!=nextToRead) EXM_THROW(63, "Read error ");
-    nextToRead = LZ4F_decompress(ctx, NULL, &outBuffSize, headerBuff, &sizeCheck, NULL);
-    errorCode = LZ4F_getFrameInfo(ctx, &frameInfo, NULL, &inBuffSize);
-    if (LZ4F_isError(errorCode)) EXM_THROW(64, "can't decode frame header : %s", LZ4F_getErrorName(errorCode));
+    if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode));
+    LZ4IO_writeLE32(headerBuff, LZ4IO_MAGICNUMBER);   /* regenerated here, as it was already read from finput */
 
     /* Allocate Memory */
-    outBuffSize = LZ4IO_setBlockSizeID(frameInfo.blockSizeID);
-    inBuffSize = outBuffSize + 4;
-    inBuff = (char*)malloc(inBuffSize);
-    outBuff = (char*)malloc(outBuffSize);
-    if (!inBuff || !outBuff) EXM_THROW(65, "Allocation error : not enough memory");
+    inBuff = malloc(256 KB);
+    outBuff = malloc(256 KB);
+    if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory");
+
+    /* Init feed with magic number (already consumed from FILE) */
+    {
+        size_t inSize = 4;
+        size_t outSize=0;
+        LZ4IO_writeLE32(inBuff, LZ4IO_MAGICNUMBER);
+        errorCode = LZ4F_decompress(ctx, outBuff, &outSize, inBuff, &inSize, NULL);
+        if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode));
+    }
+
 
     /* Main Loop */
-    while (nextToRead != 0)
+    for (;;)
     {
-        size_t decodedBytes = outBuffSize;
+        size_t readSize;
+        size_t pos = 0;
 
-        /* Read Block */
-        sizeCheck = fread(inBuff, 1, nextToRead, finput);
-        if (sizeCheck!=nextToRead) EXM_THROW(66, "Read error ");
+        /* Read input */
+        readSize = fread(inBuff, 1, inBuffSize, finput);
+        if (!readSize) break;   /* empty file or stream */
 
-        /* Decode Block */
-        errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, inBuff, &sizeCheck, NULL);
-        if (LZ4F_isError(errorCode)) EXM_THROW(67, "Decompression error : %s", LZ4F_getErrorName(errorCode));
-        if (sizeCheck!=nextToRead) EXM_THROW(67, "Synchronization error");
-        nextToRead = errorCode;
-        filesize += decodedBytes;
+        while (pos < readSize)
+        {
+            /* Decode Input (at least partially) */
+            size_t remaining = readSize - pos;
+            size_t decodedBytes = outBuffSize;
+            errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, (char*)inBuff+pos, &remaining, NULL);
+            if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode));
+            pos += remaining;
 
-        /* Write Block */
-        sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
-        if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n");
+            if (decodedBytes)
+            {
+                /* Write Block */
+                filesize += decodedBytes;
+                if (g_sparseFileSupport)
+                {
+                    size_t* const oBuffStartT = (size_t*)outBuff;   /* since outBuff is malloc'ed, it's aligned on size_t */
+                    size_t* oBuffPosT = oBuffStartT;
+                    size_t  oBuffSizeT = decodedBytes / sizeT;
+                    size_t* const oBuffEndT = oBuffStartT + oBuffSizeT;
+                    static const size_t bs0T = (32 KB) / sizeT;
+                    while (oBuffPosT < oBuffEndT)
+                    {
+                        size_t seg0SizeT = bs0T;
+                        size_t nb0T;
+                        int seekResult;
+                        if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT;
+                        oBuffSizeT -= seg0SizeT;
+                        for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ;
+                        storedSkips += (unsigned)(nb0T * sizeT);
+                        if (storedSkips > 1 GB)   /* avoid int overflow */
+                        {
+                            seekResult = fseek(foutput, 1 GB, SEEK_CUR);
+                            if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)");
+                            storedSkips -= 1 GB;
+                        }
+                        if (nb0T != seg0SizeT)   /* not all 0s */
+                        {
+                            seekResult = fseek(foutput, storedSkips, SEEK_CUR);
+                            if (seekResult) EXM_THROW(68, "Skip error (sparse file)");
+                            storedSkips = 0;
+                            seg0SizeT -= nb0T;
+                            oBuffPosT += nb0T;
+                            sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput);
+                            if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block");
+                        }
+                        oBuffPosT += seg0SizeT;
+                    }
+                    if (decodedBytes & maskT)   /* size not multiple of sizeT (necessarily end of block) */
+                    {
+                        const char* const restStart = (char*)oBuffEndT;
+                        const char* restPtr = restStart;
+                        size_t  restSize =  decodedBytes & maskT;
+                        const char* const restEnd = restStart + restSize;
+                        for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
+                        storedSkips += (unsigned) (restPtr - restStart);
+                        if (restPtr != restEnd)
+                        {
+                            int seekResult = fseek(foutput, storedSkips, SEEK_CUR);
+                            if (seekResult) EXM_THROW(68, "Skip error (end of block)");
+                            storedSkips = 0;
+                            sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput);
+                            if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block");
+                        }
+                    }
+                }
+                else
+                {
+                    sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
+                    if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block");
+                }
+            }
+        }
+
+    }
+
+    if ((g_sparseFileSupport) && (storedSkips>0))
+    {
+        int seekResult;
+        storedSkips --;
+        seekResult = fseek(foutput, storedSkips, SEEK_CUR);
+        if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
+        memset(outBuff, 0, 1);
+        sizeCheck = fwrite(outBuff, 1, 1, foutput);
+        if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
     }
 
     /* Free */
@@ -591,6 +718,28 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
 }
 
 
+static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char U32store[MAGICNUMBER_SIZE])
+{
+    void* buffer = malloc(64 KB);
+    size_t read = 1, sizeCheck;
+    unsigned long long total = MAGICNUMBER_SIZE;
+
+    sizeCheck = fwrite(U32store, 1, MAGICNUMBER_SIZE, foutput);
+    if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through error at start");
+
+    while (read)
+    {
+        read = fread(buffer, 1, 64 KB, finput);
+        total += read;
+        sizeCheck = fwrite(buffer, 1, read, foutput);
+        if (sizeCheck != read) EXM_THROW(50, "Pass-through error");
+    }
+
+    free(buffer);
+    return total;
+}
+
+
 #define ENDOFSTREAM ((unsigned long long)-1)
 static unsigned long long selectDecoder( FILE* finput,  FILE* foutput)
 {
@@ -598,22 +747,26 @@ static unsigned long long selectDecoder( FILE* finput,  FILE* foutput)
     unsigned magicNumber, size;
     int errorNb;
     size_t nbReadBytes;
+    static unsigned nbCalls = 0;
+
+    /* init */
+    nbCalls++;
 
     /* Check Archive Header */
     nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput);
     if (nbReadBytes==0) return ENDOFSTREAM;                  /* EOF */
     if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
     magicNumber = LZ4IO_readLE32(U32store);   /* Little Endian format */
-    if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0;  /* fold skippable magic numbers */
+    if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0;  /* fold skippable magic numbers */
 
     switch(magicNumber)
     {
-    case LZ4S_MAGICNUMBER:
+    case LZ4IO_MAGICNUMBER:
         return DEFAULT_DECOMPRESSOR(finput, foutput);
     case LEGACY_MAGICNUMBER:
         DISPLAYLEVEL(4, "Detected : Legacy format \n");
         return decodeLegacyStream(finput, foutput);
-    case LZ4S_SKIPPABLE0:
+    case LZ4IO_SKIPPABLE0:
         DISPLAYLEVEL(4, "Skipping detected skippable area \n");
         nbReadBytes = fread(U32store, 1, 4, finput);
         if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
@@ -623,14 +776,19 @@ static unsigned long long selectDecoder( FILE* finput,  FILE* foutput)
         return selectDecoder(finput, foutput);
     EXTENDED_FORMAT;
     default:
-        if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded");   /* Wrong magic number at the beginning of 1st stream */
+        if (nbCalls == 1)   /* just started */
+        {
+            if (g_overwrite)
+                return LZ4IO_passThrough(finput, foutput, U32store);
+            EXM_THROW(44,"Unrecognized header : file cannot be decoded");   /* Wrong magic number at the beginning of 1st stream */
+        }
         DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
         return ENDOFSTREAM;
     }
 }
 
 
-int LZ4IO_decompressFilename(char* input_filename, char* output_filename)
+int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename)
 {
     unsigned long long filesize = 0, decodedSize=0;
     FILE* finput;
@@ -642,6 +800,9 @@ int LZ4IO_decompressFilename(char* input_filename, char* output_filename)
     start = clock();
     get_fileHandle(input_filename, output_filename, &finput, &foutput);
 
+    /* sparse file */
+    if (g_sparseFileSupport && foutput) { SET_SPARSE_FILE_MODE(foutput); }
+
     /* Loop over multiple streams */
     do
     {

programs/lz4io.h

@@ -1,6 +1,6 @@
 /*
   LZ4io.h - LZ4 File/Stream Interface
-  Copyright (C) Yann Collet 2011-2014
+  Copyright (C) Yann Collet 2011-2015
   GPL v2 License
 
   This program is free software; you can redistribute it and/or modify
@@ -18,7 +18,7 @@
   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
   You can contact the author at :
-  - LZ4 source repository : http://code.google.com/p/lz4/
+  - LZ4 source repository : https://github.com/Cyan4973/lz4
   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 /*
@@ -29,17 +29,18 @@
   - The license of this source file is GPLv2.
 */
 
+#pragma once
 
 /* ************************************************** */
 /* Special input/output values                        */
 /* ************************************************** */
 #define NULL_OUTPUT "null"
-static char stdinmark[] = "stdin";
-static char stdoutmark[] = "stdout";
+static char const stdinmark[] = "stdin";
+static char const stdoutmark[] = "stdout";
 #ifdef _WIN32
-static char nulmark[] = "nul";
+static char const nulmark[] = "nul";
 #else
-static char nulmark[] = "/dev/null";
+static char const nulmark[] = "/dev/null";
 #endif
 
 
@@ -47,8 +48,10 @@ static char nulmark[] = "/dev/null";
 /* ****************** Functions ********************* */
 /* ************************************************** */
 
-int LZ4IO_compressFilename  (char* input_filename, char* output_filename, int compressionlevel);
-int LZ4IO_decompressFilename(char* input_filename, char* output_filename);
+int LZ4IO_compressFilename  (const char* input_filename, const char* output_filename, int compressionlevel);
+int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename);
+
+int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel);
 
 
 /* ************************************************** */
@@ -67,11 +70,17 @@ int LZ4IO_setBlockSizeID(int blockSizeID);
 typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t;
 int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode);
 
-/* Default setting : no checksum */
+/* Default setting : no block checksum */
 int LZ4IO_setBlockChecksumMode(int xxhash);
 
-/* Default setting : checksum enabled */
+/* Default setting : stream checksum enabled */
 int LZ4IO_setStreamChecksumMode(int xxhash);
 
 /* Default setting : 0 (no notification) */
 int LZ4IO_setNotificationLevel(int level);
+
+/* Default setting : 0 (disabled) */
+int LZ4IO_setSparseFile(int enable);
+
+/* Default setting : 0 (disabled) */
+int LZ4IO_setContentSize(int enable);

visual/2012/fuzzer/fuzzer.vcxproj

@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{18B9F1A7-9C66-4352-898B-30804DADE0FD}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>fuzzer</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v110</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v110</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v110</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v110</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\programs\fuzzer.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

visual/2012/fuzzer/fuzzer.vcxproj.filters

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Fichiers sources">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Fichiers d%27en-tête">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+    </Filter>
+    <Filter Include="Fichiers de ressources">
+      <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\programs\fuzzer.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\lz4.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\lz4hc.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\xxhash.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\lib\lz4hc.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\lib\xxhash.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+  </ItemGroup>
+</Project>

visual/2012/fuzzer/fuzzer.vcxproj.user

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup />
+</Project>

visual/2012/lz4.sln

@@ -0,0 +1,46 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Express 2012 for Windows Desktop
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{E30329AC-0057-4FE0-8FDA-7F650D398C4C}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Mixed Platforms = Debug|Mixed Platforms
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Mixed Platforms = Release|Mixed Platforms
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.Build.0 = Debug|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64
+		{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.Build.0 = Debug|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64
+		{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

visual/2012/lz4/lz4.vcxproj

@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{E30329AC-0057-4FE0-8FDA-7F650D398C4C}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>lz4</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v110</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v110</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v110</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v110</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\lz4.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame.h" />
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+    <ClInclude Include="..\..\..\lib\lz4hc.h" />
+    <ClInclude Include="..\..\..\lib\xxhash.h" />
+    <ClInclude Include="..\..\..\programs\bench.h" />
+    <ClInclude Include="..\..\..\programs\lz4io.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\lz4.c" />
+    <ClCompile Include="..\..\..\lib\lz4frame.c" />
+    <ClCompile Include="..\..\..\lib\lz4hc.c" />
+    <ClCompile Include="..\..\..\lib\xxhash.c" />
+    <ClCompile Include="..\..\..\programs\bench.c" />
+    <ClCompile Include="..\..\..\programs\lz4cli.c" />
+    <ClCompile Include="..\..\..\programs\lz4io.c" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

visual/2012/lz4/lz4.vcxproj.filters

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Fichiers sources">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+    <Filter Include="Fichiers d%27en-tête">
+      <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+      <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+    </Filter>
+    <Filter Include="Fichiers de ressources">
+      <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+      <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\programs\lz4io.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\programs\bench.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\lib\xxhash.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\lib\lz4hc.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\lib\lz4frame_static.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\lib\lz4frame.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\..\lib\lz4.h">
+      <Filter>Fichiers d%27en-tête</Filter>
+    </ClInclude>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\programs\lz4io.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\programs\lz4cli.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\programs\bench.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\xxhash.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\lz4hc.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\lz4frame.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\lib\lz4.c">
+      <Filter>Fichiers sources</Filter>
+    </ClCompile>
+  </ItemGroup>
+</Project>

visual/2012/lz4/lz4.vcxproj.user

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup />
+</Project>