Merge pull request #1463 from yijinfb/getenv

Add support for environment variable ZSTD_CLEVEL in CLI

programs/README.md

@@ -172,6 +172,11 @@ Benchmark arguments :
 --priority=rt : set process priority to real-time
 ```
 
+#### Restricted usage of Environment Variables
+Using environment variables to set compression/decompression parameters has security implications. Therefore,
+we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level.
+If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
+Note that command line options will override corresponding environment variable settings.
 
 #### Long distance matching mode
 The long distance matching mode, enabled with `--long`, is designed to improve

programs/zstdcli.c

@@ -28,6 +28,7 @@
 #include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
 #include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
 #include <stdio.h>    /* fprintf(), stdin, stdout, stderr */
+#include <stdlib.h>   /* getenv */
 #include <string.h>   /* strcmp, strlen */
 #include <errno.h>    /* errno */
 #include "fileio.h"   /* stdinmark, stdoutmark, ZSTD_EXTENSION */
@@ -233,32 +234,44 @@ static void errorOut(const char* msg)
     DISPLAY("%s \n", msg); exit(1);
 }
 
-/*! readU32FromChar() :
- * @return : unsigned integer value read from input in `char` format.
+/*! readU32FromCharChecked() :
+ * @return 0 if success, and store the result in *value.
  *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
  *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
- *  Note : function will exit() program if digit sequence overflows */
-static unsigned readU32FromChar(const char** stringPtr)
+ * @return 1 if an overflow error occurs */
+static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
 {
-    const char errorMsg[] = "error: numeric value too large";
+    static unsigned const max = (((unsigned)(-1)) / 10) - 1;
     unsigned result = 0;
     while ((**stringPtr >='0') && (**stringPtr <='9')) {
-        unsigned const max = (((unsigned)(-1)) / 10) - 1;
-        if (result > max) errorOut(errorMsg);
+        if (result > max) return 1; // overflow error
         result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
     }
     if ((**stringPtr=='K') || (**stringPtr=='M')) {
         unsigned const maxK = ((unsigned)(-1)) >> 10;
-        if (result > maxK) errorOut(errorMsg);
+        if (result > maxK) return 1; // overflow error
         result <<= 10;
         if (**stringPtr=='M') {
-            if (result > maxK) errorOut(errorMsg);
+            if (result > maxK) return 1; // overflow error
             result <<= 10;
         }
         (*stringPtr)++;  /* skip `K` or `M` */
         if (**stringPtr=='i') (*stringPtr)++;
         if (**stringPtr=='B') (*stringPtr)++;
     }
+    *value = result;
+    return 0;
+}
+
+/*! readU32FromChar() :
+ * @return : unsigned integer value read from input in `char` format.
+ *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+ *  Note : function will exit() program if digit sequence overflows */
+static unsigned readU32FromChar(const char** stringPtr) {
+    static const char errorMsg[] = "error: numeric value too large";
+    unsigned result;
+    if (readU32FromCharChecked(stringPtr, &result)) { errorOut(errorMsg); }
     return result;
 }
 
@@ -452,6 +465,38 @@ static void printVersion(void)
 #endif
 }
 
+/* Environment variables for parameter setting */
+#define ENV_CLEVEL "ZSTD_CLEVEL"
+
+/* functions that pick up environment variables */
+static int init_cLevel(void) {
+    const char* const env = getenv(ENV_CLEVEL);
+    if (env) {
+        const char *ptr = env;
+        int sign = 1;
+        if (*ptr == '-') {
+            sign = -1;
+            ptr++;
+        } else if (*ptr == '+') {
+            ptr++;
+        }
+
+        if ((*ptr>='0') && (*ptr<='9')) {
+            unsigned absLevel;
+            if (readU32FromCharChecked(&ptr, &absLevel)) { 
+                DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
+                return ZSTDCLI_CLEVEL_DEFAULT;
+            } else if (*ptr == 0) {
+                return sign * absLevel;
+            }
+        }
+
+        DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid integer value\n", ENV_CLEVEL, env);
+    }
+
+    return ZSTDCLI_CLEVEL_DEFAULT;
+}
+
 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
 
 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
@@ -493,7 +538,7 @@ int main(int argCount, const char* argv[])
     size_t blockSize = 0;
     zstd_operation_mode operation = zom_compress;
     ZSTD_compressionParameters compressionParams;
-    int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
+    int cLevel;
     int cLevelLast = -1000000000;
     unsigned recursive = 0;
     unsigned memLimit = 0;
@@ -528,6 +573,7 @@ int main(int argCount, const char* argv[])
     if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
     filenameTable[0] = stdinmark;
     g_displayOut = stderr;
+    cLevel = init_cLevel();
     programName = lastNameFromPath(programName);
 #ifdef ZSTD_MULTITHREAD
     nbWorkers = 1;

tests/playTests.sh

@@ -122,6 +122,18 @@ $ZSTD --fast=5000000000 -f tmp && die "too large numeric value : must fail"
 $ZSTD -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
 $ECHO "test : too large numeric argument"
 $ZSTD --fast=9999999999 -f tmp  && die "should have refused numeric value"
+$ECHO "test : set compression level with environment variable ZSTD_CLEVEL"
+ZSTD_CLEVEL=12  $ZSTD -f tmp # positive compression level
+ZSTD_CLEVEL=-12 $ZSTD -f tmp # negative compression level
+ZSTD_CLEVEL=+12 $ZSTD -f tmp # valid: verbose '+' sign 
+ZSTD_CLEVEL=    $ZSTD -f tmp # empty env var, warn and revert to default setting
+ZSTD_CLEVEL=-   $ZSTD -f tmp # malformed env var, warn and revert to default setting
+ZSTD_CLEVEL=a   $ZSTD -f tmp # malformed env var, warn and revert to default setting
+ZSTD_CLEVEL=+a  $ZSTD -f tmp # malformed env var, warn and revert to default setting
+ZSTD_CLEVEL=3a7 $ZSTD -f tmp # malformed env var, warn and revert to default setting
+ZSTD_CLEVEL=50000000000  $ZSTD -f tmp # numeric value too large, warn and revert to default setting
+$ECHO "test : override ZSTD_CLEVEL with command line option"
+ZSTD_CLEVEL=12  $ZSTD --fast=3 -f tmp # overridden by command line option 
 $ECHO "test : compress to stdout"
 $ZSTD tmp -c > tmpCompressed
 $ZSTD tmp --stdout > tmpCompressed       # long command format