From 612b346ed51c8497e40c521c77808833361945ab Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 11 Jul 2018 15:50:28 -0700 Subject: [PATCH] Add explanation for split=100 --- lib/dictBuilder/zdict.h | 2 +- programs/zstd.1 | 3 ++- programs/zstd.1.md | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 9357e40a..4094669d 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -86,7 +86,7 @@ typedef struct { unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ - double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, 0 means default (1.0) */ + double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ ZDICT_params_t zParams; } ZDICT_cover_params_t; diff --git a/programs/zstd.1 b/programs/zstd.1 index 3e9e2942..9d1e45cf 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -218,7 +218,8 @@ A dictionary ID is a locally unique ID that a decoder can use to verify it is us . .TP \fB\-\-train\-cover[=k#,d=#,steps=#,split=#]\fR -Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0 or \fIsplit\fR > 100, then the default value of 100 is used\. Requires that \fId\fR <= \fIk\fR\. +Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or \fIsplit\fR <= 0, then the default value of 100 is used\. If \fIsplit\fR is 100, all input samples are used for both training and testing +to find optimal _d_ and _k_ to build dictionary.Requires that \fId\fR <= \fIk\fR\. . .IP Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. Supports multithreading if \fBzstd\fR is compiled with threading support\. diff --git a/programs/zstd.1.md b/programs/zstd.1.md index df6f777d..7e21073d 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -228,7 +228,7 @@ Compression of small files similar to the sample set will be greatly improved. If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8. If _k_ is not specified, then it tries _steps_ values in the range [50, 2000]. If _steps_ is not specified, then the default value of 40 is used. - If _split_ is not specified or split <= 0 or split > 100, then the default value of 100 is used. + If _split_ is not specified or split <= 0, then the default value of 100 is used. Requires that _d_ <= _k_. Selects segments of size _k_ with highest score to put in the dictionary. @@ -238,6 +238,8 @@ Compression of small files similar to the sample set will be greatly improved. algorithm will run faster with d <= _8_. Good values for _k_ vary widely based on the input data, but a safe range is [2 * _d_, 2000]. + If _split_ is 100, all input samples are used for both training and testing + to find optimal _d_ and _k_ to build dictionary. Supports multithreading if `zstd` is compiled with threading support. Examples: