cli : add support for --threads=# command
updated documentation add relevant test case
This commit is contained in:
Yann Collet
parent
d0b1846cf4
commit
230d7acc7d
@ -5,7 +5,7 @@
|
||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||
.
|
||||
.SH "SYNOPSIS"
|
||||
\fBzstd\fR [\fIOPTIONS\fR] [\-|<INPUT\-FILE>] [\-o <OUTPUT\-FILE>]
|
||||
\fBzstd\fR [\fIOPTIONS\fR] [\-|\fIINPUT\-FILE\fR] [\-o \fIOUTPUT\-FILE\fR]
|
||||
.
|
||||
.P
|
||||
\fBzstdmt\fR is equivalent to \fBzstd \-T0\fR
|
||||
@ -100,8 +100,8 @@ Use FILEs as a training set to create a dictionary\. The training set should con
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
|
||||
.
|
||||
.TP
|
||||
\fB\-T#\fR
|
||||
Compress using # threads (default: 1)\. If \fB#\fR is 0, attempt to detect the number of physical CPU cores and compress with that many threads\. This modified does nothing if \fBzstd\fR was compiled without multithread support\.
|
||||
\fB\-T#\fR, \fB\-\-threads=#\fR
|
||||
Compress using \fB#\fR threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
|
||||
.
|
||||
.TP
|
||||
\fB\-D file\fR
|
||||
@ -113,7 +113,7 @@ do not store dictionary ID within frame header (dictionary compression)\. The de
|
||||
.
|
||||
.TP
|
||||
\fB\-o file\fR
|
||||
save result into \fBfile\fR (only possible with a single INPUT\-FILE)
|
||||
save result into \fBfile\fR (only possible with a single \fIINPUT\-FILE\fR)
|
||||
.
|
||||
.TP
|
||||
\fB\-f\fR, \fB\-\-force\fR
|
||||
@ -172,7 +172,7 @@ use FILEs as training set to create a dictionary\. The training set should conta
|
||||
.
|
||||
.TP
|
||||
\fB\-o file\fR
|
||||
dictionary saved into \fBfile\fR (default: dictionary)
|
||||
dictionary saved into \fBfile\fR (default name: dictionary)
|
||||
.
|
||||
.TP
|
||||
\fB\-\-maxdict=#\fR
|
||||
@ -198,15 +198,18 @@ Example: \fB\-\-train \-\-cover=k=64,d=8 FILEs\fR\.
|
||||
If \fIsteps\fR is not specified, the default value of 32 is used\. If \fIk\fR is not specified, the \fIk\fR values in [16, 2048] are checked for each value of \fId\fR\. If \fId\fR is not specified, the values checked are [6, 8, \.\.\., 16]\.
|
||||
.
|
||||
.IP
|
||||
Runs the cover dictionary builder for each parameter set and saves the optimal parameters and dictionary\. Prints the optimal parameters and writes the optimal dictionary to the output file\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
|
||||
Runs the cover dictionary builder for each parameter set and saves the optimal parameters and dictionary\. Prints optimal parameters and writes optimal dictionary into output file\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
|
||||
.
|
||||
.IP
|
||||
The parameter \fIk\fR is more sensitive than \fId\fR, and is faster to optimize over\. Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set\. Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024])\.
|
||||
.
|
||||
.IP
|
||||
Examples :
|
||||
.
|
||||
.IP
|
||||
\fBzstd \-\-train \-\-optimize\-cover FILEs\fR
|
||||
.
|
||||
.br
|
||||
.IP
|
||||
\fBzstd \-\-train \-\-optimize\-cover=d=d,steps=512 FILEs\fR
|
||||
.
|
||||
.SH "BENCHMARK"
|
||||
@ -233,9 +236,6 @@ set process priority to real\-time
|
||||
.
|
||||
.SH "ADVANCED COMPRESSION OPTIONS"
|
||||
.
|
||||
.SS "\-B#:"
|
||||
Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
|
||||
.
|
||||
.SS "\-\-zstd[=options]:"
|
||||
\fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
|
||||
.
|
||||
@ -310,6 +310,9 @@ Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This pa
|
||||
.IP
|
||||
The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\.
|
||||
.
|
||||
.SS "\-B#:"
|
||||
Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
|
||||
.
|
||||
.SS "Example"
|
||||
The following parameters sets advanced compression options to those of predefined level 19 for files bigger than 256 KB:
|
||||
.
|
||||
|
||||
@ -4,7 +4,7 @@ zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files
|
||||
SYNOPSIS
|
||||
--------
|
||||
|
||||
`zstd` [*OPTIONS*] [-|<INPUT-FILE>] [-o <OUTPUT-FILE>]
|
||||
`zstd` [*OPTIONS*] [-|_INPUT-FILE_] [-o _OUTPUT-FILE_]
|
||||
|
||||
`zstdmt` is equivalent to `zstd -T0`
|
||||
|
||||
@ -101,11 +101,10 @@ the last one takes effect.
|
||||
* `--ultra`:
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
|
||||
Note that decompression will also require more memory when using these levels.
|
||||
* `-T#`:
|
||||
Compress using # threads (default: 1).
|
||||
If `#` is 0, attempt to detect the number of physical CPU cores and compress with
|
||||
that many threads.
|
||||
This modified does nothing if `zstd` was compiled without multithread support.
|
||||
* `-T#`, `--threads=#`:
|
||||
Compress using `#` threads (default: 1).
|
||||
If `#` is 0, attempt to detect and use the number of physical CPU cores.
|
||||
This modifier does nothing if `zstd` is compiled without multithread support.
|
||||
* `-D file`:
|
||||
use `file` as Dictionary to compress or decompress FILE(s)
|
||||
* `--nodictID`:
|
||||
@ -113,7 +112,7 @@ the last one takes effect.
|
||||
The decoder will have to rely on implicit knowledge about which dictionary to use,
|
||||
it won't be able to check if it's correct.
|
||||
* `-o file`:
|
||||
save result into `file` (only possible with a single INPUT-FILE)
|
||||
save result into `file` (only possible with a single _INPUT-FILE_)
|
||||
* `-f`, `--force`:
|
||||
overwrite output without prompting, and (de)compress symbolic links
|
||||
* `-c`, `--stdout`:
|
||||
@ -164,7 +163,7 @@ Typical gains range from 10% (at 64KB) to x5 better (at <1KB).
|
||||
and weight typically 100x the target dictionary size
|
||||
(for example, 10 MB for a 100 KB dictionary).
|
||||
* `-o file`:
|
||||
dictionary saved into `file` (default: dictionary)
|
||||
dictionary saved into `file` (default name: dictionary)
|
||||
* `--maxdict=#`:
|
||||
limit dictionary to specified size (default : (112640)
|
||||
* `--dictID=#`:
|
||||
@ -198,9 +197,9 @@ Typical gains range from 10% (at 64KB) to x5 better (at <1KB).
|
||||
value of _d_.
|
||||
If _d_ is not specified, the values checked are [6, 8, ..., 16].
|
||||
|
||||
Runs the cover dictionary builder for each parameter set and saves the
|
||||
optimal parameters and dictionary.
|
||||
Prints the optimal parameters and writes the optimal dictionary to the output file.
|
||||
Runs the cover dictionary builder for each parameter set
|
||||
and saves the optimal parameters and dictionary.
|
||||
Prints optimal parameters and writes optimal dictionary into output file.
|
||||
Supports multithreading if `zstd` is compiled with threading support.
|
||||
|
||||
The parameter _k_ is more sensitive than _d_, and is faster to optimize over.
|
||||
@ -208,7 +207,10 @@ Typical gains range from 10% (at 64KB) to x5 better (at <1KB).
|
||||
Once it completes, use the value of _d_ it selects with a higher _steps_
|
||||
(in the range [256, 1024]).
|
||||
|
||||
`zstd --train --optimize-cover FILEs` <br />
|
||||
Examples :
|
||||
|
||||
`zstd --train --optimize-cover FILEs`
|
||||
|
||||
`zstd --train --optimize-cover=d=d,steps=512 FILEs`
|
||||
|
||||
|
||||
@ -229,14 +231,6 @@ BENCHMARK
|
||||
|
||||
ADVANCED COMPRESSION OPTIONS
|
||||
----------------------------
|
||||
### -B#:
|
||||
Select the size of each compression job.
|
||||
This parameter is available only when multi-threading is enabled.
|
||||
Default value is `4 * windowSize`, which means it varies depending on compression level.
|
||||
`-B#` makes it possible to select a custom value.
|
||||
Note that job size must respect a minimum value which is enforced transparently.
|
||||
This minimum is either 1 MB, or `overlapSize`, whichever is largest.
|
||||
|
||||
### --zstd[=options]:
|
||||
`zstd` provides 22 predefined compression levels.
|
||||
The selected or default predefined compression level can be changed with
|
||||
@ -319,6 +313,13 @@ The list of available _options_:
|
||||
Default _ovlog_ is 6, which means "reload `windowSize / 8`".
|
||||
Exception : the maximum compression level (22) has a default _ovlog_ of 9.
|
||||
|
||||
### -B#:
|
||||
Select the size of each compression job.
|
||||
This parameter is available only when multi-threading is enabled.
|
||||
Default value is `4 * windowSize`, which means it varies depending on compression level.
|
||||
`-B#` makes it possible to select a custom value.
|
||||
Note that job size must respect a minimum value which is enforced transparently.
|
||||
This minimum is either 1 MB, or `overlapSize`, whichever is largest.
|
||||
|
||||
### Example
|
||||
The following parameters sets advanced compression options to those of
|
||||
|
||||
@ -420,6 +420,7 @@ int main(int argCount, const char* argv[])
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
if (longCommandWArg(&argument, "--threads=")) { nbThreads = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
|
||||
@ -471,6 +471,7 @@ then
|
||||
$ECHO "\n**** zstdmt round-trip tests **** "
|
||||
roundTripTest -g4M "1 -T0"
|
||||
roundTripTest -g8M "3 -T2"
|
||||
roundTripTest -g8000K "2 --threads=2"
|
||||
fileRoundTripTest -g4M "19 -T2 -B1M"
|
||||
else
|
||||
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
|
||||
|
||||
Loading…
Reference in New Issue
Block a user