AuroraMiddleware/liblzma
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make the default memory usage limit 40 % of RAM for both

Browse Source 
compressing and decompressing. This should be OK now that
xz automatically scales down the compression settings if
they would exceed the memory usage limit (earlier, the limit
for compression was increased to 90 % because low limit broke
scripts that used "xz -9" on systems with low RAM).

Support spcifying the memory usage limit as a percentage
of RAM (e.g. --memory=50%).

Support --threads=0 to reset the thread limit to the default
value (number of available CPU cores). Use UINT32_MAX instead
of SIZE_MAX as the maximum in args.c. hardware.c was already
expecting uint32_t value.

Cleaned up the output of --help and --long-help.
This commit is contained in:
Lasse Collin 2009-05-22 11:29:50 +03:00
parent 071b825b23
commit b0063023f8
5 changed files with 84 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/xz/args.c
View File

@ -114,13 +114,27 @@ parse_real(args_info *args, int argc, char **argv)
break;
// --memory
case 'M':
// On 32-bit systems, SIZE_MAX would make more sense
// than UINT64_MAX. But use UINT64_MAX still so that
// scripts that assume > 4 GiB values don't break.
case 'M': {
// Support specifying the limit as a percentage of
// installed physical RAM.
size_t len = strlen(optarg);
if (len > 0 && optarg[len - 1] == '%') {
optarg[len - 1] = '\0';
hardware_memlimit_set_percentage(
str_to_uint64(
"memory%", optarg, 1, 100));
} else {
// On 32-bit systems, SIZE_MAX would make more
// sense than UINT64_MAX. But use UINT64_MAX
// still so that scripts that assume > 4 GiB
// values don't break.
hardware_memlimit_set(str_to_uint64(
"memory", optarg, 0, UINT64_MAX));
"memory", optarg,
0, UINT64_MAX));
}
break;
}
// --suffix
case 'S':
@ -129,7 +143,7 @@ parse_real(args_info *args, int argc, char **argv)
case 'T':
hardware_threadlimit_set(str_to_uint64(
"threads", optarg, 1, SIZE_MAX));
"threads", optarg, 0, UINT32_MAX));
break;
// --version

90
src/xz/hardware.c
View File

@ -17,37 +17,24 @@
/// Maximum number of free *coder* threads. This can be set with
/// the --threads=NUM command line option.
static uint32_t threads_max;
static uint32_t threadlimit;
/// Memory usage limit for encoding
static uint64_t memlimit_encoder;
/// Memory usage limit for decoding
static uint64_t memlimit_decoder;
/// Memory usage limit given on the command line or environment variable.
/// Zero indicates the default (memlimit_encoder or memlimit_decoder).
static uint64_t memlimit_custom = 0;
/// Get the number of CPU cores, and set opt_threads to default to that value.
/// User can then override this with --threads command line option.
static void
hardware_threadlimit_init(void)
{
threads_max = cpucores();
if (threads_max == 0)
threads_max = 1;
return;
}
/// Memory usage limit
static uint64_t memlimit;
extern void
hardware_threadlimit_set(uint32_t threadlimit)
hardware_threadlimit_set(uint32_t new_threadlimit)
{
threads_max = threadlimit;
if (new_threadlimit == 0) {
// The default is the number of available CPU cores.
threadlimit = cpucores();
if (threadlimit == 0)
threadlimit = 1;
} else {
threadlimit = new_threadlimit;
}
return;
}
@ -55,13 +42,30 @@ hardware_threadlimit_set(uint32_t threadlimit)
extern uint32_t
hardware_threadlimit_get(void)
{
return threads_max;
return threadlimit;
}
static void
hardware_memlimit_init(void)
extern void
hardware_memlimit_set(uint64_t new_memlimit)
{
if (new_memlimit == 0) {
// The default is 40 % of total installed physical RAM.
hardware_memlimit_set_percentage(40);
} else {
memlimit = new_memlimit;
}
return;
}
extern void
hardware_memlimit_set_percentage(uint32_t percentage)
{
assert(percentage > 0);
assert(percentage <= 100);
uint64_t mem = physmem();
// If we cannot determine the amount of RAM, assume 32 MiB. Maybe
@ -70,40 +74,22 @@ hardware_memlimit_init(void)
if (mem == 0)
mem = UINT64_C(32) * 1024 * 1024;
// Use at maximum of 90 % of RAM when encoding and 33 % when decoding.
memlimit_encoder = mem - mem / 10;
memlimit_decoder = mem / 3;
return;
}
extern void
hardware_memlimit_set(uint64_t memlimit)
{
memlimit_custom = memlimit;
memlimit = percentage * mem / 100;
return;
}
extern uint64_t
hardware_memlimit_encoder(void)
hardware_memlimit_get(void)
{
return memlimit_custom != 0 ? memlimit_custom : memlimit_encoder;
}
extern uint64_t
hardware_memlimit_decoder(void)
{
return memlimit_custom != 0 ? memlimit_custom : memlimit_decoder;
return memlimit;
}
extern void
hardware_init(void)
{
hardware_memlimit_init();
hardware_threadlimit_init();
hardware_memlimit_set(0);
hardware_threadlimit_set(0);
return;
}

10
src/xz/hardware.h
View File

@ -27,9 +27,9 @@ extern uint32_t hardware_threadlimit_get(void);
/// decoding. Zero indicates resetting the limit back to defaults.
extern void hardware_memlimit_set(uint64_t memlimit);
/// Get the memory usage limit for encoding. By default this is 90 % of RAM.
extern uint64_t hardware_memlimit_encoder(void);
/// Set custom memory usage limit as a percentage of installed RAM.
/// The percentage must be in the range [1, 100].
extern void hardware_memlimit_set_percentage(uint32_t percentage);
/// Get the memory usage limit for decoding. By default this is 30 % of RAM.
extern uint64_t hardware_memlimit_decoder(void);
/// Get the current memory usage limit.
extern uint64_t hardware_memlimit_get(void);

22
src/xz/message.c
View File

@ -1072,6 +1072,7 @@ message_help(bool long_help)
" -e, --extreme use more CPU time when encoding to increase compression\n"
" ratio without increasing memory usage of the decoder"));
if (long_help)
puts(_(
" -M, --memory=NUM use roughly NUM bytes of memory at maximum; 0 indicates\n"
" the default setting, which depends on the operation mode\n"
@ -1085,9 +1086,9 @@ message_help(bool long_help)
|| defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2)
puts(_(
"\n"
" --lzma1=[OPTS] LZMA1 or LZMA2; OPTS is a comma-separated list of zero or\n"
" --lzma2=[OPTS] more of the following options (valid values; default):\n"
" preset=NUM reset options to preset number NUM (1-9)\n"
" --lzma1[=OPTS] LZMA1 or LZMA2; OPTS is a comma-separated list of zero or\n"
" --lzma2[=OPTS] more of the following options (valid values; default):\n"
" preset=NUM reset options to preset number NUM (0-9)\n"
" dict=NUM dictionary size (4KiB - 1536MiB; 8MiB)\n"
" lc=NUM number of literal context bits (0-4; 3)\n"
" lp=NUM number of literal position bits (0-4; 0)\n"
@ -1110,7 +1111,7 @@ message_help(bool long_help)
#if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA)
puts(_(
"\n"
" --delta=[OPTS] Delta filter; valid OPTS (valid values; default):\n"
" --delta[=OPTS] Delta filter; valid OPTS (valid values; default):\n"
" dist=NUM distance between bytes being subtracted\n"
" from each other (1-256; 1)"));
#endif
@ -1118,7 +1119,7 @@ message_help(bool long_help)
#if defined(HAVE_ENCODER_SUBBLOCK) || defined(HAVE_DECODER_SUBBLOCK)
puts(_(
"\n"
" --subblock=[OPTS] Subblock filter; valid OPTS (valid values; default):\n"
" --subblock[=OPTS] Subblock filter; valid OPTS (valid values; default):\n"
" size=NUM number of bytes of data per subblock\n"
" (1 - 256Mi; 4Ki)\n"
" rle=NUM run-length encoder chunk size (0-256; 0)"));
@ -1149,19 +1150,16 @@ message_help(bool long_help)
if (long_help) {
printf(_(
"On this system and configuration, the tool will use at maximum of\n"
" * roughly %'" PRIu64 " MiB RAM for compression;\n"
" * roughly %'" PRIu64 " MiB RAM for decompression; and\n"),
hardware_memlimit_encoder() / (1024 * 1024),
hardware_memlimit_decoder() / (1024 * 1024));
printf(N_(" * one thread for (de)compression.\n\n",
" * %'" PRIu32 " threads for (de)compression.\n\n",
"On this system and configuration, this program will use at maximum of roughly\n"
"%'" PRIu64 " MiB RAM and "), hardware_memlimit_get() / (1024 * 1024));
printf(N_("one thread.\n\n", "%'" PRIu32 " threads.\n\n",
hardware_threadlimit_get()),
hardware_threadlimit_get());
}
printf(_("Report bugs to <%s> (in English or Finnish).\n"),
PACKAGE_BUGREPORT);
printf(_("XZ Utils home page: <http://tukaani.org/xz/>\n"));
my_exit(E_SUCCESS);
}

18
src/xz/process.c
View File

@ -152,15 +152,12 @@ coder_set_compression_settings(void)
// If using --format=raw, we can be decoding. The memusage function
// also validates the filter chain and the options used for the
// filters.
const uint64_t memory_limit = hardware_memlimit_get();
uint64_t memory_usage;
uint64_t memory_limit;
if (opt_mode == MODE_COMPRESS) {
if (opt_mode == MODE_COMPRESS)
memory_usage = lzma_raw_encoder_memusage(filters);
memory_limit = hardware_memlimit_encoder();
} else {
else
memory_usage = lzma_raw_decoder_memusage(filters);
memory_limit = hardware_memlimit_decoder();
}
if (memory_usage == UINT64_MAX)
message_fatal("Unsupported filter chain or filter options");
@ -286,17 +283,17 @@ coder_init(void)
switch (opt_format) {
case FORMAT_AUTO:
ret = lzma_auto_decoder(&strm,
hardware_memlimit_decoder(), flags);
hardware_memlimit_get(), flags);
break;
case FORMAT_XZ:
ret = lzma_stream_decoder(&strm,
hardware_memlimit_decoder(), flags);
hardware_memlimit_get(), flags);
break;
case FORMAT_LZMA:
ret = lzma_alone_decoder(&strm,
hardware_memlimit_decoder());
hardware_memlimit_get());
break;
case FORMAT_RAW:
@ -436,8 +433,7 @@ coder_run(file_pair *pair)
// Figure out how much memory it would have
// actually needed.
uint64_t memusage = lzma_memusage(&strm);
uint64_t memlimit
= hardware_memlimit_decoder();
uint64_t memlimit = hardware_memlimit_get();
// Round the memory limit down and usage up.
// This way we don't display a ridiculous