diff --git a/contrib/linux-kernel/xxhash.diff b/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch similarity index 84% rename from contrib/linux-kernel/xxhash.diff rename to contrib/linux-kernel/0001-lib-Add-xxhash-module.patch index 2aac5c0c..9a8f50a2 100644 --- a/contrib/linux-kernel/xxhash.diff +++ b/contrib/linux-kernel/0001-lib-Add-xxhash-module.patch @@ -1,3 +1,86 @@ +From e75beb7c2e05550b2846e31ad8a0082c188504da Mon Sep 17 00:00:00 2001 +From: Nick Terrell +Date: Wed, 21 Jun 2017 17:27:42 -0700 +Subject: [PATCH 1/4] lib: Add xxhash module + +Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an +extremely fast non-cryptographic hash algorithm for checksumming. +The zstd compression and decompression modules added in the next patch +require xxhash. I extracted it out from zstd since it is useful on its +own. I copied the code from the upstream XXHash source repository and +translated it into kernel style. I ran benchmarks and tests in the kernel +and tests in userland. + +I benchmarked xxhash as a special character device. I ran in four modes, +no-op, xxh32, xxh64, and crc32. The no-op mode simply copies the data to +kernel space and ignores it. The xxh32, xxh64, and crc32 modes compute +hashes on the copied data. I also ran it with four different buffer sizes. +The benchmark file is located in the upstream zstd source repository under +`contrib/linux-kernel/xxhash_test.c` [1]. + +I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. +The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, +16 GB of RAM, and a SSD. I benchmarked using the file `filesystem.squashfs` +from `ubuntu-16.10-desktop-amd64.iso`, which is 1,536,217,088 B large. +Run the following commands for the benchmark: + + modprobe xxhash_test + mknod xxhash_test c 245 0 + time cp filesystem.squashfs xxhash_test + +The time is reported by the time of the userland `cp`. +The GB/s is computed with + + 1,536,217,008 B / time(buffer size, hash) + +which includes the time to copy from userland. +The Normalized GB/s is computed with + + 1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)). + + +| Buffer Size (B) | Hash | Time (s) | GB/s | Adjusted GB/s | +|-----------------|-------|----------|------|---------------| +| 1024 | none | 0.408 | 3.77 | - | +| 1024 | xxh32 | 0.649 | 2.37 | 6.37 | +| 1024 | xxh64 | 0.542 | 2.83 | 11.46 | +| 1024 | crc32 | 1.290 | 1.19 | 1.74 | +| 4096 | none | 0.380 | 4.04 | - | +| 4096 | xxh32 | 0.645 | 2.38 | 5.79 | +| 4096 | xxh64 | 0.500 | 3.07 | 12.80 | +| 4096 | crc32 | 1.168 | 1.32 | 1.95 | +| 8192 | none | 0.351 | 4.38 | - | +| 8192 | xxh32 | 0.614 | 2.50 | 5.84 | +| 8192 | xxh64 | 0.464 | 3.31 | 13.60 | +| 8192 | crc32 | 1.163 | 1.32 | 1.89 | +| 16384 | none | 0.346 | 4.43 | - | +| 16384 | xxh32 | 0.590 | 2.60 | 6.30 | +| 16384 | xxh64 | 0.466 | 3.30 | 12.80 | +| 16384 | crc32 | 1.183 | 1.30 | 1.84 | + +Tested in userland using the test-suite in the zstd repo under +`contrib/linux-kernel/test/XXHashUserlandTest.cpp` [2] by mocking the +kernel functions. A line in each branch of every function in `xxhash.c` +was commented out to ensure that the test-suite fails. Additionally +tested while testing zstd and with SMHasher [3]. + +[1] https://phabricator.intern.facebook.com/P57526246 +[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/XXHashUserlandTest.cpp +[3] https://github.com/aappleby/smhasher + +zstd source repository: https://github.com/facebook/zstd +XXHash source repository: https://github.com/cyan4973/xxhash + +Signed-off-by: Nick Terrell +--- + include/linux/xxhash.h | 236 +++++++++++++++++++++++ + lib/Kconfig | 3 + + lib/Makefile | 1 + + lib/xxhash.c | 500 +++++++++++++++++++++++++++++++++++++++++++++++++ + 4 files changed, 740 insertions(+) + create mode 100644 include/linux/xxhash.h + create mode 100644 lib/xxhash.c + diff --git a/include/linux/xxhash.h b/include/linux/xxhash.h new file mode 100644 index 0000000..9e1f42c @@ -255,10 +338,10 @@ index 0c8b78a..b6009d7 100644 bool depends on AUDIT && !AUDIT_ARCH diff --git a/lib/Makefile b/lib/Makefile -index 320ac46..e16f94a 100644 +index 0166fbc..1338226 100644 --- a/lib/Makefile +++ b/lib/Makefile -@@ -101,6 +101,7 @@ obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o +@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o obj-$(CONFIG_CRC7) += crc7.o obj-$(CONFIG_LIBCRC32C) += libcrc32c.o obj-$(CONFIG_CRC8) += crc8.o @@ -772,3 +855,6 @@ index 0000000..dc94904 + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("xxHash"); +-- +2.9.3 + diff --git a/contrib/linux-kernel/zstd.diff b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch similarity index 98% rename from contrib/linux-kernel/zstd.diff rename to contrib/linux-kernel/0002-lib-Add-zstd-modules.patch index e83b56ed..f94afe36 100644 --- a/contrib/linux-kernel/zstd.diff +++ b/contrib/linux-kernel/0002-lib-Add-zstd-modules.patch @@ -1,3 +1,145 @@ +From b52ae824ae6c0f7c7786380b34da9daaa54bfc26 Mon Sep 17 00:00:00 2001 +From: Nick Terrell +Date: Wed, 21 Jun 2017 17:31:24 -0700 +Subject: [PATCH 2/4] lib: Add zstd modules + +Add zstd compression and decompression kernel modules. +zstd offers a wide varity of compression speed and quality trade-offs. +It can compress at speeds approaching lz4, and quality approaching lzma. +zstd decompressions at speeds more than twice as fast as zlib, and +decompression speed remains roughly the same across all compression levels. + +The code was ported from the upstream zstd source repository. The +`linux/zstd.h` header was modified to match linux kernel style. +The cross-platform and allocation code was stripped out. Instead zstd +requires the caller to pass a preallocated workspace. The source files +were clang-formatted [1] to match the Linux Kernel style as much as +possible. Otherwise, the code was unmodified. We would like to avoid +as much further manual modification to the source code as possible, so it +will be easier to keep the kernel zstd up to date. + +I benchmarked zstd compression as a special character device. I ran zstd +and zlib compression at several levels, as well as performing no +compression, which measure the time spent copying the data to kernel space. +Data is passed to the compresser 4096 B at a time. The benchmark file is +located in the upstream zstd source repository under +`contrib/linux-kernel/zstd_compress_test.c` [2]. + +I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. +The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, +16 GB of RAM, and a SSD. I benchmarked using `silesia.tar` [3], which is +211,988,480 B large. Run the following commands for the benchmark: + + sudo modprobe zstd_compress_test + sudo mknod zstd_compress_test c 245 0 + sudo cp silesia.tar zstd_compress_test + +The time is reported by the time of the userland `cp`. +The MB/s is computed with + + 1,536,217,008 B / time(buffer size, hash) + +which includes the time to copy from userland. +The Adjusted MB/s is computed with + + 1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)). + +The memory reported is the amount of memory the compressor requests. + +| Method | Size (B) | Time (s) | Ratio | MB/s | Adj MB/s | Mem (MB) | +|----------|----------|----------|-------|---------|----------|----------| +| none | 11988480 | 0.100 | 1 | 2119.88 | - | - | +| zstd -1 | 73645762 | 1.044 | 2.878 | 203.05 | 224.56 | 1.23 | +| zstd -3 | 66988878 | 1.761 | 3.165 | 120.38 | 127.63 | 2.47 | +| zstd -5 | 65001259 | 2.563 | 3.261 | 82.71 | 86.07 | 2.86 | +| zstd -10 | 60165346 | 13.242 | 3.523 | 16.01 | 16.13 | 13.22 | +| zstd -15 | 58009756 | 47.601 | 3.654 | 4.45 | 4.46 | 21.61 | +| zstd -19 | 54014593 | 102.835 | 3.925 | 2.06 | 2.06 | 60.15 | +| zlib -1 | 77260026 | 2.895 | 2.744 | 73.23 | 75.85 | 0.27 | +| zlib -3 | 72972206 | 4.116 | 2.905 | 51.50 | 52.79 | 0.27 | +| zlib -6 | 68190360 | 9.633 | 3.109 | 22.01 | 22.24 | 0.27 | +| zlib -9 | 67613382 | 22.554 | 3.135 | 9.40 | 9.44 | 0.27 | + +I benchmarked zstd decompression using the same method on the same machine. +The benchmark file is located in the upstream zstd repo under +`contrib/linux-kernel/zstd_decompress_test.c` [4]. The memory reported is +the amount of memory required to decompress data compressed with the given +compression level. If you know the maximum size of your input, you can +reduce the memory usage of decompression irrespective of the compression +level. + +| Method | Time (s) | MB/s | Adjusted MB/s | Memory (MB) | +|----------|----------|---------|---------------|-------------| +| none | 0.025 | 8479.54 | - | - | +| zstd -1 | 0.358 | 592.15 | 636.60 | 0.84 | +| zstd -3 | 0.396 | 535.32 | 571.40 | 1.46 | +| zstd -5 | 0.396 | 535.32 | 571.40 | 1.46 | +| zstd -10 | 0.374 | 566.81 | 607.42 | 2.51 | +| zstd -15 | 0.379 | 559.34 | 598.84 | 4.61 | +| zstd -19 | 0.412 | 514.54 | 547.77 | 8.80 | +| zlib -1 | 0.940 | 225.52 | 231.68 | 0.04 | +| zlib -3 | 0.883 | 240.08 | 247.07 | 0.04 | +| zlib -6 | 0.844 | 251.17 | 258.84 | 0.04 | +| zlib -9 | 0.837 | 253.27 | 287.64 | 0.04 | + +Tested in userland using the test-suite in the zstd repo under +`contrib/linux-kernel/test/UserlandTest.cpp` [5] by mocking the kernel +functions. Fuzz tested using libfuzzer [6] with the fuzz harnesses under +`contrib/linux-kernel/test/{RoundTripCrash.c,DecompressCrash.c}` [7] [8] +with ASAN, UBSAN, and MSAN. Additionaly, it was tested while testing the +BtrFS and SquashFS patches coming next. + +[1] https://clang.llvm.org/docs/ClangFormat.html +[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/zstd_compress_test.c +[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia +[4] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/zstd_decompress_test.c +[5] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/UserlandTest.cpp +[6] http://llvm.org/docs/LibFuzzer.html +[7] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/RoundTripCrash.c +[8] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/DecompressCrash.c + +zstd source repository: https://github.com/facebook/zstd + +Signed-off-by: Nick Terrell +--- + include/linux/zstd.h | 1157 +++++++++++++++ + lib/Kconfig | 8 + + lib/Makefile | 2 + + lib/zstd/Makefile | 18 + + lib/zstd/bitstream.h | 374 +++++ + lib/zstd/compress.c | 3468 +++++++++++++++++++++++++++++++++++++++++++++ + lib/zstd/decompress.c | 2514 ++++++++++++++++++++++++++++++++ + lib/zstd/entropy_common.c | 244 ++++ + lib/zstd/error_private.h | 53 + + lib/zstd/fse.h | 584 ++++++++ + lib/zstd/fse_compress.c | 857 +++++++++++ + lib/zstd/fse_decompress.c | 313 ++++ + lib/zstd/huf.h | 203 +++ + lib/zstd/huf_compress.c | 731 ++++++++++ + lib/zstd/huf_decompress.c | 920 ++++++++++++ + lib/zstd/mem.h | 151 ++ + lib/zstd/zstd_common.c | 75 + + lib/zstd/zstd_internal.h | 269 ++++ + lib/zstd/zstd_opt.h | 1014 +++++++++++++ + 19 files changed, 12955 insertions(+) + create mode 100644 include/linux/zstd.h + create mode 100644 lib/zstd/Makefile + create mode 100644 lib/zstd/bitstream.h + create mode 100644 lib/zstd/compress.c + create mode 100644 lib/zstd/decompress.c + create mode 100644 lib/zstd/entropy_common.c + create mode 100644 lib/zstd/error_private.h + create mode 100644 lib/zstd/fse.h + create mode 100644 lib/zstd/fse_compress.c + create mode 100644 lib/zstd/fse_decompress.c + create mode 100644 lib/zstd/huf.h + create mode 100644 lib/zstd/huf_compress.c + create mode 100644 lib/zstd/huf_decompress.c + create mode 100644 lib/zstd/mem.h + create mode 100644 lib/zstd/zstd_common.c + create mode 100644 lib/zstd/zstd_internal.h + create mode 100644 lib/zstd/zstd_opt.h + diff --git a/include/linux/zstd.h b/include/linux/zstd.h new file mode 100644 index 0000000..249575e @@ -1168,7 +1310,7 @@ index b6009d7..f00ddab 100644 @@ -241,6 +241,14 @@ config LZ4HC_COMPRESS config LZ4_DECOMPRESS tristate - + +config ZSTD_COMPRESS + select XXHASH + tristate @@ -1178,13 +1320,13 @@ index b6009d7..f00ddab 100644 + tristate + source "lib/xz/Kconfig" - + # diff --git a/lib/Makefile b/lib/Makefile -index e16f94a..0cfd529 100644 +index 1338226..4fcef16 100644 --- a/lib/Makefile +++ b/lib/Makefile -@@ -115,6 +115,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/ +@@ -116,6 +116,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/ obj-$(CONFIG_LZ4_COMPRESS) += lz4/ obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/ obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/ @@ -1192,7 +1334,7 @@ index e16f94a..0cfd529 100644 +obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/ obj-$(CONFIG_XZ_DEC) += xz/ obj-$(CONFIG_RAID6_PQ) += raid6/ - + diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile new file mode 100644 index 0000000..dd0a359 @@ -13077,3 +13219,5 @@ index 0000000..55e1b4c +} + +#endif /* ZSTD_OPT_H_91842398743 */ +-- +2.9.3 diff --git a/contrib/linux-kernel/btrfs.diff b/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch similarity index 80% rename from contrib/linux-kernel/btrfs.diff rename to contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch index aa3ec1d9..53d03d3c 100644 --- a/contrib/linux-kernel/btrfs.diff +++ b/contrib/linux-kernel/0003-btrfs-Add-zstd-support.patch @@ -1,3 +1,83 @@ +From 599f8f2aaace3df939cb145368574a52268d82d0 Mon Sep 17 00:00:00 2001 +From: Nick Terrell +Date: Wed, 21 Jun 2017 17:31:39 -0700 +Subject: [PATCH 3/4] btrfs: Add zstd support + +Add zstd compression and decompression support to BtrFS. zstd at its +fastest level compresses almost as well as zlib, while offering much +faster compression and decompression, approaching lzo speeds. + +I benchmarked btrfs with zstd compression against no compression, lzo +compression, and zlib compression. I benchmarked two scenarios. Copying +a set of files to btrfs, and then reading the files. Copying a tarball +to btrfs, extracting it to btrfs, and then reading the extracted files. +After every operation, I call `sync` and include the sync time. +Between every pair of operations I unmount and remount the filesystem +to avoid caching. The benchmark files can be found in the upstream +zstd source repository under +`contrib/linux-kernel/{btrfs-benchmark.sh,btrfs-extract-benchmark.sh}` +[1] [2]. + +I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. +The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, +16 GB of RAM, and a SSD. + +The first compression benchmark is copying 10 copies of the unzipped +Silesia corpus [3] into a BtrFS filesystem mounted with +`-o compress-force=Method`. The decompression benchmark times how long +it takes to `tar` all 10 copies into `/dev/null`. The compression ratio is +measured by comparing the output of `df` and `du`. See the benchmark file +[1] for details. I benchmarked multiple zstd compression levels, although +the patch uses zstd level 1. + +| Method | Ratio | Compression MB/s | Decompression speed | +|---------|-------|------------------|---------------------| +| None | 0.99 | 504 | 686 | +| lzo | 1.66 | 398 | 442 | +| zlib | 2.58 | 65 | 241 | +| zstd 1 | 2.57 | 260 | 383 | +| zstd 3 | 2.71 | 174 | 408 | +| zstd 6 | 2.87 | 70 | 398 | +| zstd 9 | 2.92 | 43 | 406 | +| zstd 12 | 2.93 | 21 | 408 | +| zstd 15 | 3.01 | 11 | 354 | + +The next benchmark first copies `linux-4.11.6.tar` [4] to btrfs. Then it +measures the compression ratio, extracts the tar, and deletes the tar. +Then it measures the compression ratio again, and `tar`s the extracted +files into `/dev/null`. See the benchmark file [2] for details. + +| Method | Tar Ratio | Extract Ratio | Copy (s) | Extract (s)| Read (s) | +|--------|-----------|---------------|----------|------------|----------| +| None | 0.97 | 0.78 | 0.981 | 5.501 | 8.807 | +| lzo | 2.06 | 1.38 | 1.631 | 8.458 | 8.585 | +| zlib | 3.40 | 1.86 | 7.750 | 21.544 | 11.744 | +| zstd 1 | 3.57 | 1.85 | 2.579 | 11.479 | 9.389 | + +[1] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-benchmark.sh +[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-extract-benchmark.sh +[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia +[4] https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.11.6.tar.xz + +zstd source repository: https://github.com/facebook/zstd + +Signed-off-by: Nick Terrell +--- + fs/btrfs/Kconfig | 2 + + fs/btrfs/Makefile | 2 +- + fs/btrfs/compression.c | 1 + + fs/btrfs/compression.h | 6 +- + fs/btrfs/ctree.h | 1 + + fs/btrfs/disk-io.c | 2 + + fs/btrfs/ioctl.c | 6 +- + fs/btrfs/props.c | 6 + + fs/btrfs/super.c | 12 +- + fs/btrfs/sysfs.c | 2 + + fs/btrfs/zstd.c | 433 +++++++++++++++++++++++++++++++++++++++++++++ + include/uapi/linux/btrfs.h | 8 +- + 12 files changed, 469 insertions(+), 12 deletions(-) + create mode 100644 fs/btrfs/zstd.c + diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig index 80e9c18..a26c63b 100644 --- a/fs/btrfs/Kconfig @@ -25,7 +105,7 @@ index 128ce17..962a95a 100644 reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ uuid-tree.o props.o hash.o free-space-tree.o diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c -index c7721a6..66d4ced 100644 +index 10e6b28..3beb0d0 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -761,6 +761,7 @@ static struct { @@ -34,7 +114,7 @@ index c7721a6..66d4ced 100644 &btrfs_lzo_compress, + &btrfs_zstd_compress, }; - + void __init btrfs_init_compress(void) diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index 39ec43a..d99fc21 100644 @@ -50,20 +130,20 @@ index 39ec43a..d99fc21 100644 + BTRFS_COMPRESS_TYPES = 3, + BTRFS_COMPRESS_LAST = 4, }; - + struct btrfs_compress_op { @@ -92,5 +93,6 @@ struct btrfs_compress_op { - + extern const struct btrfs_compress_op btrfs_zlib_compress; extern const struct btrfs_compress_op btrfs_lzo_compress; +extern const struct btrfs_compress_op btrfs_zstd_compress; - + #endif diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h -index 29b7fc2..878b23b9 100644 +index 4f8f75d..61dd3dd 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h -@@ -270,6 +270,7 @@ struct btrfs_super_block { +@@ -271,6 +271,7 @@ struct btrfs_super_block { BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \ BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \ BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \ @@ -72,24 +152,24 @@ index 29b7fc2..878b23b9 100644 BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \ BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \ diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c -index 08b74da..0c43e4e 100644 +index 5f678dc..49c0e91 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c -@@ -2853,6 +2853,8 @@ int open_ctree(struct super_block *sb, +@@ -2831,6 +2831,8 @@ int open_ctree(struct super_block *sb, features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF; if (fs_info->compress_type == BTRFS_COMPRESS_LZO) features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; -+ else if (tree_root->fs_info->compress_type == BTRFS_COMPRESS_ZSTD) ++ else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD) + features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD; - + if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA) btrfs_info(fs_info, "has skinny extents"); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c -index dabfc7a..d8ea727 100644 +index e176375..f732cfd 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg) - + if (fs_info->compress_type == BTRFS_COMPRESS_LZO) comp = "lzo"; - else @@ -101,13 +181,13 @@ index dabfc7a..d8ea727 100644 comp, strlen(comp), 0); if (ret) @@ -1463,6 +1465,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, - + if (range->compress_type == BTRFS_COMPRESS_LZO) { btrfs_set_fs_incompat(fs_info, COMPRESS_LZO); + } else if (range->compress_type == BTRFS_COMPRESS_ZSTD) { + btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD); } - + ret = defrag_count; diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c index d6cb155..162105f 100644 @@ -119,7 +199,7 @@ index d6cb155..162105f 100644 return 0; + else if (!strncmp("zstd", value, len)) + return 0; - + return -EINVAL; } @@ -405,6 +407,8 @@ static int prop_compression_apply(struct inode *inode, @@ -130,7 +210,7 @@ index d6cb155..162105f 100644 + type = BTRFS_COMPRESS_ZSTD; else return -EINVAL; - + @@ -422,6 +426,8 @@ static const char *prop_compression_extract(struct inode *inode) return "zlib"; case BTRFS_COMPRESS_LZO: @@ -138,10 +218,10 @@ index d6cb155..162105f 100644 + case BTRFS_COMPRESS_ZSTD: + return "zstd"; } - + return NULL; diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c -index da687dc..b064456 100644 +index 4f1cdd5..4f792d5 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options, @@ -159,7 +239,7 @@ index da687dc..b064456 100644 } else if (strncmp(args[0].from, "no", 2) == 0) { compress_type = "no"; btrfs_clear_opt(info->mount_opt, COMPRESS); -@@ -1230,8 +1238,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) +@@ -1240,8 +1248,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) if (btrfs_test_opt(info, COMPRESS)) { if (info->compress_type == BTRFS_COMPRESS_ZLIB) compress_type = "zlib"; @@ -193,7 +273,7 @@ index 1f157fb..b0dec90 100644 BTRFS_FEAT_ATTR_PTR(raid56), diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c new file mode 100644 -index 0000000..45ea326 +index 0000000..838741b --- /dev/null +++ b/fs/btrfs/zstd.c @@ -0,0 +1,433 @@ @@ -231,7 +311,7 @@ index 0000000..45ea326 + +static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len) +{ -+ ZSTD_parameters params = ZSTD_getParams(3, src_len, 0); ++ ZSTD_parameters params = ZSTD_getParams(1, src_len, 0); + + if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) + params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; @@ -631,7 +711,7 @@ index 0000000..45ea326 + .decompress = zstd_decompress, +}; diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h -index db4c253..f26c34f 100644 +index a456e53..992c150 100644 --- a/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h @@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args { @@ -646,6 +726,8 @@ index db4c253..f26c34f 100644 - */ -#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4) +#define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4) - + /* * older kernels tried to do bigger metadata blocks, but the +-- +2.9.3 diff --git a/contrib/linux-kernel/squashfs.diff b/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch similarity index 70% rename from contrib/linux-kernel/squashfs.diff rename to contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch index b524819d..e9c4b98c 100644 --- a/contrib/linux-kernel/squashfs.diff +++ b/contrib/linux-kernel/0004-squashfs-Add-zstd-support.patch @@ -1,11 +1,67 @@ +From 5ff6a64abaea7b7f11d37cb0fdf08642316a3a90 Mon Sep 17 00:00:00 2001 +From: Nick Terrell +Date: Mon, 12 Jun 2017 12:18:23 -0700 +Subject: [PATCH 4/4] squashfs: Add zstd support + +Add zstd compression and decompression support to SquashFS. zstd is a +great fit for SquashFS because it can compress at ratios approaching xz, +while decompressing twice as fast as zlib. For SquashFS in particular, +it can decompress as fast as lzo and lz4. It also has the flexibility +to turn down the compression ratio for faster compression times. + +The compression benchmark is run on the file tree from the SquashFS archive +found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the +default block size (128 KB) and and various compression algorithms/levels. +xz and zstd are also benchmarked with 256 KB blocks. The decompression +benchmark times how long it takes to `tar` the file tree into `/dev/null`. +See the benchmark file in the upstream zstd source repository located under +`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details. + +I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. +The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, +16 GB of RAM, and a SSD. + +| Method | Ratio | Compression MB/s | Decompression MB/s | +|----------------|-------|------------------|--------------------| +| gzip | 2.92 | 15 | 128 | +| lzo | 2.64 | 9.5 | 217 | +| lz4 | 2.12 | 94 | 218 | +| xz | 3.43 | 5.5 | 35 | +| xz 256 KB | 3.53 | 5.4 | 40 | +| zstd 1 | 2.71 | 96 | 210 | +| zstd 5 | 2.93 | 69 | 198 | +| zstd 10 | 3.01 | 41 | 225 | +| zstd 15 | 3.13 | 11.4 | 224 | +| zstd 16 256 KB | 3.24 | 8.1 | 210 | + +This patch was written by Sean Purcell , but I will be +taking over the submission process. + +[1] http://releases.ubuntu.com/16.10/ +[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh + +zstd source repository: https://github.com/facebook/zstd + +Cc: Sean Purcell +Signed-off-by: Nick Terrell +--- + fs/squashfs/Kconfig | 14 +++++ + fs/squashfs/Makefile | 1 + + fs/squashfs/decompressor.c | 7 +++ + fs/squashfs/decompressor.h | 4 ++ + fs/squashfs/squashfs_fs.h | 1 + + fs/squashfs/zstd_wrapper.c | 150 +++++++++++++++++++++++++++++++++++++++++++++ + 6 files changed, 177 insertions(+) + create mode 100644 fs/squashfs/zstd_wrapper.c + diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig index ffb093e..1adb334 100644 --- a/fs/squashfs/Kconfig +++ b/fs/squashfs/Kconfig @@ -165,6 +165,20 @@ config SQUASHFS_XZ - + If unsure, say N. - + +config SQUASHFS_ZSTD + bool "Include support for ZSTD compressed file systems" + depends on SQUASHFS @@ -39,7 +95,7 @@ index d2bc136..8366398 100644 @@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = { }; #endif - + +#ifndef CONFIG_SQUASHFS_ZSTD +static const struct squashfs_decompressor squashfs_zstd_comp_ops = { + NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0 @@ -56,7 +112,7 @@ index d2bc136..8366398 100644 + &squashfs_zstd_comp_ops, &squashfs_unknown_comp_ops }; - + diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h index a25713c..0f5a8e4 100644 --- a/fs/squashfs/decompressor.h @@ -64,7 +120,7 @@ index a25713c..0f5a8e4 100644 @@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops; extern const struct squashfs_decompressor squashfs_zlib_comp_ops; #endif - + +#ifdef CONFIG_SQUASHFS_ZSTD +extern const struct squashfs_decompressor squashfs_zstd_comp_ops; +#endif @@ -79,7 +135,7 @@ index 506f4ba..24d12fd 100644 #define XZ_COMPRESSION 4 #define LZ4_COMPRESSION 5 +#define ZSTD_COMPRESSION 6 - + struct squashfs_super_block { __le32 s_magic; diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c @@ -238,3 +294,5 @@ index 0000000..8cb7c76 + .name = "zstd", + .supported = 1 +}; +-- +2.9.3 diff --git a/contrib/linux-kernel/btrfs-extract-benchmark.sh b/contrib/linux-kernel/btrfs-extract-benchmark.sh new file mode 100755 index 00000000..69721d09 --- /dev/null +++ b/contrib/linux-kernel/btrfs-extract-benchmark.sh @@ -0,0 +1,99 @@ +# !/bin/sh +set -e + +# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. +# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and +# 16 GB of RAM and an SSD. + +# silesia is a directory that can be downloaded from +# http://mattmahoney.net/dc/silesia.html +# ls -l silesia/ +# total 203M +# -rwxr-xr-x 1 terrelln 9.8M Apr 12 2002 dickens +# -rwxr-xr-x 1 terrelln 49M May 31 2002 mozilla +# -rwxr-xr-x 1 terrelln 9.6M Mar 20 2003 mr +# -rwxr-xr-x 1 terrelln 32M Apr 2 2002 nci +# -rwxr-xr-x 1 terrelln 5.9M Jul 4 2002 ooffice +# -rwxr-xr-x 1 terrelln 9.7M Apr 11 2002 osdb +# -rwxr-xr-x 1 terrelln 6.4M Apr 2 2002 reymont +# -rwxr-xr-x 1 terrelln 21M Mar 25 2002 samba +# -rwxr-xr-x 1 terrelln 7.0M Mar 24 2002 sao +# -rwxr-xr-x 1 terrelln 40M Mar 25 2002 webster +# -rwxr-xr-x 1 terrelln 8.1M Apr 4 2002 x-ray +# -rwxr-xr-x 1 terrelln 5.1M Nov 30 2000 xml + +# $HOME is on a ext4 filesystem +BENCHMARK_FILE="linux-4.11.6.tar" +BENCHMARK_DIR="$HOME/$BENCHMARK_FILE" + +# Normalize the environment +sudo umount /mnt/btrfs 2> /dev/null > /dev/null || true +sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs +sudo rm -rf /mnt/btrfs/* +sync +sudo umount /mnt/btrfs +sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs + +# Run the benchmark +echo "Copy" +time sh -c "sudo cp -r $BENCHMARK_DIR /mnt/btrfs/$BENCHMARK_FILE && sync" + +echo "Approximate tarred compression ratio" +printf "%d / %d\n" \ + $(df /mnt/btrfs --output=used -B 1 | tail -n 1) \ + $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1); + +# Unmount and remount to avoid any caching +sudo umount /mnt/btrfs +sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs + +echo "Extract" +time sh -c "sudo tar -C /mnt/btrfs -xf /mnt/btrfs/$BENCHMARK_FILE && sync" + +# Remove the tarball, leaving only the extracted data +sudo rm /mnt/btrfs/$BENCHMARK_FILE +# Unmount and remount to avoid any caching +sudo umount /mnt/btrfs +sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs + +echo "Approximate extracted compression ratio" +printf "%d / %d\n" \ + $(df /mnt/btrfs --output=used -B 1 | tail -n 1) \ + $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1); + +echo "Read" +time sudo tar -c /mnt/btrfs 2> /dev/null | wc -c > /dev/null + +sudo rm -rf /mnt/btrfs/* +sudo umount /mnt/btrfs + +# Run for each of -o compress-force={none, lzo, zlib, zstd} 5 times and take the +# min time and ratio. + +# none +# copy: 0.981 s +# extract: 5.501 s +# read: 8.807 s +# tarball ratio: 0.97 +# extracted ratio: 0.78 + +# lzo +# copy: 1.631 s +# extract: 8.458 s +# read: 8.585 s +# tarball ratio: 2.06 +# extracted ratio: 1.38 + +# zlib +# copy: 7.750 s +# extract: 21.544 s +# read: 11.744 s +# tarball ratio : 3.40 +# extracted ratio: 1.86 + +# zstd 1 +# copy: 2.579 s +# extract: 11.479 s +# read: 9.389 s +# tarball ratio : 3.57 +# extracted ratio: 1.85 diff --git a/contrib/linux-kernel/fs/btrfs/zstd.c b/contrib/linux-kernel/fs/btrfs/zstd.c index 45ea3263..838741b3 100644 --- a/contrib/linux-kernel/fs/btrfs/zstd.c +++ b/contrib/linux-kernel/fs/btrfs/zstd.c @@ -32,7 +32,7 @@ static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len) { - ZSTD_parameters params = ZSTD_getParams(3, src_len, 0); + ZSTD_parameters params = ZSTD_getParams(1, src_len, 0); if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;